mirror of
https://github.com/tiyn/stud.ip-crawler.git
synced 2025-07-03 03:07:49 +02:00
fa36e0f29e
- mysql creates database and tables to given mysql if not existent already - mysql reads last change values from db - mysql saves ch_date after downloading - run now takes care for the variables of mysql and studip
48 lines
1.7 KiB
Python
Executable File
48 lines
1.7 KiB
Python
Executable File
#!/bin/env python3
|
|
import os
|
|
import argparse
|
|
|
|
import studip
|
|
import mysql
|
|
|
|
|
|
parser = argparse.ArgumentParser(description='Download Files from StudIP.')
|
|
parser.add_argument('-o', '--output', type=str,
|
|
default='./data', help='path to output directory')
|
|
parser.add_argument('-u', '--user', type=str,
|
|
help='studip username', required=True)
|
|
parser.add_argument('-p', '--passwd', type=str,
|
|
help='studip password', required=True)
|
|
parser.add_argument('-s', '--url', type=str, help='studip url', required=True)
|
|
parser.add_argument('--chunk', type=int, default=1024 *
|
|
1024, help='chunksize for downloading data')
|
|
parser.add_argument('-r', '--reset_dl_date', action='store_true', help='downloads everything and ignores last download date')
|
|
parser.add_argument('--host', type=str, default='localhost', help='mysql host')
|
|
parser.add_argument('--port', type=int, default=3306, help='mysql port')
|
|
parser.add_argument('--db_name', type=str, default='studip', help='mysql database name')
|
|
parser.add_argument('--db_user', type=str, default='root', help='mysql database user')
|
|
parser.add_argument('--db_passwd', type=str, default='secret-pw', help='mysql database password')
|
|
args = parser.parse_args()
|
|
|
|
BASE_DIR = os.path.abspath(args.output)
|
|
USERNAME = args.user
|
|
PASSWORD = args.passwd
|
|
|
|
db = mysql.Database()
|
|
|
|
db.HOST = args.host
|
|
db.PORT = args.port
|
|
db.DB_NAME = args.db_name
|
|
db.USER = args.db_user
|
|
db.PASSW = args.db_passwd
|
|
db.RESET_DL_DATE = args.reset_dl_date
|
|
db.setup_db()
|
|
|
|
crwlr = studip.Crawler(db)
|
|
|
|
crwlr.CHUNK_SIZE = args.chunk
|
|
crwlr.STUDIP_DOMAIN = args.url
|
|
crwlr.USER = (USERNAME, PASSWORD)
|
|
|
|
crwlr.download_curr_courses(BASE_DIR)
|