database: files id and chdates are stored

- mysql creates database and tables to given mysql if not existent already
- mysql reads last change values from db
- mysql saves ch_date after downloading
- run now takes care for the variables of mysql and studip

src/studip.py

@@ -0,0 +1,162 @@
+#!/bin/env python3
+import time
+import os
+import argparse
+
+from tqdm import tqdm
+import requests as req
+from requests.auth import HTTPBasicAuth
+
+
+class Crawler:
+
+    def __init__(self, db):
+        self.CHUNK_SIZE = None
+        self.STUDIP_DOMAIN = None
+        self.USER = None
+        self.db = db
+
+    def create_dir(self, dir):
+        if not os.path.exists(dir):
+            print('creating folder', dir)
+            os.mkdir(dir)
+
+    def get_uid(self):
+        url = self.STUDIP_DOMAIN + '/api.php/user/'
+        rsp = req.get(url, auth=self.USER)
+        user_id = rsp.json()['user_id']
+        return user_id
+
+    def get_curr_semester(self):
+        url = self.STUDIP_DOMAIN + '/api.php/semesters/'
+        rsp = req.get(url, auth=self.USER)
+        curr_time = int(str(int(time.time())))
+        semesters = rsp.json()['collection']
+        for sem_uri in semesters:
+            semester = semesters[sem_uri]
+            sem_begin = semester['begin']
+            sem_end = semester['end']
+            if sem_begin < curr_time < sem_end:
+                return sem_uri
+        return 0
+
+    def get_ordered_semesters(self):
+        url = self.STUDIP_DOMAIN + '/api.php/semesters/'
+        rsp = req.get(url, auth=self.USER)
+        semesters = rsp.json()['collection']
+        order_sems = []
+        for sem_uri in semesters:
+            order_sems.append(sem_uri)
+        return order_sems
+
+    def get_curr_courses(self, user_id, semester):
+        url = self.STUDIP_DOMAIN + '/api.php/user/' + user_id + '/courses'
+        rsp = req.get(url, auth=self.USER)
+        ord_sems = self.get_ordered_semesters()
+        courses = rsp.json()['collection']
+        i = 0
+        course_list = {}
+        for course_uri in courses:
+            course = courses[course_uri]
+            start_sem = course['start_semester']
+            if start_sem != None:
+                start_ind = ord_sems.index(start_sem)
+            else:
+                start_ind = 100
+            end_sem = course['end_semester']
+            if end_sem != None:
+                end_ind = ord_sems.index(end_sem)
+            else:
+                end_ind = 100
+            curr_ind = ord_sems.index(semester)
+            if start_ind <= curr_ind <= end_ind:
+                course_title = course['title']
+                course_id = course['course_id']
+                course_list[course_id] = course_title
+        return course_list
+
+    def get_top_folder(self, course):
+        url = self.STUDIP_DOMAIN + '/api.php/course/' + course + '/top_folder'
+        rsp = req.get(url, auth=self.USER)
+        top_folder = rsp.json()
+        tf_id = top_folder['id']
+        return(tf_id)
+
+    def get_docs(self, folder):
+        url = self.STUDIP_DOMAIN + '/api.php/folder/' + folder
+        rsp = req.get(url, auth=self.USER)
+        docs = rsp.json()['file_refs']
+        res_docs = []
+        for doc in docs:
+            doc_id = doc['id']
+            res_docs.append(doc_id)
+        return(res_docs)
+
+    def download(self, doc):
+        url1 = self.STUDIP_DOMAIN + '/api.php/file/' + doc
+        rsp1 = req.get(url1, auth=self.USER)
+        doc_name = rsp1.json()['name']
+        doc_chdate = rsp1.json()['chdate']
+        last_dl = self.db.get_last_file_dl(doc)
+        if last_dl == None or last_dl < doc_chdate:
+            print('downloading ', doc_name)
+            url2 = self.STUDIP_DOMAIN + '/api.php/file/' + doc + '/download'
+            rsp2 = req.get(url2, auth=self.USER, stream=True)
+            total_size = int(rsp2.headers.get('content-length', 0))
+            progbar = tqdm(total=total_size, unit='iB', unit_scale=True)
+            with open(doc_name, 'wb') as doc_file:
+                for chunk in rsp2.iter_content(self.CHUNK_SIZE):
+                    progbar.update(len(chunk))
+                    doc_file.write(chunk)
+            self.db.set_last_file_dl(str(doc), str(int(time.time())))
+
+    def get_subdirs(self, folder):
+        url = self.STUDIP_DOMAIN + '/api.php/folder/' + folder
+        rsp = req.get(url, auth=self.USER)
+        subdirs = rsp.json()['subfolders']
+        docs = rsp.json()['file_refs']
+        res_subdirs = {}
+        for subdir in subdirs:
+            sub_id = subdir['id']
+            sub_name = subdir['name']
+            res_subdirs[sub_id] = sub_name
+        return res_subdirs
+
+    def download_folder(self, folder):
+        docs = self.get_docs(folder)
+        for doc in docs:
+            print('found doc ', doc)
+            self.download(doc)
+
+    def download_folder_rec(self, folder, base_dir):
+        print('folder ', folder)
+        self.create_dir(base_dir)
+        self.download_folder(folder)
+        subdirs = self.get_subdirs(folder)
+        os.chdir(base_dir)
+        for subdir in subdirs:
+            subdir_name = subdirs[subdir].replace('/', '-')
+            subdir_path = os.path.join(base_dir, subdir_name)
+            print(subdir_path)
+            self.create_dir(subdir_path)
+            os.chdir(subdir_path)
+            self.download_folder_rec(subdir, subdir_path)
+
+    def download_course(self, course, base_dir):
+        print('course ', course)
+        self.create_dir(base_dir)
+        os.chdir(base_dir)
+        root = self.get_top_folder(course)
+        self.download_folder_rec(root, base_dir)
+
+    def download_curr_courses(self, base_dir):
+        print('Start downloading all current courses')
+        self.create_dir(base_dir)
+        curr_courses = self.get_curr_courses(
+            self.get_uid(), self.get_curr_semester())
+        os.chdir(base_dir)
+        for course in curr_courses:
+            print('course is ', curr_courses[course])
+            course_name = curr_courses[course].replace('/', '-')
+            path = os.path.join(base_dir, course_name)
+            self.download_course(course, path)