tiyn/stud.ip-crawler
1
0
Fork 0
mirror of https://github.com/tiyn/stud.ip-crawler.git synced 2026-02-22 06:34:48 +01:00
Code Issues Projects Releases Wiki Activity

log: added logging for jobs to finish

Browse Source
This commit is contained in:
TiynGER
2020-11-10 11:19:24 +01:00
parent bd4c9ab497
commit 719c89e7cf
3 changed files with 22 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
src/crawler.py
View File

@@ -10,8 +10,7 @@ class Crawler:
self.studip = studip
def download_folder(self, folder):
"""
Download all documents in a folder.
"""Download all documents in a folder.
Parameters:
folder(string): id of the folder to download
@@ -22,8 +21,7 @@ class Crawler:
self.studip.download(doc)
def download_folder_rec(self, folder, base_dir):
"""
Download all documents in a folder and its subfolders.
"""Download all documents in a folder and its subfolders.
This keeps the folder structure.
Parameters:
@@ -42,10 +40,10 @@ class Crawler:
self.create_dir(subdir_path)
os.chdir(subdir_path)
self.download_folder_rec(subdir, subdir_path)
log.info('Finished crawling folder ' + folder)
def download_course(self, course, base_dir):
"""
Download all documents in course.
"""Download all documents in course.
This keeps the folder structure.
Parameters:
@@ -57,29 +55,29 @@ class Crawler:
os.chdir(base_dir)
root = self.studip.get_top_folder(course)
self.download_folder_rec(root, base_dir)
log.info('Finished crawling course ' + course)
def download_curr_courses(self, base_dir):
"""
Download all documents of all current courses.
"""Download all documents of all current courses.
This keeps the folder structure.
Parameters:
base_dir(string): directory where to put the download
"""
log.info('Start crawling all current courses')
self.create_dir(base_dir)
curr_courses = self.studip.get_curr_courses(
self.studip.get_uid(), self.studip.get_curr_semester())
log.info('crawling all current courses' + curr_courses)
os.chdir(base_dir)
for course in curr_courses:
log.debug('course is ' + curr_courses[course])
course_name = curr_courses[course].replace('/', '-')
path = os.path.join(base_dir, course_name)
self.download_course(course, path)
log.info('Finished crawling all current courses')
def create_dir(self, dir):
"""
Creates a dir if it doesnt exist already.
"""Creates a dir if it doesnt exist already.
Parameters:
dir(string): directory path to create