tiyn/stud.ip-crawler
1
0
Fork 0
mirror of https://github.com/tiyn/stud.ip-crawler.git synced 2026-02-22 06:34:48 +01:00
Code Issues Projects Releases Wiki Activity

Documentation: added docstrings

Browse Source
This commit is contained in:
TiynGER
2020-10-25 16:22:02 +01:00
parent a5d7b0304b
commit 6f9ef9bbd6
3 changed files with 123 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
src/crawler.py
View File

@@ -10,12 +10,26 @@ class Crawler:
self.studip = studip
def download_folder(self, folder):
"""
Download all documents in a folder.
Parameters:
folder(string): id of the folder to download
"""
docs = self.studip.get_docs(folder)
for doc in docs:
log.info('found doc ' + doc)
self.studip.download(doc)
def download_folder_rec(self, folder, base_dir):
"""
Download all documents in a folder and its subfolders.
This keeps the folder structure.
Parameters:
folder(string): id of the folder to download
base_dir(string): directory where to put the download
"""
log.info('crawling folder ' + folder)
self.create_dir(base_dir)
self.download_folder(folder)
@@ -30,6 +44,14 @@ class Crawler:
self.download_folder_rec(subdir, subdir_path)
def download_course(self, course, base_dir):
"""
Download all documents in course.
This keeps the folder structure.
Parameters:
course(string): id of the course to download
base_dir(string): directory where to put the download
"""
log.info('crawling course ' + course)
self.create_dir(base_dir)
os.chdir(base_dir)
@@ -37,6 +59,13 @@ class Crawler:
self.download_folder_rec(root, base_dir)
def download_curr_courses(self, base_dir):
"""
Download all documents of all current courses.
This keeps the folder structure.
Parameters:
base_dir(string): directory where to put the download
"""
log.info('Start crawling all current courses')
self.create_dir(base_dir)
curr_courses = self.studip.get_curr_courses(
@@ -49,6 +78,12 @@ class Crawler:
self.download_course(course, path)
def create_dir(self, dir):
"""
Creates a dir if it doesnt exist already.
Parameters:
dir(string): directory path to create
"""
if not os.path.exists(dir):
log.info('creating folder' + dir)
os.mkdir(dir)