tiyn
/
stud.ip-crawler
mirror of https://github.com/tiyn/stud.ip-crawler
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

log: added logging for jobs to finish

Browse Source
master
TiynGER 4 years ago
parent bd4c9ab497
commit 719c89e7cf
3 changed files with 22 additions and 37 deletions
Show Stats Download Patch File Download Diff File

20
src/crawler.py
Unescape View File

@ -10,8 +10,7 @@ class Crawler:
self.studip = studip self.studip = studip
def download_folder(self, folder): def download_folder(self, folder):
""" """Download all documents in a folder.
Download all documents in a folder.
Parameters: Parameters:
folder(string): id of the folder to download folder(string): id of the folder to download
@ -22,8 +21,7 @@ class Crawler:
self.studip.download(doc) self.studip.download(doc)
def download_folder_rec(self, folder, base_dir): def download_folder_rec(self, folder, base_dir):
""" """Download all documents in a folder and its subfolders.
Download all documents in a folder and its subfolders.
This keeps the folder structure. This keeps the folder structure.
Parameters: Parameters:
@ -42,10 +40,10 @@ class Crawler:
self.create_dir(subdir_path) self.create_dir(subdir_path)
os.chdir(subdir_path) os.chdir(subdir_path)
self.download_folder_rec(subdir, subdir_path) self.download_folder_rec(subdir, subdir_path)
log.info('Finished crawling folder ' + folder)
def download_course(self, course, base_dir): def download_course(self, course, base_dir):
""" """Download all documents in course.
Download all documents in course.
This keeps the folder structure. This keeps the folder structure.
Parameters: Parameters:
@ -57,29 +55,29 @@ class Crawler:
os.chdir(base_dir) os.chdir(base_dir)
root = self.studip.get_top_folder(course) root = self.studip.get_top_folder(course)
self.download_folder_rec(root, base_dir) self.download_folder_rec(root, base_dir)
log.info('Finished crawling course ' + course)
def download_curr_courses(self, base_dir): def download_curr_courses(self, base_dir):
""" """Download all documents of all current courses.
Download all documents of all current courses.
This keeps the folder structure. This keeps the folder structure.
Parameters: Parameters:
base_dir(string): directory where to put the download base_dir(string): directory where to put the download
""" """
log.info('Start crawling all current courses')
self.create_dir(base_dir) self.create_dir(base_dir)
curr_courses = self.studip.get_curr_courses( curr_courses = self.studip.get_curr_courses(
self.studip.get_uid(), self.studip.get_curr_semester()) self.studip.get_uid(), self.studip.get_curr_semester())
log.info('crawling all current courses' + curr_courses)
os.chdir(base_dir) os.chdir(base_dir)
for course in curr_courses: for course in curr_courses:
log.debug('course is ' + curr_courses[course]) log.debug('course is ' + curr_courses[course])
course_name = curr_courses[course].replace('/', '-') course_name = curr_courses[course].replace('/', '-')
path = os.path.join(base_dir, course_name) path = os.path.join(base_dir, course_name)
self.download_course(course, path) self.download_course(course, path)
log.info('Finished crawling all current courses')
def create_dir(self, dir): def create_dir(self, dir):
""" """Creates a dir if it doesnt exist already.
Creates a dir if it doesnt exist already.
Parameters: Parameters:
dir(string): directory path to create dir(string): directory path to create

12
src/mysql.py
Unescape View File

@ -17,8 +17,7 @@ class Database:
self.setup_db() self.setup_db()
def connect(self): def connect(self):
""" """Connect to an existing database instance based on the object attributes.
Connect to an existing database instance based on the object attributes.
""" """
return pymysql.connect( return pymysql.connect(
host=self.HOST, host=self.HOST,
@ -30,8 +29,7 @@ class Database:
) )
def setup_db(self): def setup_db(self):
""" """Creates a database with tables.
Creates a database with tables.
""" """
db = self.connect() db = self.connect()
crs = db.cursor() crs = db.cursor()
@ -46,8 +44,7 @@ class Database:
log.debug(db) log.debug(db)
def set_last_file_dl(self, file_id, time): def set_last_file_dl(self, file_id, time):
""" """Insert a downloaded file to the database.
Insert a downloaded file to the database.
Parameters: Parameters:
file_id (string): id of the file downloaded file_id (string): id of the file downloaded
@ -64,8 +61,7 @@ class Database:
db.commit() db.commit()
def get_last_file_dl(self, file_id): def get_last_file_dl(self, file_id):
""" """Check when a file was downloaded.
Check when a file was downloaded.
Parameters: Parameters:
file_id(string): id of the file to check file_id(string): id of the file to check

27
src/studip.py
Unescape View File

@ -15,8 +15,7 @@ class Studip:
self.db = db self.db = db
def auth_req(self, url): def auth_req(self, url):
""" """Creates a request for a user.
Creates a request for a user.
Parameter: Parameter:
url(string): URL to send the request to url(string): URL to send the request to
@ -28,8 +27,7 @@ class Studip:
return req.get(url, auth=self.USER) return req.get(url, auth=self.USER)
def get_uid(self): def get_uid(self):
""" """Get the user id of the user specified in the object.
Get the user id of the user specified in the object.
Returns: Returns:
string: user id string: user id
@ -39,8 +37,7 @@ class Studip:
return user_id return user_id
def get_curr_semester(self): def get_curr_semester(self):
""" """Get the current semester of the studip instance specified in the object.
Get the current semester of the studip instance specified in the object.
Returns: Returns:
string: id for current semester string: id for current semester
@ -57,8 +54,7 @@ class Studip:
return 0 return 0
def get_ordered_semesters(self): def get_ordered_semesters(self):
""" """Get the a list of semesters of studip instance specified in the object.
Get the a list of semesters of studip instance specified in the object.
Returns: Returns:
list(string): all semesters of the user list(string): all semesters of the user
@ -71,8 +67,7 @@ class Studip:
return order_sems return order_sems
def get_curr_courses(self, user_id, semester): def get_curr_courses(self, user_id, semester):
""" """Get the a list of semesters of studip instance specified in the object.
Get the a list of semesters of studip instance specified in the object.
Returns: Returns:
string: id of the current semester string: id of the current semester
@ -101,8 +96,7 @@ class Studip:
return course_list return course_list
def get_top_folder(self, course): def get_top_folder(self, course):
""" """Retrieves the top folder id of a given course.
Retrieves the top folder id of a given course.
Parameters: Parameters:
course (string): course to get the top folder of course (string): course to get the top folder of
@ -116,8 +110,7 @@ class Studip:
return(tf_id) return(tf_id)
def get_docs(self, folder): def get_docs(self, folder):
""" """Get all the documents of a given folder.
Get all the documents of a given folder.
Parameters: Parameters:
folder(string): id of the folder to get documents of folder(string): id of the folder to get documents of
@ -134,8 +127,7 @@ class Studip:
return(res_docs) return(res_docs)
def download(self, doc): def download(self, doc):
""" """Download a document.
Download a document.
Parameters: Parameters:
doc (string): id of the document to download doc (string): id of the document to download
@ -156,8 +148,7 @@ class Studip:
self.db.set_last_file_dl(str(doc), str(int(time.time()))) self.db.set_last_file_dl(str(doc), str(int(time.time())))
def get_subdirs(self, folder): def get_subdirs(self, folder):
""" """Get all the subdirectories of a given folder.
Get all the subdirectories of a given folder.
Parameters: Parameters:
folder(string): id of the folder to get subdirectories of folder(string): id of the folder to get subdirectories of

Write Preview
Loading…
Cancel
Save