1
0
mirror of https://github.com/tiyn/stud.ip-crawler.git synced 2026-02-22 06:34:48 +01:00

database: switching from mysql to sqlite

Currently we only need the db to keep track of the files that were
already downloaded, thus we only use one database and only one table.
A complete sql database is a bit too bulky for this purpose.
By using sqlite we can minimize and embed the db.
This commit is contained in:
TiynGER
2020-11-17 11:04:03 +01:00
parent 415a21da3b
commit 4a8cf45ad3
7 changed files with 29 additions and 71 deletions

View File

@@ -146,13 +146,10 @@ class Studip:
last_dl = self.db.get_last_file_dl(doc)
if last_dl == None or last_dl < doc_chdate:
rsp2 = self.auth_req('/api.php/file/' + doc + '/download')
#total_size = int(rsp2.headers.get('content-length', 0))
log.info('downloading ' + doc_name)
#progbar = tqdm(total=total_size, unit='iB', unit_scale=True)
try:
with open(doc_name, 'wb') as doc_file:
for chunk in rsp2.iter_content(self.CHUNK_SIZE):
#progbar.update(len(chunk))
doc_file.write(chunk)
self.db.set_last_file_dl(str(doc), str(int(time.time())))
except OSError:
@@ -172,7 +169,7 @@ class Studip:
try:
subdirs = rsp.json()['subfolders']
except ValueError:
return res_docs
return res_subdirs
for subdir in subdirs:
try:
sub_id = subdir['id']