tiyn/container-critique
1
0
Fork 0
mirror of https://github.com/tiyn/container-critique.git synced 2025-10-31 11:11:21 +01:00
Code Issues Projects Releases Wiki Activity

search: added full-text search and docstrings

Browse Source
This commit is contained in:
tiyn
2022-08-05 01:01:21 +02:00
parent a6e1735cac
commit 284a597d4a
15 changed files with 515 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -1,2 +1,3 @@
data
data.db
indexdir

137
src/app.py
View File

@@ -1,14 +1,17 @@
from flask import Flask, flash, render_template, redirect, abort, url_for
from flask import Flask, flash, render_template, redirect, abort, url_for, \
request
from flask_ckeditor import CKEditor
from flask_login import current_user, login_user, LoginManager, logout_user, \
login_required
from flask_wtf import CSRFProtect
import os
from werkzeug.exceptions import HTTPException
import config
from content import rating_to_star
from database import Database
from forms import LoginForm, RegisterForm, WriteForm
from forms import LoginForm, RegisterForm, WriteForm, SearchForm
from search import create_search_index, ft_search
app = Flask(__name__)
@@ -23,20 +26,55 @@ login = LoginManager(app)
login.login_view = "login"
@login.user_loader
def load_user(ident):
"""
Returns a user by id.
Parameters:
ident(int): id of the user
Returns:
User: user that matches the id, None if none matches the id
"""
user = db.get_user_by_id(ident)
if user is not None:
return user
return None
@app.context_processor
def inject_title():
return dict(title=config.TITLE, style=config.STYLE, \
description=config.DESCRIPTION, \
"""
Injects variables to the jinja2 templates.
Returns:
dict: dictionary of variables to inject.
"""
return dict(title=config.TITLE, style=config.STYLE,
description=config.DESCRIPTION,
registration=config.ALLOW_REGISTRATION, r_to_star=rating_to_star)
@app.errorhandler(404)
@app.errorhandler(HTTPException)
def page_not_found(e):
return render_template("error.html", errorcode="404"), 404
"""
Renders the error pages.
Returns:
str: html formatted Error page
"""
return render_template("error.html", errorcode=str(e.code) + " " + e.name), e.code
@app.route("/")
def index():
"""
Renders the index page.
Returns:
str: html formatted index page
"""
entries = db.get_entries()
entries.reverse()
return render_template("index.html", entries=entries)
@@ -44,6 +82,12 @@ def index():
@app.route("/archive")
def archive():
"""
Renders the archive page.
Returns:
str: html formatted archive page
"""
entries = db.get_entries()
entries.sort(key=lambda y: y.item.name)
entries.reverse()
@@ -54,7 +98,16 @@ def archive():
@app.route("/user/<name>")
def user(name):
entries = db.get_entries_by_user(name)
"""
Renders the user page of a specific user.
Parameters:
name(str): name of the user
Returns:
str: html formatted user page
"""
entries = db.get_entries_by_username(name)
entries.sort(key=lambda y: y.item.name)
entries.reverse()
entries.sort(key=lambda y: y.item.date)
@@ -66,6 +119,15 @@ def user(name):
@app.route("/entry/<ident>")
def entry(ident):
"""
Renders the entry page of a specific entry.
Parameters:
ident(str): ident of the entry
Returns:
str: html formatted entry page
"""
entry = db.get_entry_by_id(ident)
if entry is not None:
return render_template("standalone.html", entry=entry)
@@ -74,22 +136,43 @@ def entry(ident):
@app.route("/feed")
def feed():
"""
Renders the rss feed of a the feed.
Returns:
str: xml formatted feed
"""
entries = db.get_entries()
entries.reverse()
rss_xml = render_template("rss.xml", entries=entries)
return rss_xml
@login.user_loader
def load_user(ident):
user = db.get_user_by_id(ident)
if user is not None:
return user
return None
@app.route("/search", methods=["GET", "POST"])
def search():
"""
Renders the search page.
Returns:
str: html formatted search page.
"""
form = SearchForm()
if request.method == "POST":
query_str = request.form["query_str"]
query_res = ft_search(query_str)
return render_template("search.html", form=form, results=query_res), 200
return render_template("search.html", form=form, content=""), 200
@app.route("/login", methods=["GET", "POST"])
def login():
"""
Logs the user in.
Returns:
str: html formatted login page, if login is successful renders the index
page.
"""
if current_user.is_authenticated:
return redirect(url_for("index"))
form = LoginForm()
@@ -107,12 +190,25 @@ def login():
@app.route('/logout')
def logout():
"""
Logs out the current user.
Returns:
str: html formatted index page.
"""
logout_user()
return redirect(url_for("index"))
@app.route("/register", methods=["GET", "POST"])
def register():
"""
Registers new users.
Returns:
str: html formatted registration page, if registration is successful
renders the index page.
"""
if current_user.is_authenticated or not config.ALLOW_REGISTRATION:
return redirect(url_for("index"))
form = RegisterForm()
@@ -132,12 +228,20 @@ def register():
@app.route("/write_entry", methods=["GET", "POST"])
@login_required
def write_entry():
"""
Stores newly written entries.
Returns:
str: html formatted write entry page, if posting of the entry is successful
renders the index page.
"""
if not current_user.is_authenticated:
return redirect(url_for("index"))
form = WriteForm()
if form.validate_on_submit():
db.insert_entry(form.name.data, form.date.data,
form.text.data, form.rating.data, current_user.id)
create_search_index()
return redirect(url_for("index"))
return render_template("write.html", form=form)
@@ -145,10 +249,17 @@ def write_entry():
@app.route("/delete_entry/<ident>", methods=["GET", "POST"])
@login_required
def delete_entry(ident):
"""
Deletes an existing entry.
Returns:
str: html formatted index entry page.
"""
if not current_user.is_authenticated:
return redirect(url_for("index"))
if current_user.id == db.get_entry_by_id(ident).user.id:
db.delete_entry(ident)
create_search_index()
return redirect(url_for("index"))

6
src/config.py
View File

@@ -12,3 +12,9 @@ STYLE = "dark"
# Allow new registrations
ALLOW_REGISTRATION = True
# Location of the search-indexing directory
INDEX_DIR = "indexdir"
# Number of search results to show
SEARCH_NUMBER = 10

4
src/content.py
View File

@@ -8,10 +8,10 @@ def rating_to_star(rating):
Creates a string with stars based on the rating.
Parameters:
rating: rating with minimum of 0 and maximum of 100.
rating: rating with minimum of 0 and maximum of 100
Returns:
string: unicode-formatted star-rating string.
string: unicode-formatted star-rating string
"""
res = u"\u272D"*int(rating/20)
length = len(res)

259
src/database.py
View File

@@ -5,6 +5,17 @@ from werkzeug.security import generate_password_hash, check_password_hash
class User():
"""
A class to represent a user.
Attributes:
name (str): name of the user
id (int): id of the user
is_active (bool): check if the user is active
is_authenticated (bool): check if the user is logged in
is_anonymous (bool): check if the user is is_anonymous
pass_hash (str): hash of the users password
"""
def __init__(self, name, pass_hash=None):
self.name = name
@@ -15,19 +26,61 @@ class User():
self.pass_hash = pass_hash
def set_password(self, password):
"""
Set the password hash of the user from a password.
Parameters:
password (str): password to add to the user
Returns:
None
"""
self.pass_hash = generate_password_hash(password)
def set_id(self, ident):
"""
Set the id of the user.
Parameters:
id (str): id to add to the user
Returns:
None
"""
self.id = ident
def check_password(self, password):
"""
Check if a given password matches the one of the users by comparing the
hashes.
Parameters:
password (str): password to compare the users password to
Returns:
bool: True if it matches the users password, False otherwise
"""
return check_password_hash(self.pass_hash, password)
def get_id(self):
"""
Get the id of the user.
Returns:
int: id of the user
"""
return self.id
class Item():
"""
A class to represent an item.
Attributes:
name (str): name of the item
id (int): id of the item
date (str): date the item was created
"""
def __init__(self, name, date):
self.name = name
@@ -35,29 +88,73 @@ class Item():
self.id = None
def set_id(self, ident):
"""
Set the id of the item.
Returns:
int: id of the item
"""
self.id = ident
class Entry():
"""
A class to represent an entry.
def __init__(self, text, rating, reviewed):
Attributes:
text (str): text of the entry
rating (int): rating of the item
date (str): date the entry was created
item (Item): item that is referenced by the entry
user (User): user that authored the entry
id (int): id of the item
"""
def __init__(self, text, rating, date):
self.text = text
self.rating = rating
self.reviewed = reviewed
self.date = date
self.item = None
self.user = None
def set_id(self, ident):
"""
Set the id of the entry.
Parameters:
ident(int): id of the entry
"""
self.id = ident
def set_item(self, item):
"""
Set the item of the entry.
Parameters:
item(Item): item of the entry
"""
self.item = item
def set_user(self, user):
"""
Set the user of the entry.
Parameters:
user(User): user of the entry
"""
self.user = user
class Database:
"""
A class to represent an entry.
Attributes:
USER_TABLE_FILE (str): name of the user table
ENTRY_TABLE_FILE (str): name of the entry table
ITEM_TABLE_FILE (str): name of the item table
DB_DIR(PathLike): path that leads to the directory containing the database
"""
def __init__(self):
self.USER_TABLE_FILE = 'USERS'
@@ -70,12 +167,17 @@ class Database:
"""
Connect to an existing database instance based on the object
attributes.
Return:
Connection: connection to the database
"""
path = os.path.join(self.DB_DIR, "data.db")
return sqlite3.connect(path)
def setup_db(self):
"""Creates a database with tables."""
"""
Creates a database with the needed tables if it doesn't already exits.
"""
db = self.connect()
crs = db.cursor()
query = "CREATE TABLE IF NOT EXISTS " + self.USER_TABLE_FILE + \
@@ -95,11 +197,21 @@ class Database:
"text TEXT NOT NULL," + \
"rating INTEGER NOT NULL," +\
"user_id INTEGER REFERENCES " + self.USER_TABLE_FILE + "(id),"\
"reviewed CHAR(10) NOT NULL)"
"date CHAR(10) NOT NULL)"
crs.execute(query)
db.commit()
def insert_user(self, username, password):
"""
Insert a row in the user table.
Parameters:
username (str): name of the user to add
password (str): password of the user to add
Returns:
int: number of the line the row was added, None if it wasn't successful
"""
pass_hash = generate_password_hash(password)
if self.get_user_by_name(username) is None and pass_hash is not None:
db = self.connect()
@@ -113,6 +225,19 @@ class Database:
return None
def insert_entry(self, name, date, text, rating, user_id=None):
"""
Insert a row in the entry table.
Parameters:
name (str): name of the entry to add
date (str): date of the entry to add
text (str): text of the entry to add
rating (str): rating of the entry to add
user_id (int): id of the user referenced by the entry to add
Returns:
int: number of the line the row was added
"""
db = self.connect()
crs = db.cursor()
query = "INSERT OR IGNORE INTO " + self.ITEM_TABLE_FILE + \
@@ -122,15 +247,24 @@ class Database:
" WHERE name = ? AND date = ?"
crs.execute(query, (name, date))
item_id = crs.fetchone()[0]
reviewed = dt.today().strftime('%Y-%m-%d')
date = dt.today().strftime('%Y-%m-%d')
query = "INSERT INTO " + self.ENTRY_TABLE_FILE + \
"(`item_id`, `text`, `rating`, `user_id`, `reviewed`)" + \
"(`item_id`, `text`, `rating`, `user_id`, `date`)" + \
"VALUES (?, ?, ?, ?, ?)"
crs.execute(query, (item_id, text, rating, user_id, reviewed))
crs.execute(query, (item_id, text, rating, user_id, date))
db.commit()
return crs.lastrowid
def delete_entry(self, ident):
"""
Delete a row from the entry table based on the entrys id.
Parameters:
ident (int): id of the entry to remove
Returns:
int: number of the line the row was removed from
"""
db = self.connect()
crs = db.cursor()
query = "DELETE FROM " + self.ENTRY_TABLE_FILE + " WHERE id = ?"
@@ -139,16 +273,31 @@ class Database:
return crs.lastrowid
def get_entries(self):
"""
Return all the entries stored in the database.
Return:
List(Entry): list of entries in database
"""
db = self.connect()
crs = db.cursor()
query = "SELECT * FROM " + self.ENTRY_TABLE_FILE
crs.execute(query)
res = []
for item in crs.fetchall():
res.append(self.db_to_entry(*item))
res.append(self.entry_from_db(*item))
return res
def get_entry_by_id(self, ident):
"""
Return an entry stored in the database based on the entrys id.
Parameters:
ident (int): id of the entry to return
Returns:
Entry: entry that matched the given id
"""
db = self.connect()
crs = db.cursor()
query = "SELECT * FROM " + self.ENTRY_TABLE_FILE + " WHERE id = ?"
@@ -157,21 +306,39 @@ class Database:
if fetched is None:
return None
else:
return self.db_to_entry(*fetched)
return self.entry_from_db(*fetched)
def get_entries_by_user(self, name):
def get_entries_by_username(self, username):
"""
Return a entries stored in the database based on the entries name.
Parameters:
username (str): name of the entries to return
Returns:
List(Entry): entries that matched the given name
"""
db = self.connect()
crs = db.cursor()
query = "SELECT * FROM " + self.ENTRY_TABLE_FILE + \
" WHERE user_id = (SELECT id FROM " + self.USER_TABLE_FILE + \
" WHERE name = ?)"
crs.execute(query, (name, ))
crs.execute(query, (username, ))
res = []
for item in crs.fetchall():
res.append(self.db_to_entry(*item))
res.append(self.entry_from_db(*item))
return res
def get_item_by_id(self, ident):
"""
Return an item stored in the database based on the items id.
Parameters:
ident (int): id of the item to return
Returns:
Item: item that matched the given id
"""
db = self.connect()
crs = db.cursor()
query = "SELECT * FROM " + self.ITEM_TABLE_FILE + " WHERE id = ?"
@@ -180,9 +347,18 @@ class Database:
if fetched is None:
return None
else:
return self.db_to_item(*fetched)
return self.item_from_db(*fetched)
def get_user_by_id(self, ident):
"""
Return a user stored in the database based on the users id.
Parameters:
ident (int): id of the user to return
Returns:
Item: user that matched the given id
"""
db = self.connect()
crs = db.cursor()
query = "SELECT * FROM " + self.USER_TABLE_FILE + " WHERE id = ?"
@@ -191,9 +367,18 @@ class Database:
if fetched is None:
return None
else:
return self.db_to_user(*fetched)
return self.user_from_db(*fetched)
def get_user_by_name(self, name):
"""
Return a user stored in the database based on the user name.
Parameters:
name (str): name of the user to return
Returns:
Entry: user that matched the given name
"""
db = self.connect()
crs = db.cursor()
query = "SELECT * FROM " + self.USER_TABLE_FILE + " WHERE name = ?"
@@ -202,20 +387,56 @@ class Database:
if fetched is None:
return None
else:
return self.db_to_user(*fetched)
return self.user_from_db(*fetched)
def db_to_user(self, ident, name, pass_hash):
def user_from_db(self, ident, name, pass_hash):
"""
Return a user from given database parameters.
Parameters:
ident: id of the user
name: text of the user
pass_hash: password hash of the user
Returns:
User: user element with given variables
"""
user = User(name, pass_hash)
user.set_id(ident)
return user
def db_to_item(self, ident, name, date):
def item_from_db(self, ident, name, date):
"""
Return an item from given database parameters.
Parameters:
ident: id of the item
name: text of the item
date: date of the day the item was created
Returns:
Item: entry element with given variables
"""
item = Item(name, date)
item.set_id(ident)
return item
def db_to_entry(self, ident, item_id, text, rating, user_id, reviewed):
entry = Entry(text, rating, reviewed)
def entry_from_db(self, ident, item_id, text, rating, user_id, date):
"""
Return an entry from given database parameters.
Parameters:
ident: id of the entry
item_id: id of the referenced item
text: text of the entry
rating: rating of the entry
user_id: id of the user that authored the entry
date: date of the day the entry was written
Returns:
Entry: entry element with given variables
"""
entry = Entry(text, rating, date)
entry.set_id(ident)
entry.set_item(self.get_item_by_id(item_id))
entry.set_user(self.get_user_by_id(user_id))

32
src/forms.py
View File

@@ -1,13 +1,16 @@
from datetime import date
from flask_ckeditor import CKEditorField
from flask_wtf import FlaskForm
from wtforms import StringField, PasswordField, SubmitField
from wtforms import StringField, PasswordField, SubmitField, TextField
from wtforms.fields.html5 import IntegerField
from wtforms.validators import DataRequired, EqualTo, InputRequired, \
NumberRange, ValidationError, Length
class LoginForm(FlaskForm):
"""
A Class for the Form that is used while logging in.
"""
username = StringField("Username", validators=[DataRequired(),
Length(min=4, max=32)])
password = PasswordField("Password", validators=[DataRequired(),
@@ -16,6 +19,9 @@ class LoginForm(FlaskForm):
class RegisterForm(FlaskForm):
"""
A Class for the Form that is used while registering.
"""
username = StringField("Username", validators=[DataRequired(),
Length(min=4, max=32)])
password = PasswordField("Password", validators=[DataRequired(),
@@ -25,7 +31,19 @@ class RegisterForm(FlaskForm):
submit = SubmitField("Register")
class SearchForm(FlaskForm):
"""
A Class for the Form that is used while searching.
"""
query_str = TextField(
"Query", [DataRequired("Please enter the search term")])
submit = SubmitField("Search")
class WriteForm(FlaskForm):
"""
A Class for the Form that is used while writing a new entry.
"""
name = StringField("Name", validators=[DataRequired(),
Length(min=2, max=64)])
date = IntegerField("Release Year", default=date.today().year, validators=[
@@ -37,5 +55,17 @@ class WriteForm(FlaskForm):
submit = SubmitField("Publish")
def validate_text(self, text):
"""
Validate a given input for html level one headers.
Parameters:
text (str): text to validate
Returns:
None
Raises:
ValidatenError: if the text contains a first level html tag
"""
if "<h1>" in text.data or "</h1>" in text.data:
raise ValidationError("Headings on level 1 are not permitted.")

0
src/indexdir/.gitkeep Normal file
View File

80
src/search.py Normal file
View File

@@ -0,0 +1,80 @@
import os
import re
from whoosh import scoring
from whoosh.index import create_in, open_dir
from whoosh.fields import Schema, TEXT, ID
from whoosh.qparser import QueryParser
import config
from database import Database
CLEANR = re.compile('<.*?>|&([a-z0-9]+|#[0-9]{1,6}|#x[0-9a-f]{1,6});')
def remove_html_tags(text):
"""
Convert a text from html formatted to unformatted.
Parameters:
text (str): text to clean
Returns:
str: text without html tags
"""
res = re.sub(CLEANR, '', text)
return res
def create_search_index():
"""
Create the index data to search all entries.
"""
db = Database()
schema = Schema(title=TEXT(stored=True),
path=ID(stored=True), content=TEXT(stored=True))
if not os.path.exists(config.INDEX_DIR):
os.mkdir(config.INDEX_DIR)
ix = create_in(config.INDEX_DIR, schema)
writer = ix.writer()
for entry in db.get_entries():
path = str(entry.id)
text = entry.item.name + " " + entry.item.date + " " + entry.text + \
" by " + entry.user.name + " " + entry.date
writer.add_document(title=entry.item.name, path=path, content=text)
writer.commit()
def ft_search_times(query_str, number):
"""
Search for a given term and returns a specific amount of results.
Parameters:
query_str (str): term to search for
number (int): number of results to return
Returns:
List(Entry): list of entries that matched the search
"""
ix = open_dir(config.INDEX_DIR)
results = []
db = Database()
with ix.searcher(weighting=scoring.BM25F) as s:
query = QueryParser("content", ix.schema).parse(query_str)
matches = s.search(query, limit=number)
for match in matches:
results.append(db.get_entry_by_id(match["path"]))
return results
def ft_search(query_str):
"""
Search for a given term and show the predefined amount of results.
Parameters:
query_str (str): term to search for
Returns:
List(Entry): list of entries that matched the search
"""
return ft_search_times(query_str, config.SEARCH_NUMBER)

2
src/templates/archive.html
View File

@@ -27,7 +27,7 @@
{% set ns.open_li = True -%}
{% endif -%}
<a href="{{ url_for('entry', ident=entry.id) }}">
{{ entry.reviewed }} {{ r_to_star(entry.rating) }} by {{ entry.user.name }}
{{ entry.date }} {{ r_to_star(entry.rating) }} by {{ entry.user.name }}
</a>
{% set ns.prev_item_date = entry.item.date -%}
{% set ns.prev_item_id = entry.item.id -%}

2
src/templates/index.html
View File

@@ -15,7 +15,7 @@
rated {{ entry.rating }}/100 by
<a href="{{ url_for('user', name=entry.user.name) }}">
{{ entry.user.name }}
</a> on {{ entry.reviewed }}
</a> on {{ entry.date }}
</small><br>
{% autoescape off -%}
{{ entry.text }}

2
src/templates/rss.xml
View File

@@ -16,7 +16,7 @@
{{ url_for("index", _anchor=entry.id, _external=True) }}
</guid>
<pubDate>
{{ entry.reviewed }}
{{ entry.date }}
</pubDate>
<description>
{% autoescape off -%}

23
src/templates/search.html Normal file
View File

@@ -0,0 +1,23 @@
{% extends "template.html" -%}
{% block content %}
<div class="container">
<h1>Search</h1><br>
<div class="search">
<form action="{{ url_for('search') }}" method=post>
{{ form.hidden_tag() }}
{{ form.query_str }}
{{ form.submit }}
</form>
<ul>
{% for entry in results -%}
<li>
<a href="{{ url_for('entry', ident=entry.id) }}">
{{ entry.date }} {{ r_to_star(entry.rating) }} {{ entry.item.name }} ({{ entry.item.date }}) by {{ entry.user.name }}
</a>
</li>
{% endfor -%}
</ul>
</div>
</div>
{% endblock -%}

2
src/templates/standalone.html
View File

@@ -14,7 +14,7 @@
</a>
on
<a href="{{ url_for('index', _anchor=entry.id) }}">
{{ entry.reviewed }}
{{ entry.date }}
</a>
</small><br>
{% if current_user.id == entry.user.id -%}

5
src/templates/template.html
View File

@@ -1,6 +1,7 @@
{% set navigation_bar = [
(url_for("index"), 'index', 'Blog'),
(url_for("archive"), 'archive', 'Archive')
(url_for("index"), "index", "Blog"),
(url_for("archive"), "archive", "Archive"),
(url_for("search"), "search", "Search")
] -%}
<!DOCTYPE html>

2
src/templates/user.html
View File

@@ -27,7 +27,7 @@
{% set ns.open_li = True -%}
{% endif -%}
<a href="{{ url_for('entry', ident=entry.id) }}">
{{ entry.reviewed }} {{ r_to_star(entry.rating) }}
{{ entry.date }} {{ r_to_star(entry.rating) }}
</a>
{% set ns.prev_item_date = entry.item.date -%}
{% set ns.prev_item_id = entry.item.id -%}