<commit_before>from datetime import datetime

import html5lib
from html5lib import sanitizer, serializer, treewalkers
import markdown
from sqlalchemy.sql.expression import asc

import prefs
from models import Post

html_parser = html5lib.HTMLParser(tokenizer=sanitizer.HTMLSanitizer)
html_serializer = serializer.HTMLSerializer()

def post_list(session, thread_id, number=prefs.POSTS_PER_PAGE, page=None, start_at=None):
    return session.query(Post).filter_by(thread_id=thread_id).order_by(asc(Post.time)).limit(number).all()


def make_post(session, user, thread, raw_text):
    html_text = markdown.markdown(raw_text, safe_mode='escape', output_format='html5')
    new_post = Post(time=datetime.utcnow(), content_raw=raw_text, content_html=html_text, author_id=user.id, thread_id=thread.id)
    session.add(new_post)
    thread.last_post_time = new_post.time
    thread.posts += 1
    new_post.number = thread.posts
    return new_post<commit_msg>Removed deprecated html5lib code<commit_after>from datetime import datetime

import markdown
from sqlalchemy.sql.expression import asc

import prefs
from models import Post

def post_list(session, thread_id, number=prefs.POSTS_PER_PAGE, page=None, start_at=None):
    return session.query(Post).filter_by(thread_id=thread_id).order_by(asc(Post.time)).limit(number).all()


def make_post(session, user, thread, raw_text):
    html_text = markdown.markdown(raw_text, safe_mode='escape', output_format='html5')
    new_post = Post(time=datetime.utcnow(), content_raw=raw_text, content_html=html_text, author_id=user.id, thread_id=thread.id)
    session.add(new_post)
    thread.last_post_time = new_post.time
    thread.posts += 1
    new_post.number = thread.posts
    return new_post
{'fraction_non_alphanumeric': 0.10155799192152337, 'fraction_numerical': 0.004616272360069244, 'mean_word_length': 6.285714285714286, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6112775', 'n_tokens_mistral': 578, 'n_tokens_neox': 567, 'n_words': 110}