<commit_before>from random import SystemRandom
import string
import psycopg2
import psycopg2.extras
from cgi import parse_qs
import re
from http.cookies import SimpleCookie
from datetime import datetime, timedelta


random = SystemRandom()

# https://wiki.python.org/moin/EscapingHtml
html_escape_table = {
    "&": "&amp;",
    '"': "&quot;",
    "'": "&#x27;",
    ">": "&gt;",
    "<": "&lt;",
    "/": "&#x2F;"
}
allowed = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
skip = '&"\'<>/'


def html_escape(text):
    if not isinstance(text, str):
        return None
    return "".join(html_escape_table.get(c, c) for c in text)


def ensure_id(id_):
    if not isinstance(id_, str):
        return None
    return ''.join(filter(lambda x: x in allowed, id_))


def skip_bad(text):
    if not isinstance(text, str):
        return None
    return ''.join(filter(lambda x: x not in skip, text))


def is_email(value):
    """
    Ensure the given value is formatted as an email.
    """

    if value is None:
        return
    return re.match(r'\S+@\S+\.\S+', value)


def uniqid():
    """
    Generate a unique string with 24 characters.
    https://stackoverflow.com/a/2257449
    """
    return ''.join(
        random.choice(string.ascii_lowercase
                      + string.ascii_uppercase
                      + string.digits)
        for i in range(24)
    )


def clean_email(email):
    if email:
        email = skip_bad(email)
        if not is_email(email):
            email = None
    return email


def pull_cookies(environ):
    """
    Pulls and formats cookies stored by user for domain.
    <http://pwp.stevecassidy.net/wsgi/cookies.html>
    """

    cookie = SimpleCookie(environ.get('HTTP_COOKIE', ''))
    return {key: morsel.value for key, morsel in cookie.items()}


def get_session_id(environ):
    """
    """

    cookies = pull_cookies(environ)
    return ensure_id(
        cookies.get('suggest_session_id', uniqid())
    )


def get_set_cookie_header(session_id):
    expires = ((datetime.utcnow() + timedelta(weeks=2))
               .strftime('%a, %d-%b-%Y %H:%M:%S GMT'))
    return ('Set-Cookie', '; '.join([
        'suggest_session_id={session_id}',
        'expires={expires}',
        'Path=/',
        'HttpOnly',
    ]).format(session_id=session_id, expires=expires))


def list_suggests(conn):
    data = None
    try:
        cur = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
        with cur:
            cur.execute("""
                SELECT suggests.*, (
                    SELECT COUNT(*)
                    FROM suggests_followers
                    WHERE suggests_followers.suggest_id = suggests.id
                ) AS COUNT
                FROM suggests
                ORDER BY count DESC, created DESC;
            """)
            data = cur.fetchall()
        data = [row for row in data]<commit_msg>Top 3 are random each time<commit_after>from random import SystemRandom
import string
import psycopg2
import psycopg2.extras
from cgi import parse_qs
import re
from http.cookies import SimpleCookie
from datetime import datetime, timedelta
from random import sample


random = SystemRandom()

# https://wiki.python.org/moin/EscapingHtml
html_escape_table = {
    "&": "&amp;",
    '"': "&quot;",
    "'": "&#x27;",
    ">": "&gt;",
    "<": "&lt;",
    "/": "&#x2F;"
}
allowed = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
skip = '&"\'<>/'


def html_escape(text):
    if not isinstance(text, str):
        return None
    return "".join(html_escape_table.get(c, c) for c in text)


def ensure_id(id_):
    if not isinstance(id_, str):
        return None
    return ''.join(filter(lambda x: x in allowed, id_))


def skip_bad(text):
    if not isinstance(text, str):
        return None
    return ''.join(filter(lambda x: x not in skip, text))


def is_email(value):
    """
    Ensure the given value is formatted as an email.
    """

    if value is None:
        return
    return re.match(r'\S+@\S+\.\S+', value)


def uniqid():
    """
    Generate a unique string with 24 characters.
    https://stackoverflow.com/a/2257449
    """
    return ''.join(
        random.choice(string.ascii_lowercase
                      + string.ascii_uppercase
                      + string.digits)
        for i in range(24)
    )


def clean_email(email):
    if email:
        email = skip_bad(email)
        if not is_email(email):
            email = None
    return email


def pull_cookies(environ):
    """
    Pulls and formats cookies stored by user for domain.
    <http://pwp.stevecassidy.net/wsgi/cookies.html>
    """

    cookie = SimpleCookie(environ.get('HTTP_COOKIE', ''))
    return {key: morsel.value for key, morsel in cookie.items()}


def get_session_id(environ):
    """
    """

    cookies = pull_cookies(environ)
    return ensure_id(
        cookies.get('suggest_session_id', uniqid())
    )


def get_set_cookie_header(session_id):
    expires = ((datetime.utcnow() + timedelta(weeks=2))
               .strftime('%a, %d-%b-%Y %H:%M:%S GMT'))
    return ('Set-Cookie', '; '.join([
        'suggest_session_id={session_id}',
        'expires={expires}',
        'Path=/',
        'HttpOnly',
    ]).format(session_id=session_id, expires=expires))


def list_suggests(conn):
    data = None
    try:
        cur = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
        with cur:
            cur.execute("""
                SELECT suggests.*, (
                    SELECT COUNT(*)
                    FROM suggests_followers
                    WHERE suggests_followers.suggest_id = suggests.id
                ) AS COUNT
                FROM suggests
                ORDER BY count DESC, created DESC;
            """)
            data = cur.fetchall()
        data = [row for row in data]
        if len(data) > 3:
            top = sample(data, 3)
            top_ids = [row['id'] for row in top]
            data = [row for row in data if row['id'] not in top_ids]
            data = top + data
{'fraction_non_alphanumeric': 0.12923483721698892, 'fraction_numerical': 0.009750454470335481, 'mean_word_length': 2.2891304347826087, 'pattern_counts': {'":': 10, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1951712', 'n_tokens_mistral': 2039, 'n_tokens_neox': 1927, 'n_words': 474}