<commit_before># -*- coding: utf-8 -*-
# Copyright (C) 2011-2013 by <NAME>, Kozea
# This file is part of pystil, licensed under a 3-clause BSD license.


"""Utility functions to help processing data"""
from datetime import timedelta
from urllib.parse import urlparse, parse_qs
from pystil.context import pystil
from pystil.db import Visit
from sqlalchemy.sql import literal, func
import re

IPV4RE = re.compile(r"(\d{1,3}(\.|$)){4}")
BROWSER_VERSION_NUMBERS = {
    'opera': 1,
    'safari': 1,
    'chrome': 1}

# Stripped down user agent parsing from werkzeug
PLATFORMS = [(b, re.compile(a, re.I)) for a, b in (
    ('iphone|ios', 'iphone'),
    (r'darwin|mac|os\s*x', 'macos'),
    ('win', 'windows'),
    (r'android', 'android'),
    (r'x11|lin(\b|ux)?', 'linux'),
    ('(sun|i86)os', 'solaris'),
    (r'nintendo\s+wii', 'wii'),
    ('irix', 'irix'),
    ('hp-?ux', 'hpux'),
    ('aix', 'aix'),
    ('sco|unix_sv', 'sco'),
    ('bsd', 'bsd'),
    ('amiga', 'amiga')
)]

BROWSERS = [(
    b, re.compile(r'(?:%s)[/\sa-z(]*(\d+[.\da-z]+)?(?i)' % a)
) for a, b in (
    ('googlebot', 'google'),
    ('msnbot', 'msn'),
    ('yahoo', 'yahoo'),
    ('ask jeeves', 'ask'),
    (r'aol|america\s+online\s+browser', 'aol'),
    ('opera', 'opera'),
    ('chrome', 'chrome'),
    ('firefox|firebird|phoenix|iceweasel', 'firefox'),
    ('galeon', 'galeon'),
    ('safari', 'safari'),
    ('webkit', 'webkit'),
    ('camino', 'camino'),
    ('konqueror', 'konqueror'),
    ('k-meleon', 'kmeleon'),
    ('netscape', 'netscape'),
    (r'msie|microsoft\s+internet\s+explorer', 'msie'),
    ('lynx', 'lynx'),
    ('links', 'links'),
    ('seamonkey|mozilla', 'seamonkey')
)]


def on(host, table=Visit.__table__):
    """Generate a filter on a host """
    if host == 'all':
        return literal(True)
    return table.c.host.like('%' + host)


def between(from_date, to_date, table=Visit.__table__):
    """Generate a filter between 2 dates"""
    return ((from_date <= table.c.date) &
            (table.c.date < to_date + timedelta(1)))


def visit_to_table_line(visit):
    html = '<tr data-visit-uuid="%s">' % visit.uuid
    for key in [
            'date', 'site', 'ip', 'country',
            'city', 'page', 'pretty_referrer']:
        html += '<td>'
        val = getattr(visit, key)
        if val:
            if key == 'date':
                val = val.strftime('%Y-%m-%d %H:%M:%S')
            html += val<commit_msg>Better pretty referrers<commit_after># -*- coding: utf-8 -*-
# Copyright (C) 2011-2013 by <NAME>, Kozea
# This file is part of pystil, licensed under a 3-clause BSD license.


"""Utility functions to help processing data"""
from datetime import timedelta
from urllib.parse import urlparse, parse_qs
from pystil.context import pystil
from pystil.db import Visit
from sqlalchemy.sql import literal, func
import re

IPV4RE = re.compile(r"(\d{1,3}(\.|$)){4}")
BROWSER_VERSION_NUMBERS = {
    'opera': 1,
    'safari': 1,
    'chrome': 1}

# Stripped down user agent parsing from werkzeug
PLATFORMS = [(b, re.compile(a, re.I)) for a, b in (
    ('iphone|ios', 'iphone'),
    (r'darwin|mac|os\s*x', 'macos'),
    ('win', 'windows'),
    (r'android', 'android'),
    (r'x11|lin(\b|ux)?', 'linux'),
    ('(sun|i86)os', 'solaris'),
    (r'nintendo\s+wii', 'wii'),
    ('irix', 'irix'),
    ('hp-?ux', 'hpux'),
    ('aix', 'aix'),
    ('sco|unix_sv', 'sco'),
    ('bsd', 'bsd'),
    ('amiga', 'amiga')
)]

BROWSERS = [(
    b, re.compile(r'(?:%s)[/\sa-z(]*(\d+[.\da-z]+)?(?i)' % a)
) for a, b in (
    ('googlebot', 'google'),
    ('msnbot', 'msn'),
    ('yahoo', 'yahoo'),
    ('ask jeeves', 'ask'),
    (r'aol|america\s+online\s+browser', 'aol'),
    ('opera', 'opera'),
    ('chrome', 'chrome'),
    ('firefox|firebird|phoenix|iceweasel', 'firefox'),
    ('galeon', 'galeon'),
    ('safari', 'safari'),
    ('webkit', 'webkit'),
    ('camino', 'camino'),
    ('konqueror', 'konqueror'),
    ('k-meleon', 'kmeleon'),
    ('netscape', 'netscape'),
    (r'msie|microsoft\s+internet\s+explorer', 'msie'),
    ('lynx', 'lynx'),
    ('links', 'links'),
    ('seamonkey|mozilla', 'seamonkey')
)]


def on(host, table=Visit.__table__):
    """Generate a filter on a host """
    if host == 'all':
        return literal(True)
    return table.c.host.like('%' + host)


def between(from_date, to_date, table=Visit.__table__):
    """Generate a filter between 2 dates"""
    return ((from_date <= table.c.date) &
            (table.c.date < to_date + timedelta(1)))


def visit_to_table_line(visit):
    html = '<tr data-visit-uuid="%s">' % visit.uuid
    for key in [
            'date', 'site', 'ip', 'country',
            'city', 'page', 'referrer']:
        html += '<td>'
        val = getattr(visit, key)
        if val:
            if key == 'date':
                val = val.strftime('%Y-%m-%d %H:%M:%S')
            if key == 'referrer':
                val = parse_referrer(val, True, True)
            html += val
{'fraction_non_alphanumeric': 0.22294416243654822, 'fraction_numerical': 0.009340101522842639, 'mean_word_length': 3.077814569536424, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2797257', 'n_tokens_mistral': 1944, 'n_tokens_neox': 1863, 'n_words': 432}