<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2011 by <NAME>, Kozea
# This file is part of pystil, licensed under a 3-clause BSD license.

"""Utility functions to help processing data"""

from datetime import datetime, timedelta
from urlparse import urlparse, parse_qs
from pystil.db import Visit, fields
from datetime import date, time
from calendar import timegm
from decimal import Decimal
import re
import json

IPV4RE = re.compile(r"(\d{1,3}(\.|$)){4}")
BROWSER_VERSION_NUMBERS = {
    'opera': 1,
    'safari': 1,
    'chrome': 1}


class PystilEncoder(json.JSONEncoder):

    def default(self, obj):
        if isinstance(obj, datetime):
            return obj.strftime("%Y-%m-%d %H:%M:%S")
        if isinstance(obj, timedelta):
            return obj.seconds
        if isinstance(obj, time):
            return obj.strftime("%H:%M:%S")
        if isinstance(obj, date):
            return obj.strftime("%Y-%m-%d")
        if isinstance(obj, str):
            return obj
        if isinstance(obj, Decimal):
            return float(obj)
        return json.JSONEncoder.default(self, obj)


def labelize(string):
    """return the label for a criteria"""
    return {
        'new': 'New visits',
        'unique': 'Visits',
        'all': 'Page hits',
        'spent_time': 'Time spent on site',
        'hour': 'Visits per hour'
    }[string]


def date_to_time(date):
    """Convert a python date to js stamp"""
    return int(1000 * timegm(date.timetuple()))


def time_to_date(time):
    """Convert a js stamp to python date"""
    return datetime.fromtimestamp(time / 1000)


def on(host):
    """Generate a filter on a host """
    if host == 'all':
        return Visit.host == Visit.host
    return Visit.host.like('%' + host)


def between(from_date, to_date):
    """Generate a filter between 2 dates"""
    return ((time_to_date(from_date) <= Visit.date) &
            (Visit.date < time_to_date(to_date) + timedelta(1)))


def transform_for_pie(results, site, from_date, to_date,
                      reparse_referrer=False):
    """Transform result for pie display"""
    visits = [{'label': (parse_referrer(visit.key, host_only=True,
                                        second_pass=True)
                         if reparse_referrer else visit.key),
               'data': visit.count}
              for visit in results]
    all_visits = (Visit.query
                  .filter(on(site))
                  .filter(between(from_date, to_date))
                  .count())
    other = all_visits - sum(visit['data'] for visit in visits)
    if other:
        visits = visits + [{'label': 'Other', 'data': other}]
    return {'list': visits}


def make_time_serie(results, criteria, from_date, to_date):
    """Create a serie with 0 days at 0 for Flot from request"""
    visits = {date_to_time(visit.key): visit.count for visit in results}

    for time in range(from_date, to_date, 1000 * 3600 * 24):
        visits[time] = visits.get(time, 0)
    data = visits.items()
    data.sort()
    return {'label': labelize(criteria),
            'data': data}


def make_serie(results, criteria, is_time=False):
    """Create a serie for Flot from request"""
    return {'label': labelize(criteria),
            'data': [(visit.key, visit.count)
                     for visit in results]}


def parse_referrer(referrer, with_query=False, host_only=False,
                   second_pass=False):
    """Return a pretty format for most search engines"""
    if referrer:
        up = urlparse(referrer)
        netloc = up.netloc
        if not netloc:
            if with_query:
                return "Local: %s" % up.path
            if second_pass:
                return referrer
            return "Local"

        query = up.query
        parsed = parse_qs(query.decode('latin1').encode('utf-8'))<commit_msg>Fix referre (???)<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2011 by <NAME>, Kozea
# This file is part of pystil, licensed under a 3-clause BSD license.

"""Utility functions to help processing data"""

from datetime import datetime, timedelta
from urlparse import urlparse, parse_qs
from pystil.db import Visit, fields
from datetime import date, time
from calendar import timegm
from decimal import Decimal
import re
import json

IPV4RE = re.compile(r"(\d{1,3}(\.|$)){4}")
BROWSER_VERSION_NUMBERS = {
    'opera': 1,
    'safari': 1,
    'chrome': 1}


class PystilEncoder(json.JSONEncoder):

    def default(self, obj):
        if isinstance(obj, datetime):
            return obj.strftime("%Y-%m-%d %H:%M:%S")
        if isinstance(obj, timedelta):
            return obj.seconds
        if isinstance(obj, time):
            return obj.strftime("%H:%M:%S")
        if isinstance(obj, date):
            return obj.strftime("%Y-%m-%d")
        if isinstance(obj, str):
            return obj
        if isinstance(obj, Decimal):
            return float(obj)
        return json.JSONEncoder.default(self, obj)


def labelize(string):
    """return the label for a criteria"""
    return {
        'new': 'New visits',
        'unique': 'Visits',
        'all': 'Page hits',
        'spent_time': 'Time spent on site',
        'hour': 'Visits per hour'
    }[string]


def date_to_time(date):
    """Convert a python date to js stamp"""
    return int(1000 * timegm(date.timetuple()))


def time_to_date(time):
    """Convert a js stamp to python date"""
    return datetime.fromtimestamp(time / 1000)


def on(host):
    """Generate a filter on a host """
    if host == 'all':
        return Visit.host == Visit.host
    return Visit.host.like('%' + host)


def between(from_date, to_date):
    """Generate a filter between 2 dates"""
    return ((time_to_date(from_date) <= Visit.date) &
            (Visit.date < time_to_date(to_date) + timedelta(1)))


def transform_for_pie(results, site, from_date, to_date,
                      reparse_referrer=False):
    """Transform result for pie display"""
    visits = [{'label': (parse_referrer(visit.key, host_only=True,
                                        second_pass=True)
                         if reparse_referrer else visit.key),
               'data': visit.count}
              for visit in results]
    all_visits = (Visit.query
                  .filter(on(site))
                  .filter(between(from_date, to_date))
                  .count())
    other = all_visits - sum(visit['data'] for visit in visits)
    if other:
        visits = visits + [{'label': 'Other', 'data': other}]
    return {'list': visits}


def make_time_serie(results, criteria, from_date, to_date):
    """Create a serie with 0 days at 0 for Flot from request"""
    visits = {date_to_time(visit.key): visit.count for visit in results}

    for time in range(from_date, to_date, 1000 * 3600 * 24):
        visits[time] = visits.get(time, 0)
    data = visits.items()
    data.sort()
    return {'label': labelize(criteria),
            'data': data}


def make_serie(results, criteria, is_time=False):
    """Create a serie for Flot from request"""
    return {'label': labelize(criteria),
            'data': [(visit.key, visit.count)
                     for visit in results]}


def parse_referrer(referrer, with_query=False, host_only=False,
                   second_pass=False):
    """Return a pretty format for most search engines"""
    if referrer:
        up = urlparse(referrer.decode('latin1').encode('utf8'))
        netloc = up.netloc
        if not netloc:
            if with_query:
                return "Local: %s" % up.path
            if second_pass:
                return referrer
            return "Local"

        query = up.query
        parsed = parse_qs(query.decode('latin1').encode('utf-8'))
{'fraction_non_alphanumeric': 0.12760751995879474, 'fraction_numerical': 0.010043780582024209, 'mean_word_length': 2.4200792602377805, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3786472', 'n_tokens_mistral': 2419, 'n_tokens_neox': 2316, 'n_words': 687}