<commit_before>import datetime
import json
from urllib.error import HTTPError, URLError
from django import forms
from django.core.urlresolvers import reverse
from django.forms import Widget
import urllib
from django.utils import formats
from django.utils.html import format_html
from django.utils.safestring import mark_safe
from jet.modules import DashboardModule
from django.utils.translation import ugettext_lazy as _
from django.conf import settings

JET_MODULE_YANDEX_METRIKA_CLIENT_ID = getattr(settings, 'JET_MODULE_YANDEX_METRIKA_CLIENT_ID', '')
JET_MODULE_YANDEX_METRIKA_CLIENT_SECRET = getattr(settings, 'JET_MODULE_YANDEX_METRIKA_CLIENT_SECRET', '')


class YandexMetrikaClient:
    OAUTH_BASE_URL = 'https://oauth.yandex.ru/'
    API_BASE_URL = 'https://api-metrika.yandex.ru/'
    CLIENT_ID = JET_MODULE_YANDEX_METRIKA_CLIENT_ID
    CLIENT_SECRET = JET_MODULE_YANDEX_METRIKA_CLIENT_SECRET

    def __init__(self, access_token=None):
        self.access_token = access_token

    def request(self, base_url, url, data=None, headers=None):
        url = '%s%s' % (base_url, url)

        if data is not None:
            data = urllib.parse.urlencode(data).encode()

        if headers is None:
            headers = {}

        req = urllib.request.Request(url, data, headers)

        try:
            f = urllib.request.urlopen(req)
            result = f.read().decode()
            result = json.loads(result)
        except URLError as e:
            return None, e

        return result, None

    def get_oauth_authorize_url(self, state=''):
        return '%sauthorize' \
               '?response_type=code' \
               '&state=%s' \
               '&client_id=%s' % (self.OAUTH_BASE_URL, state, self.CLIENT_ID)

    def oauth_request(self, url, data=None):
        return self.request(self.OAUTH_BASE_URL, url, data)

    def oath_token_request(self, code):
        data = {
            'grant_type': 'authorization_code',
            'code': code,
            'client_id': self.CLIENT_ID,
            'client_secret': self.CLIENT_SECRET
        }
        return self.oauth_request('token', data)

    def api_request(self, url, data=None):
        headers = None
        if self.access_token is not None:
            headers = {'Authorization': 'OAuth %s' % self.access_token}
        return self.request(self.API_BASE_URL, url, data, headers)

    def api_counters_request(self):
        return self.api_request('counters.json')

    def api_stat_traffic_summary(self, counter, date1, date2, group=None):
        if group is None:
            group = 'day'
        return self.api_request('stat/traffic/summary.json?id=%s&date1=%s&date2=%s&group=%s' % (
            counter,
            date1.strftime('%Y%m%d'),
            date2.strftime('%Y%m%d'),
            group
        ))


class AccessTokenWidget(Widget):
    module = None

    def render(self, name, value, attrs=None):
        if value and len(value) > 0:
            link = '<a href="%s">Revoke access</a>' % reverse('jet:yandex-metrika-revoke', kwargs={'pk': self.module.model.pk})
        else:
            link = '<a href="%s">Grant access</a>' % reverse('jet:yandex-metrika-grant', kwargs={'pk': self.module.model.pk})
        return format_html('%s<input type="hidden" name="access_token" value="%s">' % (link, value))


class YandexMetrikaSettingsForm(forms.Form):
    access_token = forms.CharField(label=_('Token'), widget=AccessTokenWidget)
    counter = forms.ChoiceField(label=_('Counter'))
    period = forms.ChoiceField(label=_('Statistics period'), choices=(
        (0, _('Today')),
        (6, _('Last week')),
        (30, _('Last month')),
        (31 * 3 - 1, _('Last quarter')),
        (364, _('Last year')),
    ))

    def set_module(self, module):
        self.fields['access_token'].widget.module = module
        self.set_counter_choices(module)

    def set_counter_choices(self, module):
        counters = module.counters()
        if counters is not None:
            self.fields['counter'].choices = (('', _('-- none --')),)
            self.fields['counter'].choices.extend(map(lambda x: (x['id'], x['site']), counters))
        else:
            label = _('grant access first') if module.access_token is None else _('counters loading failed')
            self.fields['counter'].choices = (('', '-- %s -- ' % label),)


class YandexMetrikaChartSettingsForm(YandexMetrikaSettingsForm):
    show = forms.ChoiceField(label=_('Show'), choices=(
        ('visitors', _('Visitors')),
        ('visits', _('Visits')),
        ('page_views', _('Views')),
    ))
    group = forms.ChoiceField(label=_('Group'), choices=(
        ('day', _('By day')),
        ('week', _('By week')),
        ('month', _('By month')),
    ))


class YandexMetrikaPeriodVisitorsSettingsForm(YandexMetrikaSettingsForm):
    group = forms.ChoiceField(label=_('Group'), choices=(
        ('day', _('By day')),
        ('week', _('By week')),
        ('month', _('By month')),
    ))


class YandexMetrika(DashboardModule):
    settings_form = YandexMetrikaSettingsForm
    ajax_load = True
    contrast = True
    period = None
    access_token = None
    expires_in = None
    token_type = None
    counter = None
    error = None

    class Media:
        js = ('jet/vendor/chart.js/Chart.min.js', 'jet/modules/yandex_metrika.js')

    def __init__(self, title=None, period=None, **kwargs):
        kwargs.update({'period': period})
        super(YandexMetrika, self).__init__(title, **kwargs)

    def settings_dict(self):
        return {
            'period': self.period,
            'access_token': self.access_token,
            'expires_in': self.expires_in,
            'token_type': self.token_type,
            'counter': self.counter
        }

    def load_settings(self, settings):
        try:
            self.period = int(settings.get('period'))
        except TypeError:
            self.period = 0
        self.access_token = settings.get('access_token')
        self.expires_in = settings.get('expires_in')
        self.token_type = settings.get('token_type')
        self.counter = settings.get('counter')

    def init_with_context(self, context):
        raise NotImplementedError('subclasses of YandexMetrika must provide a init_with_context() method')

    def counters(self):
        client = YandexMetrikaClient(self.access_token)
        counters, exception = client.api_counters_request()

        if counters is not None:
            return counters['counters']
        else:
            return None

    def format_grouped_date(self, date, group):
        if group == 'week':
            date = '%s — %s' % (
                (date - datetime.timedelta(days=7)).strftime('%d.%m'),
                date.strftime('%d.%m')
            )
        elif group == 'month':
            date = date.strftime('%b, %Y')
        else:
            date = formats.date_format(date, 'DATE_FORMAT')
        return date


class YandexMetrikaVisitorsTotals(YandexMetrika):
    title = _('Yandex Metrika visitors totals')<commit_msg>Fix unicode<commit_after># encoding: utf-8
import datetime
import json
from urllib.error import HTTPError, URLError
from django import forms
from django.core.urlresolvers import reverse
from django.forms import Widget
import urllib
from django.utils import formats
from django.utils.html import format_html
from django.utils.safestring import mark_safe
from jet.modules import DashboardModule
from django.utils.translation import ugettext_lazy as _
from django.conf import settings

JET_MODULE_YANDEX_METRIKA_CLIENT_ID = getattr(settings, 'JET_MODULE_YANDEX_METRIKA_CLIENT_ID', '')
JET_MODULE_YANDEX_METRIKA_CLIENT_SECRET = getattr(settings, 'JET_MODULE_YANDEX_METRIKA_CLIENT_SECRET', '')


class YandexMetrikaClient:
    OAUTH_BASE_URL = 'https://oauth.yandex.ru/'
    API_BASE_URL = 'https://api-metrika.yandex.ru/'
    CLIENT_ID = JET_MODULE_YANDEX_METRIKA_CLIENT_ID
    CLIENT_SECRET = JET_MODULE_YANDEX_METRIKA_CLIENT_SECRET

    def __init__(self, access_token=None):
        self.access_token = access_token

    def request(self, base_url, url, data=None, headers=None):
        url = '%s%s' % (base_url, url)

        if data is not None:
            data = urllib.parse.urlencode(data).encode()

        if headers is None:
            headers = {}

        req = urllib.request.Request(url, data, headers)

        try:
            f = urllib.request.urlopen(req)
            result = f.read().decode()
            result = json.loads(result)
        except URLError as e:
            return None, e

        return result, None

    def get_oauth_authorize_url(self, state=''):
        return '%sauthorize' \
               '?response_type=code' \
               '&state=%s' \
               '&client_id=%s' % (self.OAUTH_BASE_URL, state, self.CLIENT_ID)

    def oauth_request(self, url, data=None):
        return self.request(self.OAUTH_BASE_URL, url, data)

    def oath_token_request(self, code):
        data = {
            'grant_type': 'authorization_code',
            'code': code,
            'client_id': self.CLIENT_ID,
            'client_secret': self.CLIENT_SECRET
        }
        return self.oauth_request('token', data)

    def api_request(self, url, data=None):
        headers = None
        if self.access_token is not None:
            headers = {'Authorization': 'OAuth %s' % self.access_token}
        return self.request(self.API_BASE_URL, url, data, headers)

    def api_counters_request(self):
        return self.api_request('counters.json')

    def api_stat_traffic_summary(self, counter, date1, date2, group=None):
        if group is None:
            group = 'day'
        return self.api_request('stat/traffic/summary.json?id=%s&date1=%s&date2=%s&group=%s' % (
            counter,
            date1.strftime('%Y%m%d'),
            date2.strftime('%Y%m%d'),
            group
        ))


class AccessTokenWidget(Widget):
    module = None

    def render(self, name, value, attrs=None):
        if value and len(value) > 0:
            link = '<a href="%s">Revoke access</a>' % reverse('jet:yandex-metrika-revoke', kwargs={'pk': self.module.model.pk})
        else:
            link = '<a href="%s">Grant access</a>' % reverse('jet:yandex-metrika-grant', kwargs={'pk': self.module.model.pk})
        return format_html('%s<input type="hidden" name="access_token" value="%s">' % (link, value))


class YandexMetrikaSettingsForm(forms.Form):
    access_token = forms.CharField(label=_('Token'), widget=AccessTokenWidget)
    counter = forms.ChoiceField(label=_('Counter'))
    period = forms.ChoiceField(label=_('Statistics period'), choices=(
        (0, _('Today')),
        (6, _('Last week')),
        (30, _('Last month')),
        (31 * 3 - 1, _('Last quarter')),
        (364, _('Last year')),
    ))

    def set_module(self, module):
        self.fields['access_token'].widget.module = module
        self.set_counter_choices(module)

    def set_counter_choices(self, module):
        counters = module.counters()
        if counters is not None:
            self.fields['counter'].choices = (('', _('-- none --')),)
            self.fields['counter'].choices.extend(map(lambda x: (x['id'], x['site']), counters))
        else:
            label = _('grant access first') if module.access_token is None else _('counters loading failed')
            self.fields['counter'].choices = (('', '-- %s -- ' % label),)


class YandexMetrikaChartSettingsForm(YandexMetrikaSettingsForm):
    show = forms.ChoiceField(label=_('Show'), choices=(
        ('visitors', _('Visitors')),
        ('visits', _('Visits')),
        ('page_views', _('Views')),
    ))
    group = forms.ChoiceField(label=_('Group'), choices=(
        ('day', _('By day')),
        ('week', _('By week')),
        ('month', _('By month')),
    ))


class YandexMetrikaPeriodVisitorsSettingsForm(YandexMetrikaSettingsForm):
    group = forms.ChoiceField(label=_('Group'), choices=(
        ('day', _('By day')),
        ('week', _('By week')),
        ('month', _('By month')),
    ))


class YandexMetrika(DashboardModule):
    settings_form = YandexMetrikaSettingsForm
    ajax_load = True
    contrast = True
    period = None
    access_token = None
    expires_in = None
    token_type = None
    counter = None
    error = None

    class Media:
        js = ('jet/vendor/chart.js/Chart.min.js', 'jet/modules/yandex_metrika.js')

    def __init__(self, title=None, period=None, **kwargs):
        kwargs.update({'period': period})
        super(YandexMetrika, self).__init__(title, **kwargs)

    def settings_dict(self):
        return {
            'period': self.period,
            'access_token': self.access_token,
            'expires_in': self.expires_in,
            'token_type': self.token_type,
            'counter': self.counter
        }

    def load_settings(self, settings):
        try:
            self.period = int(settings.get('period'))
        except TypeError:
            self.period = 0
        self.access_token = settings.get('access_token')
        self.expires_in = settings.get('expires_in')
        self.token_type = settings.get('token_type')
        self.counter = settings.get('counter')

    def init_with_context(self, context):
        raise NotImplementedError('subclasses of YandexMetrika must provide a init_with_context() method')

    def counters(self):
        client = YandexMetrikaClient(self.access_token)
        counters, exception = client.api_counters_request()

        if counters is not None:
            return counters['counters']
        else:
            return None

    def format_grouped_date(self, date, group):
        if group == 'week':
            date = u'%s — %s' % (
                (date - datetime.timedelta(days=7)).strftime('%d.%m'),
                date.strftime('%d.%m')
            )
        elif group == 'month':
            date = date.strftime('%b, %Y')
        else:
            date = formats.date_format(date, 'DATE_FORMAT')
        return date


class YandexMetrikaVisitorsTotals(YandexMetrika):
    title = _('Yandex Metrika visitors totals')
{'fraction_non_alphanumeric': 0.13332860285247997, 'fraction_numerical': 0.0029092457248279286, 'mean_word_length': 2.724630021141649, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1871796', 'n_tokens_mistral': 4516, 'n_tokens_neox': 4402, 'n_words': 915}