<commit_before>import couchdb
import couchdb.design
import os.path
import csv
import os
import os.path
import arrow
import time
import yaml
import requests
import argparse
import requests_cache
from requests.exceptions import RequestException
from yaml.scanner import ScannerError
from dateutil.parser import parse
from config import Config
from design import bids_owner_date, tenders_owner_date, jsonpatch
from couchdb.design import ViewDefinition
from logging import getLogger
from reports.helpers import get_cmd_parser, create_db_url, Kind, Status


views = [bids_owner_date, tenders_owner_date]


requests_cache.install_cache('audit_cache')
NEW_ALG_DATE = "2017-08-16"


class BaseUtility(object):

    def __init__(self, operation, rev=False):
        self.rev = rev
        self.headers = None
        self.operation = operation
        self.threshold_date = '2017-01-01T00:00+02:00'

    def _initialize(self, broker, period, config, tz=''):
        self.broker = broker
        self.config = Config(config, self.rev)
        self.start_date = ''
        self.end_date = ''
        self.timezone = tz
        self.payments = []

        if period:
            if len(period) == 1:
                self.start_date = self.convert_date(period[0])
            if len(period) == 2:
                self.start_date = self.convert_date(period[0])
                self.end_date = self.convert_date(period[1])
        self.get_db_connection()
        self.Logger = getLogger(self.operation)
        self.payments = self.config.payments(False)
        self.payments_before = self.config.payments(True)

    def get_db_connection(self):
        host = self.config.get_option('db', 'host')
        port = self.config.get_option('db', 'port')
        user_name = self.config.get_option('user', 'username')
        user_password = self.config.get_option('user', 'password')

        db_name = self.config.get_option('db', 'name')

        self.db = couchdb.Database(
            create_db_url(host, port, user_name, user_password, db_name),
            session=couchdb.Session(retry_delays=range(10))
        )

        a_name = self.config.get_option('admin', 'username')
        a_password = self.config.get_option('admin', 'password')
        self.adb = couchdb.Database(
            create_db_url(host, port, a_name, a_password, db_name)
        )

    def row(self):
        raise NotImplemented

    def rows(self):
        raise NotImplemented

    def convert_date(self, date):
        if len(date) < 3:
            date = time.strftime("%Y-%m-") + date
        date = arrow.get(parse(date), self.timezone)
        res = date.to('UTC').strftime("%Y-%m-%dT%H:%M:%S.%f")
        return res

    def get_payment(self, value, before_2017=False):
        p = self.payments_before if before_2017 else self.payments
        for index, threshold in enumerate(self.config.thresholds):
            if value <= threshold:
                return p[index]
        return p[-1]

    def _sync_views(self):

        ViewDefinition.sync_many(self.adb, views)
        _id = '_design/report'
        original = self.adb.get(_id)
        original['views']['lib'] = {
            'jsonpatch': jsonpatch
        }
        self.adb.save(original)
        ViewDefinition.sync_many(self.adb, views)

    def get_response(self):
        self._sync_views()

        if not self.view:
            raise NotImplemented

        if not self.start_date and not self.end_date:
            self.response = self.db.iterview(
                self.view, 1000,
                startkey=(self.broker, ""),
                endkey=(self.broker, "9999-12-30T00:00:00.000000")
            )
        elif self.start_date and not self.end_date:
            self.response = self.db.iterview(
                self.view, 1000,
                startkey=(self.broker, self.start_date),
                endkey=(self.broker, "9999-12-30T00:00:00.000000")
            )
        else:
            self.response = self.db.iterview(
                self.view, 1000,
                startkey=(self.broker, self.start_date),
                endkey=(self.broker, self.end_date)
            )

    def out_name(self):
        start = ''
        end = ''
        if self.start_date:
            start = arrow.get(parse(self.start_date))\
                .to('Europe/Kiev').strftime("%Y-%m-%d")
        if self.end_date:
            end = arrow.get(parse(self.end_date))\
                .to('Europe/Kiev').strftime("%Y-%m-%d")
        name = "{}@{}--{}-{}.csv".format(
            self.broker,
            start,
            end,
            self.operation
        )
        self.put_path = os.path.join(self.config.out_path, name)

    def write_csv(self):
        if not self.headers:
            raise ValueError
        if not os.path.exists(os.path.dirname(os.path.abspath(self.put_path))):
            os.makedirs(os.path.dirname(os.path.abspath(self.put_path)))
        with open(self.put_path, 'w') as out_file:
            writer = csv.writer(out_file)
            writer.writerow(self.headers)
            for row in self.rows():
                writer.writerow(row)

    def run(self):
        self.get_response()
        self.out_name()
        self.write_csv()


class BaseBidsUtility(BaseUtility):

    def __init__(self, operation):
        super(BaseBidsUtility, self).__init__(operation)
        self.view = 'report/bids_owner_date'
        self.skip_bids = set()<commit_msg>Use new logger<commit_after>import couchdb
import couchdb.design
import os.path
import csv
import os
import os.path
import arrow
import time
import yaml
import requests
import argparse
import requests_cache
from reports.log import getLogger
from requests.exceptions import RequestException
from yaml.scanner import ScannerError
from dateutil.parser import parse
from config import Config
from design import bids_owner_date, tenders_owner_date, jsonpatch
from couchdb.design import ViewDefinition
from reports.helpers import get_cmd_parser, create_db_url, Kind, Status


VIEWS = [bids_owner_date, tenders_owner_date]

NEW_ALG_DATE = "2017-08-16"

requests_cache.install_cache('audit_cache')


class BaseUtility(object):

    def __init__(self, operation, rev=False):
        self.rev = rev
        self.headers = None
        self.operation = operation
        self.threshold_date = '2017-01-01T00:00+02:00'

    def _initialize(self, broker, period, config, tz=''):
        self.broker = broker
        self.config = Config(config, self.rev)
        self.start_date = ''
        self.end_date = ''
        self.timezone = tz
        self.payments = []

        if period:
            if len(period) == 1:
                self.start_date = self.convert_date(period[0])
            if len(period) == 2:
                self.start_date = self.convert_date(period[0])
                self.end_date = self.convert_date(period[1])
        self.get_db_connection()
        self.Logger = getLogger("BILLING")
        self.payments = self.config.payments(False)
        self.payments_before = self.config.payments(True)

    def get_db_connection(self):
        host = self.config.get_option('db', 'host')
        port = self.config.get_option('db', 'port')
        user_name = self.config.get_option('user', 'username')
        user_password = self.config.get_option('user', 'password')

        db_name = self.config.get_option('db', 'name')

        self.db = couchdb.Database(
            create_db_url(host, port, user_name, user_password, db_name),
            session=couchdb.Session(retry_delays=range(10))
        )

        a_name = self.config.get_option('admin', 'username')
        a_password = self.config.get_option('admin', 'password')
        self.adb = couchdb.Database(
            create_db_url(host, port, a_name, a_password, db_name)
        )

    def row(self):
        raise NotImplemented

    def rows(self):
        raise NotImplemented

    def convert_date(self, date):
        if len(date) < 3:
            date = time.strftime("%Y-%m-") + date
        date = arrow.get(parse(date), self.timezone)
        res = date.to('UTC').strftime("%Y-%m-%dT%H:%M:%S.%f")
        return res

    def get_payment(self, value, before_2017=False):
        p = self.payments_before if before_2017 else self.payments
        for index, threshold in enumerate(self.config.thresholds):
            if value <= threshold:
                return p[index]
        return p[-1]

    def _sync_views(self):

        ViewDefinition.sync_many(self.adb, VIEWS)
        _id = '_design/report'
        original = self.adb.get(_id)
        original['views']['lib'] = {
            'jsonpatch': jsonpatch
        }
        self.adb.save(original)
        ViewDefinition.sync_many(self.adb, VIEWS)

    def get_response(self):
        self._sync_views()

        if not self.view:
            raise NotImplemented

        if not self.start_date and not self.end_date:
            self.response = self.db.iterview(
                self.view, 1000,
                startkey=(self.broker, ""),
                endkey=(self.broker, "9999-12-30T00:00:00.000000")
            )
        elif self.start_date and not self.end_date:
            self.response = self.db.iterview(
                self.view, 1000,
                startkey=(self.broker, self.start_date),
                endkey=(self.broker, "9999-12-30T00:00:00.000000")
            )
        else:
            self.response = self.db.iterview(
                self.view, 1000,
                startkey=(self.broker, self.start_date),
                endkey=(self.broker, self.end_date)
            )

    def out_name(self):
        start = ''
        end = ''
        if self.start_date:
            start = arrow.get(parse(self.start_date))\
                .to('Europe/Kiev').strftime("%Y-%m-%d")
        if self.end_date:
            end = arrow.get(parse(self.end_date))\
                .to('Europe/Kiev').strftime("%Y-%m-%d")
        name = "{}@{}--{}-{}.csv".format(
            self.broker,
            start,
            end,
            self.operation
        )
        self.put_path = os.path.join(self.config.out_path, name)

    def write_csv(self):
        if not self.headers:
            raise ValueError
        if not os.path.exists(os.path.dirname(os.path.abspath(self.put_path))):
            os.makedirs(os.path.dirname(os.path.abspath(self.put_path)))
        with open(self.put_path, 'w') as out_file:
            writer = csv.writer(out_file)
            writer.writerow(self.headers)
            for row in self.rows():
                writer.writerow(row)

    def run(self):
        self.Logger.info("Start generating")
        self.get_response()
        self.out_name()
        self.write_csv()


class BaseBidsUtility(BaseUtility):

    def __init__(self, operation):
        super(BaseBidsUtility, self).__init__(operation)
        self.view = 'report/bids_owner_date'
        self.skip_bids = set()
{'fraction_non_alphanumeric': 0.11710803628699716, 'fraction_numerical': 0.01704389260514982, 'mean_word_length': 2.177292576419214, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5346901', 'n_tokens_mistral': 3542, 'n_tokens_neox': 3320, 'n_words': 647}