<commit_before>import argparse
import os
import csv
import sys
import itertools
import subprocess as sb
from datetime import datetime
from logging.config import fileConfig
from ConfigParser import ConfigParser

from reports.utilities.invoices import InvoicesUtility
from reports.utilities.refunds import RefundsUtility
from reports.utilities.bids import BidsUtility, HEADERS
from reports.utilities.tenders import TendersUtility
from reports.helpers import parse_period_string
from reports.log import getLogger
from reports.utilities.send import Porter
from reports.utilities.zip import compress


SCRIPTS = [BidsUtility, InvoicesUtility, TendersUtility, RefundsUtility]
YES = ['y', 'yes', 'true', 't']
NO = ['n', 'no', 'false', 'f']
DEFAULT_KINDS = ['general', 'special', 'defense', 'other', '_kind']
DEFAULT_INCLUDE = "bids,invoices,tenders,refunds"
CONFIG = ConfigParser()

parser = argparse.ArgumentParser(description="Openprocurement Billing")
parser.add_argument('-c', '--config', required=True)
parser.add_argument('--brokers', dest='brokers', action="store")
parser.add_argument('--period', dest='period', action='store')
parser.add_argument('--notify', action='store', default='no')
parser.add_argument('--timestamp', action='store')
parser.add_argument('--include', action='store', default=DEFAULT_INCLUDE)
parser.add_argument('--notify-brokers', action="append")
parser.add_argument('--timezone', default='Europe/Kiev')
ARGS = parser.parse_args()

TIMESTAMP = ARGS.timestamp or datetime.now().strftime("%Y-%m-%d/%H-%M-%S-%f")
CONFIG.read(ARGS.config)
fileConfig(ARGS.config)

LOGGER = getLogger('BILLING')
PORTER = Porter(ARGS.config, TIMESTAMP, ARGS.notify_brokers)


def upload_and_notify(files):
    ctx = PORTER.upload_files(files)
    if ARGS.notify:
        PORTER.postman.send_emails(ctx)
    return ctx


def send_emails_from_existing_files():
    ctx = PORTER.create_emails_context_from_existing_prefix()
    if ARGS.notify:
        PORTER.postman.send_emails(ctx)
    return [entry['broker'] for entry in ctx]


def generate_for_broker(broker, period, timezone='Europe/Kiev'):
    utilities = map(lambda u: u(broker, period, ARGS.config, timezone),
                    SCRIPTS)
    for ut in utilities:
        if isinstance(ut, (TendersUtility, RefundsUtility)):
            ut.kinds = DEFAULT_KINDS
        ut.run()


def construct_filenames_of_generation(type, broker, period):
    start, end = period
    return "{}@{}--{}-{}.csv".format(broker, start, end, type)


def create_all_bids_csv(brokers, period):
    path = CONFIG.get('out', 'out_dir')
    files = [
        construct_filenames_of_generation('bids', broker, period)
        for broker in brokers
    ]
    all_file = "all@{0}--{1}-bids.csv".format(*period)
    with open(os.path.join(path, all_file), 'w') as out_stream:
        results_file = csv.writer(out_stream)
        for header, iterator in zip(
                (HEADERS, ['after_2017-01-01']),
                (itertools.takewhile, itertools.dropwhile)
                ):
            results_file.writerow(header)
            for file in files:
                with open(os.path.join(path, file)) as broker_file:
                    rows = iterator(
                            lambda line: not line[0].startswith('after'),
                            csv.reader(broker_file)
                            )
                    try:
                        skip = next(rows)
                        LOGGER.warning("Skipped {}".format(skip))
                    except StopIteration:
                        continue

                    for line in rows:
                        if line:
                            results_file.writerow(line)


def get_password_for_broker(broker):
    cmd = 'pass {}'.format(
        os.path.join(
            CONFIG.get('brokers_keys', 'path'),
            broker
            )
        )
    try:
        password = sb.check_output(cmd, shell=True)
        LOGGER.info("Got password for broker {}".format(broker))
        return password
    except Exception as e:
        LOGGER.fatal(
            "Command {} falied with error {}".format(cmd, e)
            )
        raise e

<commit_msg>Fixed generation all bids csv<commit_after>import argparse
import os
import csv
import sys
import itertools
import subprocess as sb
from datetime import datetime
from logging.config import fileConfig
from ConfigParser import ConfigParser

from reports.utilities.invoices import InvoicesUtility
from reports.utilities.refunds import RefundsUtility
from reports.utilities.bids import BidsUtility, HEADERS
from reports.utilities.tenders import TendersUtility
from reports.helpers import parse_period_string
from reports.log import getLogger
from reports.utilities.send import Porter
from reports.utilities.zip import compress


SCRIPTS = [BidsUtility, InvoicesUtility, TendersUtility, RefundsUtility]
YES = ['y', 'yes', 'true', 't']
NO = ['n', 'no', 'false', 'f']
DEFAULT_KINDS = ['general', 'special', 'defense', 'other', '_kind']
DEFAULT_INCLUDE = "bids,invoices,tenders,refunds"
CONFIG = ConfigParser()

parser = argparse.ArgumentParser(description="Openprocurement Billing")
parser.add_argument('-c', '--config', required=True)
parser.add_argument('--brokers', dest='brokers', action="store")
parser.add_argument('--period', dest='period', action='store')
parser.add_argument('--notify', action='store', default='no')
parser.add_argument('--timestamp', action='store')
parser.add_argument('--include', action='store', default=DEFAULT_INCLUDE)
parser.add_argument('--notify-brokers', action="append")
parser.add_argument('--timezone', default='Europe/Kiev')
ARGS = parser.parse_args()

TIMESTAMP = ARGS.timestamp or datetime.now().strftime("%Y-%m-%d/%H-%M-%S-%f")
CONFIG.read(ARGS.config)
fileConfig(ARGS.config)

LOGGER = getLogger('BILLING')
PORTER = Porter(ARGS.config, TIMESTAMP, ARGS.notify_brokers)


def upload_and_notify(files):
    ctx = PORTER.upload_files(files)
    if ARGS.notify:
        PORTER.postman.send_emails(ctx)
    return ctx


def send_emails_from_existing_files():
    ctx = PORTER.create_emails_context_from_existing_prefix()
    if ARGS.notify:
        PORTER.postman.send_emails(ctx)
    return [entry['broker'] for entry in ctx]


def generate_for_broker(broker, period, timezone='Europe/Kiev'):
    utilities = map(lambda u: u(broker, period, ARGS.config, timezone),
                    SCRIPTS)
    for ut in utilities:
        if isinstance(ut, (TendersUtility, RefundsUtility)):
            ut.kinds = DEFAULT_KINDS
        ut.run()


def construct_filenames_of_generation(type, broker, period):
    start, end = period
    return "{}@{}--{}-{}.csv".format(broker, start, end, type)


def create_all_bids_csv(brokers, period):
    path = CONFIG.get('out', 'out_dir')
    files = [
        construct_filenames_of_generation('bids', broker, period)
        for broker in brokers
        if broker != 'all'
    ]
    all_file = "all@{0}--{1}-bids.csv".format(*period)
    with open(os.path.join(path, all_file), 'w') as out_stream:
        results_file = csv.writer(out_stream)
        results_file.writerow(HEADERS)
        for header, iterator in zip(
                (['after_2017-01-01'], ['after 2017-08-16']),
                (itertools.takewhile, itertools.dropwhile)
                ):
            results_file.writerow(header)
            for file in files:
                with open(os.path.join(path, file)) as broker_file:
                    reader = csv.reader(broker_file)
                    next(reader)
                    for line in iterator(
                            lambda line: line != ['after 2017-08-16'],
                            reader
                            ):
                        if line and line != header:
                            results_file.writerow(line)


def get_password_for_broker(broker):
    cmd = 'pass {}'.format(
        os.path.join(
            CONFIG.get('brokers_keys', 'path'),
            broker
            )
        )
    try:
        password = sb.check_output(cmd, shell=True)
        LOGGER.info("Got password for broker {}".format(broker))
        return password
    except Exception as e:
        LOGGER.fatal(
            "Command {} falied with error {}".format(cmd, e)
            )
        raise e


{'fraction_non_alphanumeric': 0.12235777267785965, 'fraction_numerical': 0.004469138784877401, 'mean_word_length': 2.901979264844486, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7418956', 'n_tokens_mistral': 2546, 'n_tokens_neox': 2478, 'n_words': 560}