<commit_before># -*- coding: utf-8 -*-
# © 2014-2016 <NAME>/NV (<http://acsone.eu>)
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html).

from collections import defaultdict, OrderedDict
import datetime
import dateutil
from itertools import izip
import logging
import re
import time

import pytz

from openerp import api, fields, models, _
from openerp.exceptions import UserError
from openerp.tools.safe_eval import safe_eval

from .aep import AccountingExpressionProcessor as AEP
from .aggregate import _sum, _avg, _min, _max
from .accounting_none import AccountingNone
from .simple_array import SimpleArray
from .mis_safe_eval import mis_safe_eval, DataError

_logger = logging.getLogger(__name__)


class AutoStruct(object):

    def __init__(self, **kwargs):
        for k, v in kwargs.items():
            setattr(self, k, v)


class KpiMatrixRow(object):

    def __init__(self, matrix, kpi, account_id=None, parent_row=None):
        self._matrix = matrix
        self.kpi = kpi
        self.account_id = account_id
        self.comment = ''
        self.parent_row = parent_row

    @property
    def description(self):
        if not self.account_id:
            return self.kpi.description
        else:
            return self._matrix.get_account_name(self.account_id)

    @property
    def style(self):
        if not self.account_id:
            return self.kpi.style
        else:
            return None  # TODO style for expanded accounts

    def iter_cell_tuples(self, cols=None):
        if cols is None:
            cols = self._matrix.iter_cols()
        for col in cols:
            yield col.get_cell_tuple_for_row(self)

    def iter_cells(self, subcols=None):
        if subcols is None:
            subcols = self._matrix.iter_subcols()
        for subcol in subcols:
            yield subcol.get_cell_for_row(self)


class KpiMatrixCol(object):

    def __init__(self, description, comment, locals_dict, subkpis):
        self.description = description
        self.comment = comment
        self.locals_dict = locals_dict
        self.colspan = subkpis and len(subkpis) or 1
        self._subcols = []
        self.subkpis = subkpis
        if not subkpis:
            subcol = KpiMatrixSubCol(self, '', '', 0)
            self._subcols.append(subcol)
        else:
            for i, subkpi in enumerate(subkpis):
                subcol = KpiMatrixSubCol(self, subkpi.description, '', i)
                self._subcols.append(subcol)
        self._cell_tuples_by_row = {}  # {row: (cells tuple)}

    def _set_cell_tuple(self, row, cell_tuple):
        self._cell_tuples_by_row[row] = cell_tuple

    def iter_subcols(self):
        return self._subcols

    def iter_cell_tuples(self):
        return self._cells_by_row.values()

    def get_cell_tuple_for_row(self, row):
        return self._cell_tuples_by_row.get(row)


class KpiMatrixSubCol(object):

    def __init__(self, col, description, comment, index=0):
        self.col = col
        self.description = description
        self.comment = comment
        self.index = index

    @property
    def subkpi(self):
        if self.col.subkpis:
            return self.col.subkpis[self.index]

    def iter_cells(self):
        for cell_tuple in self.col.iter_cell_tuples():
            yield cell_tuple[self.index]

    def get_cell_for_row(self, row):
        cell_tuple = self.col.get_cell_tuple_for_row(row)
        if cell_tuple is None:
            return None
        return cell_tuple[self.index]


class KpiMatrixCell(object):

    def __init__(self, row, subcol,
                 val, val_rendered, val_comment,
                 style=None, drilldown_key=None):
        self.row = row
        self.subcol = subcol
        self.val = val
        self.val_rendered = val_rendered
        self.val_comment = val_comment
        self.style = style
        self.drilldown_key = None


class KpiMatrix(object):

    def __init__(self, env):
        # cache language id for faster rendering
        lang_model = env['res.lang']
        lang_id = lang_model._lang_get(env.user.lang)
        self.lang = lang_model.browse(lang_id)
        self._account_model = env['account.account']
        # data structures
        # { kpi: KpiMatrixRow }
        self._kpi_rows = OrderedDict()
        # { kpi: {account_id: KpiMatrixRow} }
        self._detail_rows = {}
        # { period_key: KpiMatrixCol }
        self._cols = OrderedDict()
        # { period_key (left of comparison): [(period_key, base_period_key)] }
        self._comparison_todo = defaultdict(list)
        self._comparison_cols = defaultdict(list)
        # { account_id: account_name }
        self._account_names = {}

    def declare_kpi(self, kpi):
        """ Declare a new kpi (row) in the matrix.

        Invoke this first for all kpi, in display order.
        """
        self._kpi_rows[kpi] = KpiMatrixRow(self, kpi)
        self._detail_rows[kpi] = {}

    def declare_period(self, period_key, description, comment,
                       locals_dict, subkpis):
        """ Declare a new period (column), giving it an identifier (key).

        Invoke this and declare_comparison in display order.
        """
        self._cols[period_key] = KpiMatrixCol(description, comment,
                                              locals_dict, subkpis)

    def declare_comparison(self, period_key, base_period_key):
        """ Declare a new comparison column.

        Invoke this and declare_period in display order.
        """
        last_period_key = list(self._cols.keys())[-1]
        self._comparison_todo[last_period_key].append(
            (period_key, base_period_key))

    def set_values(self, kpi, period_key, vals):
        """ Set values for a kpi and a period.

        Invoke this after declaring the kpi and the period.
        """
        self.set_values_detail_account(kpi, period_key, None, vals)

    def set_values_detail_account(self, kpi, period_key, account_id, vals):
        """ Set values for a kpi and a period and a detail account.

        Invoke this after declaring the kpi and the period.
        """
        if not account_id:
            row = self._kpi_rows[kpi]
        else:
            kpi_row = self._kpi_rows[kpi]
            if account_id in self._detail_rows[kpi]:
                row = self._detail_rows[kpi][account_id]
            else:
                row = KpiMatrixRow(self, kpi, account_id, parent_row=kpi_row)
                self._detail_rows[kpi][account_id] = row
        col = self._cols[period_key]
        cell_tuple = []
        assert len(vals) == col.colspan
        for val, subcol in izip(vals, col.iter_subcols()):
            if isinstance(val, DataError):
                val_rendered = val.name
                val_comment = val.msg
            else:
                val_rendered = kpi.render(self.lang, val)
                if subcol.subkpi:
                    val_comment = "{}.{} = {}".format(
                        row.kpi.name,
                        subcol.subkpi.name,
                        row.kpi.get_expression_for_subkpi(subcol.subkpi))
                else:
                    val_comment = "{} = {}".format(
                        row.kpi.name,
                        row.kpi.expression)
            # TODO style
            # TODO drilldown_key
            cell = KpiMatrixCell(row, subcol, val, val_rendered, val_comment)
            cell_tuple.append(cell)
        col._set_cell_tuple(row, cell_tuple)

    def compute_comparisons(self):
        """ Compute comparisons.

        Invoke this after setting all values.
        """
        for pos_period_key, comparisons in self._comparison_todo.items():
            for period_key, base_period_key in comparisons:
                col = self._cols[period_key]
                base_col = self._cols[base_period_key]
                common_subkpis = set(col.subkpis) & set(base_col.subkpis)
                if not common_subkpis:
                    raise UserError('Columns {} and {} are not comparable'.
                                    format(col.description,
                                           base_col.description))
                description = u'{} vs {}'.\
                    format(col.description, base_col.description)
                comparison_col = KpiMatrixCol(description, None,
                                              {}, col.subkpis)
                for row in self.iter_rows():
                    cell_tuple = col.get_cell_tuple_for_row(row)
                    base_cell_tuple = base_col.get_cell_tuple_for_row(row)
                    if cell_tuple is None and base_cell_tuple is None:
                        continue
                    if cell_tuple is None:
                        vals = [AccountingNone] * len(common_subkpis)
                    else:
                        vals = [cell.val for cell in cell_tuple
                                if cell.subcol.subkpi in common_subkpis]
                    if base_cell_tuple is None:
                        base_vals = [AccountingNone] * len(common_subkpis)
                    else:
                        base_vals = [cell.val for cell in base_cell_tuple
                                     if cell.subcol.subkpi in common_subkpis]
                    comparison_cell_tuple = []
                    for val, base_val, comparison_subcol in \
                            izip(vals,
                                 base_vals,
                                 comparison_col.iter_subcols()):
                        # TODO FIXME average factors
                        delta, delta_r = row.kpi.compare_and_render(
                            self.lang, val, base_val, 1, 1)
                        comparison_cell_tuple.append(KpiMatrixCell(
                            row, comparison_subcol, delta, delta_r, None))
                    comparison_col._set_cell_tuple(row, comparison_cell_tuple)
                self._comparison_cols[pos_period_key].append(comparison_col)

    def iter_rows(self):
        """ Iterate rows in display order.

        yields KpiMatrixRow.
        """
        for kpi_row in self._kpi_rows.values():
            yield kpi_row
            detail_rows = self._detail_rows[kpi_row.kpi].values()
            detail_rows = sorted(detail_rows, key=lambda r: r.description)
            for detail_row in detail_rows:
                yield detail_row

    def iter_cols(self):
        """ Iterate columns in display order.

        yields KpiMatrixCol: one for each period or comparison.
        """
        for period_key, col in self._cols.items():
            yield col
            for comparison_col in self._comparison_cols[period_key]:
                yield comparison_col

    def iter_subcols(self):
        """ Iterate sub columns in display order.

        yields KpiMatrixSubCol: one for each subkpi in each period
        and comparison.
        """
        for col in self.iter_cols():
            for subcol in col.iter_subcols():
                yield subcol

    def _load_account_names(self):
        account_ids = set()
        for detail_rows in self._detail_rows.values():
            account_ids.update(detail_rows.keys())
        account_ids = list(account_ids)
        accounts = self._account_model.search([('id', 'in', account_ids)])
        self._account_names = {a.id: u'{} {}'.format(a.code, a.name)
                               for a in accounts}

    def get_account_name(self, account_id):
        if account_id not in self._account_names:
            self._load_account_names()
        return self._account_names[account_id]

    def as_dict(self):
        header = [{'cols': []}, {'cols': []}]
        for col in self.iter_cols():
            header[0]['cols'].append({
                'description': col.description,
                'comment': col.comment,
                'colspan': col.colspan,
            })
            for subcol in col.iter_subcols():
                header[1]['cols'].append({
                    'description': subcol.description,
                    'comment': subcol.comment,
                    'colspan': 1,
                })

        content = []
        for row in self.iter_rows():
            row_data = {
                'row_id': id(row),
                'parent_row_id': row.parent_row and id(row.parent_row) or None,
                'description': row.description,
                'comment': row.comment,
                'style': row.style and row.style.to_css_style() or '',
                'cols': []
            }
            for cell in row.iter_cells():
                if cell is None:
                    row_data['cols'].append({})
                else:
                    row_data['cols'].append({
                        'val': (cell.val
                                if cell.val is not AccountingNone else None),
                        'val_r': cell.val_rendered,
                        'val_c': cell.val_comment,
                        # TODO FIXME style
                        # TODO FIXME drilldown
                    })
            content.append(row_data)

        return {
            'header': header,
            'content': content,
        }


def _get_selection_label(selection, value):
    for v, l in selection:
        if v == value:
            return l
    return ''


def _utc_midnight(d, tz_name, add_day=0):<commit_msg>[FIX] mis_builder: minor tweak to avoid empty style attributs in widget<commit_after># -*- coding: utf-8 -*-
# © 2014-2016 ACSONE SA/NV (<http://acsone.eu>)
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html).

from collections import defaultdict, OrderedDict
import datetime
import dateutil
from itertools import izip
import logging
import re
import time

import pytz

from openerp import api, fields, models, _
from openerp.exceptions import UserError
from openerp.tools.safe_eval import safe_eval

from .aep import AccountingExpressionProcessor as AEP
from .aggregate import _sum, _avg, _min, _max
from .accounting_none import AccountingNone
from .simple_array import SimpleArray
from .mis_safe_eval import mis_safe_eval, DataError

_logger = logging.getLogger(__name__)


class AutoStruct(object):

    def __init__(self, **kwargs):
        for k, v in kwargs.items():
            setattr(self, k, v)


class KpiMatrixRow(object):

    def __init__(self, matrix, kpi, account_id=None, parent_row=None):
        self._matrix = matrix
        self.kpi = kpi
        self.account_id = account_id
        self.comment = ''
        self.parent_row = parent_row

    @property
    def description(self):
        if not self.account_id:
            return self.kpi.description
        else:
            return self._matrix.get_account_name(self.account_id)

    @property
    def style(self):
        if not self.account_id:
            return self.kpi.style
        else:
            return None  # TODO style for expanded accounts

    def iter_cell_tuples(self, cols=None):
        if cols is None:
            cols = self._matrix.iter_cols()
        for col in cols:
            yield col.get_cell_tuple_for_row(self)

    def iter_cells(self, subcols=None):
        if subcols is None:
            subcols = self._matrix.iter_subcols()
        for subcol in subcols:
            yield subcol.get_cell_for_row(self)


class KpiMatrixCol(object):

    def __init__(self, description, comment, locals_dict, subkpis):
        self.description = description
        self.comment = comment
        self.locals_dict = locals_dict
        self.colspan = subkpis and len(subkpis) or 1
        self._subcols = []
        self.subkpis = subkpis
        if not subkpis:
            subcol = KpiMatrixSubCol(self, '', '', 0)
            self._subcols.append(subcol)
        else:
            for i, subkpi in enumerate(subkpis):
                subcol = KpiMatrixSubCol(self, subkpi.description, '', i)
                self._subcols.append(subcol)
        self._cell_tuples_by_row = {}  # {row: (cells tuple)}

    def _set_cell_tuple(self, row, cell_tuple):
        self._cell_tuples_by_row[row] = cell_tuple

    def iter_subcols(self):
        return self._subcols

    def iter_cell_tuples(self):
        return self._cells_by_row.values()

    def get_cell_tuple_for_row(self, row):
        return self._cell_tuples_by_row.get(row)


class KpiMatrixSubCol(object):

    def __init__(self, col, description, comment, index=0):
        self.col = col
        self.description = description
        self.comment = comment
        self.index = index

    @property
    def subkpi(self):
        if self.col.subkpis:
            return self.col.subkpis[self.index]

    def iter_cells(self):
        for cell_tuple in self.col.iter_cell_tuples():
            yield cell_tuple[self.index]

    def get_cell_for_row(self, row):
        cell_tuple = self.col.get_cell_tuple_for_row(row)
        if cell_tuple is None:
            return None
        return cell_tuple[self.index]


class KpiMatrixCell(object):

    def __init__(self, row, subcol,
                 val, val_rendered, val_comment,
                 style=None, drilldown_key=None):
        self.row = row
        self.subcol = subcol
        self.val = val
        self.val_rendered = val_rendered
        self.val_comment = val_comment
        self.style = style
        self.drilldown_key = None


class KpiMatrix(object):

    def __init__(self, env):
        # cache language id for faster rendering
        lang_model = env['res.lang']
        lang_id = lang_model._lang_get(env.user.lang)
        self.lang = lang_model.browse(lang_id)
        self._account_model = env['account.account']
        # data structures
        # { kpi: KpiMatrixRow }
        self._kpi_rows = OrderedDict()
        # { kpi: {account_id: KpiMatrixRow} }
        self._detail_rows = {}
        # { period_key: KpiMatrixCol }
        self._cols = OrderedDict()
        # { period_key (left of comparison): [(period_key, base_period_key)] }
        self._comparison_todo = defaultdict(list)
        self._comparison_cols = defaultdict(list)
        # { account_id: account_name }
        self._account_names = {}

    def declare_kpi(self, kpi):
        """ Declare a new kpi (row) in the matrix.

        Invoke this first for all kpi, in display order.
        """
        self._kpi_rows[kpi] = KpiMatrixRow(self, kpi)
        self._detail_rows[kpi] = {}

    def declare_period(self, period_key, description, comment,
                       locals_dict, subkpis):
        """ Declare a new period (column), giving it an identifier (key).

        Invoke this and declare_comparison in display order.
        """
        self._cols[period_key] = KpiMatrixCol(description, comment,
                                              locals_dict, subkpis)

    def declare_comparison(self, period_key, base_period_key):
        """ Declare a new comparison column.

        Invoke this and declare_period in display order.
        """
        last_period_key = list(self._cols.keys())[-1]
        self._comparison_todo[last_period_key].append(
            (period_key, base_period_key))

    def set_values(self, kpi, period_key, vals):
        """ Set values for a kpi and a period.

        Invoke this after declaring the kpi and the period.
        """
        self.set_values_detail_account(kpi, period_key, None, vals)

    def set_values_detail_account(self, kpi, period_key, account_id, vals):
        """ Set values for a kpi and a period and a detail account.

        Invoke this after declaring the kpi and the period.
        """
        if not account_id:
            row = self._kpi_rows[kpi]
        else:
            kpi_row = self._kpi_rows[kpi]
            if account_id in self._detail_rows[kpi]:
                row = self._detail_rows[kpi][account_id]
            else:
                row = KpiMatrixRow(self, kpi, account_id, parent_row=kpi_row)
                self._detail_rows[kpi][account_id] = row
        col = self._cols[period_key]
        cell_tuple = []
        assert len(vals) == col.colspan
        for val, subcol in izip(vals, col.iter_subcols()):
            if isinstance(val, DataError):
                val_rendered = val.name
                val_comment = val.msg
            else:
                val_rendered = kpi.render(self.lang, val)
                if subcol.subkpi:
                    val_comment = "{}.{} = {}".format(
                        row.kpi.name,
                        subcol.subkpi.name,
                        row.kpi.get_expression_for_subkpi(subcol.subkpi))
                else:
                    val_comment = "{} = {}".format(
                        row.kpi.name,
                        row.kpi.expression)
            # TODO style
            # TODO drilldown_key
            cell = KpiMatrixCell(row, subcol, val, val_rendered, val_comment)
            cell_tuple.append(cell)
        col._set_cell_tuple(row, cell_tuple)

    def compute_comparisons(self):
        """ Compute comparisons.

        Invoke this after setting all values.
        """
        for pos_period_key, comparisons in self._comparison_todo.items():
            for period_key, base_period_key in comparisons:
                col = self._cols[period_key]
                base_col = self._cols[base_period_key]
                common_subkpis = set(col.subkpis) & set(base_col.subkpis)
                if not common_subkpis:
                    raise UserError('Columns {} and {} are not comparable'.
                                    format(col.description,
                                           base_col.description))
                description = u'{} vs {}'.\
                    format(col.description, base_col.description)
                comparison_col = KpiMatrixCol(description, None,
                                              {}, col.subkpis)
                for row in self.iter_rows():
                    cell_tuple = col.get_cell_tuple_for_row(row)
                    base_cell_tuple = base_col.get_cell_tuple_for_row(row)
                    if cell_tuple is None and base_cell_tuple is None:
                        continue
                    if cell_tuple is None:
                        vals = [AccountingNone] * len(common_subkpis)
                    else:
                        vals = [cell.val for cell in cell_tuple
                                if cell.subcol.subkpi in common_subkpis]
                    if base_cell_tuple is None:
                        base_vals = [AccountingNone] * len(common_subkpis)
                    else:
                        base_vals = [cell.val for cell in base_cell_tuple
                                     if cell.subcol.subkpi in common_subkpis]
                    comparison_cell_tuple = []
                    for val, base_val, comparison_subcol in \
                            izip(vals,
                                 base_vals,
                                 comparison_col.iter_subcols()):
                        # TODO FIXME average factors
                        delta, delta_r = row.kpi.compare_and_render(
                            self.lang, val, base_val, 1, 1)
                        comparison_cell_tuple.append(KpiMatrixCell(
                            row, comparison_subcol, delta, delta_r, None))
                    comparison_col._set_cell_tuple(row, comparison_cell_tuple)
                self._comparison_cols[pos_period_key].append(comparison_col)

    def iter_rows(self):
        """ Iterate rows in display order.

        yields KpiMatrixRow.
        """
        for kpi_row in self._kpi_rows.values():
            yield kpi_row
            detail_rows = self._detail_rows[kpi_row.kpi].values()
            detail_rows = sorted(detail_rows, key=lambda r: r.description)
            for detail_row in detail_rows:
                yield detail_row

    def iter_cols(self):
        """ Iterate columns in display order.

        yields KpiMatrixCol: one for each period or comparison.
        """
        for period_key, col in self._cols.items():
            yield col
            for comparison_col in self._comparison_cols[period_key]:
                yield comparison_col

    def iter_subcols(self):
        """ Iterate sub columns in display order.

        yields KpiMatrixSubCol: one for each subkpi in each period
        and comparison.
        """
        for col in self.iter_cols():
            for subcol in col.iter_subcols():
                yield subcol

    def _load_account_names(self):
        account_ids = set()
        for detail_rows in self._detail_rows.values():
            account_ids.update(detail_rows.keys())
        account_ids = list(account_ids)
        accounts = self._account_model.search([('id', 'in', account_ids)])
        self._account_names = {a.id: u'{} {}'.format(a.code, a.name)
                               for a in accounts}

    def get_account_name(self, account_id):
        if account_id not in self._account_names:
            self._load_account_names()
        return self._account_names[account_id]

    def as_dict(self):
        header = [{'cols': []}, {'cols': []}]
        for col in self.iter_cols():
            header[0]['cols'].append({
                'description': col.description,
                'comment': col.comment,
                'colspan': col.colspan,
            })
            for subcol in col.iter_subcols():
                header[1]['cols'].append({
                    'description': subcol.description,
                    'comment': subcol.comment,
                    'colspan': 1,
                })

        content = []
        for row in self.iter_rows():
            row_data = {
                'row_id': id(row),
                'parent_row_id': row.parent_row and id(row.parent_row) or None,
                'description': row.description,
                'comment': row.comment,
                'style': row.style and row.style.to_css_style() or None,
                'cols': []
            }
            for cell in row.iter_cells():
                if cell is None:
                    row_data['cols'].append({})
                else:
                    row_data['cols'].append({
                        'val': (cell.val
                                if cell.val is not AccountingNone else None),
                        'val_r': cell.val_rendered,
                        'val_c': cell.val_comment,
                        # TODO FIXME style
                        # TODO FIXME drilldown
                    })
            content.append(row_data)

        return {
            'header': header,
            'content': content,
        }


def _get_selection_label(selection, value):
    for v, l in selection:
        if v == value:
            return l
    return ''


def _utc_midnight(d, tz_name, add_day=0):
{'fraction_non_alphanumeric': 0.08799283822596889, 'fraction_numerical': 0.0015666380693050842, 'mean_word_length': 1.7396280400572246, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7369283', 'n_tokens_mistral': 7765, 'n_tokens_neox': 7423, 'n_words': 1797}