<commit_before># -*- coding: utf-8 -*-
__docformat__ = "restructuredtext en"

__copyright__ = """
Copyright (C) 2012-2013 Hendrikx-ITC B.V.

Distributed under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3, or (at your option) any later
version.  The full license is in the file COPYING, distributed as part of
this software.
"""
from datetime import datetime
import logging
from functools import partial
from operator import attrgetter, methodcaller
import StringIO
from itertools import chain

import pytz
import psycopg2

from minerva.util import k, first, compose, no_op
from minerva.db.error import NoCopyInProgress, NoSuchTable, \
    NoSuchColumnError, UniqueViolation, DataTypeMismatch, DuplicateTable
from minerva.db.query import Table, Column, Eq, column_exists, ands
from minerva.directory import DataSource, EntityType, Entity
from minerva.directory.helpers_v4 import dns_to_entity_ids
from minerva.storage.generic import datatype, format_value, escape_value
from minerva.db.error import translate_postgresql_exception, \
    translate_postgresql_exceptions
from minerva.db.dbtransaction import DbTransaction, DbAction, \
    insert_before, replace, drop_action
from minerva.storage.trend.tables import DATA_TABLE_POSTFIXES
from minerva.storage.trend import schema
from minerva.storage.trend.datapackage import DataPackage
from minerva.storage.trend.granularity import create_granularity, \
    ensure_granularity
from minerva.storage.trend.tables import create_column
from minerva.storage.trend.partition import Partition
from minerva.storage.trend.partitioning import Partitioning
from minerva.storage.trend.trend import Trend, TrendDescriptor

LARGE_BATCH_THRESHOLD = 10


class TrendStoreDescriptor(object):
    def __init__(
            self, datasource, entitytype, granularity, trend_descriptors,
            partition_size):
        self.datasource = datasource
        self.entitytype = entitytype
        self.granularity = granularity
        self.trend_descriptors = trend_descriptors
        self.partition_size = partition_size


class TrendStore(object):
    """
    All data belonging to a specific datasource, entitytype and granularity.
    """
    def __init__(
            self, id, datasource, entitytype, granularity, partition_size,
            type, trends):
        self.id = id
        self.datasource = datasource
        self.entitytype = entitytype
        self.granularity = granularity
        self.partition_size = partition_size
        self.type = type
        self.partitioning = Partitioning(partition_size)
        self.trends = trends

    def __str__(self):
        return self.make_table_basename()

    def make_table_basename(self):
        granularity_name = DATA_TABLE_POSTFIXES.get(
            str(self.granularity), str(self.granularity)
        )

        return "{}_{}_{}".format(
            self.datasource.name, self.entitytype.name, granularity_name
        )

    def make_table_name(self, timestamp):
        table_basename = self.make_table_basename()

        if self.type == "view":
            return table_basename
        else:
            index = self.partitioning.index(timestamp)

            return "{}_{}".format(table_basename, index)

    def base_table(self):
        return Table("trend", self.make_table_basename())

    def partition(self, timestamp):
        if self.type == "view":
            index = None
            start = None
            end = None
        else:
            index = self.partitioning.index(timestamp)
            start, end = self.partitioning.index_to_interval(index)

        name = self.make_table_name(timestamp)

        return Partition(index, name, self, start, end)

    def index_to_interval(self, partition_index):
        return self.partitioning.index_to_interval(partition_index)

    def check_trends_exist(self, trend_descriptors):
        def f(cursor):
            base_table = self.base_table()

            for trend_descriptor in trend_descriptors:
                if not column_exists(cursor, base_table, trend_descriptor.name):
                    create_column(
                        cursor, base_table, trend_descriptor.name,
                        trend_descriptor.data_type<commit_msg>fix partition creation from trendstore<commit_after># -*- coding: utf-8 -*-
__docformat__ = "restructuredtext en"

__copyright__ = """
Copyright (C) 2012-2013 Hendrikx-ITC B.V.

Distributed under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3, or (at your option) any later
version.  The full license is in the file COPYING, distributed as part of
this software.
"""
from datetime import datetime
import logging
from functools import partial
from operator import attrgetter, methodcaller
import StringIO
from itertools import chain

import pytz
import psycopg2

from minerva.util import k, first, compose, no_op
from minerva.db.error import NoCopyInProgress, NoSuchTable, \
    NoSuchColumnError, UniqueViolation, DataTypeMismatch, DuplicateTable
from minerva.db.query import Table, Column, Eq, column_exists, ands
from minerva.directory import DataSource, EntityType, Entity
from minerva.directory.helpers_v4 import dns_to_entity_ids
from minerva.storage.generic import datatype, format_value, escape_value
from minerva.db.error import translate_postgresql_exception, \
    translate_postgresql_exceptions
from minerva.db.dbtransaction import DbTransaction, DbAction, \
    insert_before, replace, drop_action
from minerva.storage.trend.tables import DATA_TABLE_POSTFIXES
from minerva.storage.trend import schema
from minerva.storage.trend.datapackage import DataPackage
from minerva.storage.trend.granularity import create_granularity, \
    ensure_granularity
from minerva.storage.trend.tables import create_column
from minerva.storage.trend.partition import Partition
from minerva.storage.trend.partitioning import Partitioning
from minerva.storage.trend.trend import Trend, TrendDescriptor

LARGE_BATCH_THRESHOLD = 10


class TrendStoreDescriptor(object):
    def __init__(
            self, datasource, entitytype, granularity, trend_descriptors,
            partition_size):
        self.datasource = datasource
        self.entitytype = entitytype
        self.granularity = granularity
        self.trend_descriptors = trend_descriptors
        self.partition_size = partition_size


class TrendStore(object):
    """
    All data belonging to a specific datasource, entitytype and granularity.
    """
    def __init__(
            self, id, datasource, entitytype, granularity, partition_size,
            type, trends):
        self.id = id
        self.datasource = datasource
        self.entitytype = entitytype
        self.granularity = granularity
        self.partition_size = partition_size
        self.type = type
        self.partitioning = Partitioning(partition_size)
        self.trends = trends

    def __str__(self):
        return self.make_table_basename()

    def make_table_basename(self):
        granularity_name = DATA_TABLE_POSTFIXES.get(
            str(self.granularity), str(self.granularity)
        )

        return "{}_{}_{}".format(
            self.datasource.name, self.entitytype.name, granularity_name
        )

    def make_table_name(self, timestamp):
        table_basename = self.make_table_basename()

        if self.type == "view":
            return table_basename
        else:
            index = self.partitioning.index(timestamp)

            return "{}_{}".format(table_basename, index)

    def base_table(self):
        return Table("trend", self.make_table_basename())

    def partition(self, timestamp):
        return Partition(self.partitioning.index(timestamp), self)

    def index_to_interval(self, partition_index):
        return self.partitioning.index_to_interval(partition_index)

    def check_trends_exist(self, trend_descriptors):
        def f(cursor):
            base_table = self.base_table()

            for trend_descriptor in trend_descriptors:
                if not column_exists(cursor, base_table, trend_descriptor.name):
                    create_column(
                        cursor, base_table, trend_descriptor.name,
                        trend_descriptor.data_type
{'fraction_non_alphanumeric': 0.07074960895199134, 'fraction_numerical': 0.0033690289977138734, 'mean_word_length': 3.2581967213114753, 'pattern_counts': {'":': 3, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1675565', 'n_tokens_mistral': 2416, 'n_tokens_neox': 2319, 'n_words': 609}