# Path: performanceplatform/collector/piwik/base.py
# class BaseFetcher(object):
#     def __init__(self, credentials, query):
#         self.token_auth = credentials['token_auth']
#         self.url = credentials['url']
#         self.api_method = query['api_method']
#         self.site_id = query['site_id']
#         self.query_format = 'json'
# 
#     def _request_params(self):
#         return {
#             'method': self.api_method,
#             'idSite': self.site_id,
#             'format': self.query_format,
#             'token_auth': self.token_auth
#         }
# 
#     def _make_request(self):
#         return requests_with_backoff.get(
#             url="{url}?module=API".format(url=self.url),
#             params=self._request_params()
#         )
# 
#     def fetch(self):
#         response = self._make_request()
#         response.raise_for_status()
#         return response.json()
# 
# Path: performanceplatform/utils/data_pusher.py
# class Pusher(object):
#     def __init__(self, target_data_set_config, options):
#         self.data_set_client = DataSet.from_config(target_data_set_config)
#         self.chunk_size = options.get('chunk-size', 100)
#         self.empty_data_set = options.get('empty-data-set', False)
# 
#     def push(self, data):
#         if data:
#             if self.empty_data_set:
#                 self.data_set_client.empty_data_set()
#             self.data_set_client.post(
#                 data, chunk_size=self.chunk_size)
#         else:
#             logging.info("Doing nothing - no data to push")
# 
# Path: performanceplatform/utils/data_parser.py
# class DataParser(object):
#     def __init__(self, data, options, data_type):
#         # it would be nice to have some json schemas or something to validate
#         # for now here are some docs
#         """
#         data: Can be any array of dicts
#         options: {
#           mappings: "a dict: converts keys matching key of the mappings dict
#               to the value in the mapping dict",
#           additionalFields: "a dict:
#               key value pairs will be added into the returned data",
#           idMapping: "a list of keys or single key string: for each key
#               the corresponding values will be concatenated in order to create
#               a unique _humanId field on the returned data and base64
#               encoded in order to create an _id field. If no idMapping
#               are provided then the item start_date and any avaiable
#               'dimensions' will be used instead",
#           dataType: "a value to be set on a dataType attribute.
#               Overrides the data_type argument"
#           plugins: "a list of string respresentations of python classes being
#               instantiated. e.g. ComputeIdFrom('_timestamp', '_timespan').
#               To be run on the passed in data to mutate it.
#               See performanceplatform.collector.ga.plugins for more details"
#         }
#         data_type: "a string - the data_type to be set as a data_type
#             attribute on the returned data unless it is overridden in options"
#         """
#         self.data = list(data)
#         if "plugins" in options:
#             self.plugins = options["plugins"]
#         else:
#             self.plugins = None
#         self.data_type = options.get('dataType', data_type)
#         self.mappings = options.get("mappings", {})
#         self.idMapping = options.get("idMapping", None)
#         self.additionalFields = options.get('additionalFields', {})
# 
#     def get_data(self, special_fields=None):
#         """
#         special_fields: "a dict of data specific to collector type
#             that should be added to the data returned by the parser.
#             This will also be and operated on by idMapping, mappings and
#             plugins"
# 
#         This method loops through the data provided to the instance.
#         For each item it will return a dict of the format:
#         {
#             "_timestamp": "the item start_date",
#             "dataType": "self.data_type for the instance",
#             ...
#             "any additional fields": "from self.additionalFields",
#             "any special_fields": "from special fields argument",
#             "any item dimensions": "from item.dimensions",
#             ...
#             mappings changing keys in this dict from self.mappings
#                are then applied on the above
#             ...
#             "_humanId": "derived from either the values corresponding to
#                idMapping concatenated or the data_type, item.start_date,
#                timeSpan and item.dimensions values if any concatenated"
#             "_id": "derived from either the values corresponding to
#                idMapping concatenated or the data_type, item.start_date,
#                timeSpan and item.dimensions values if any concatenated and
#                then base64 encoded"
#         }
#         """
#         docs = build_document_set(self.data, self.data_type, self.mappings,
#                                   special_fields,
#                                   self.idMapping,
#                                   additionalFields=self.additionalFields)
# 
#         if self.plugins:
#             docs = run_plugins(self.plugins, list(docs))
# 
#         return docs


import re
import pytz
from datetime import datetime
from performanceplatform.collector.piwik.base import BaseFetcher
from performanceplatform.utils.data_pusher import Pusher
from performanceplatform.utils.data_parser import DataParser
        if self.api_method_arguments:
            params.update(self.api_method_arguments)
        return params


class Parser():
    def __init__(self, query, options, data_type):
        self.options = options
        self.data_type = data_type
        self.period = FREQUENCY_TO_PERIOD_MAPPING[
            query.get('frequency', 'weekly')]

    def parse(self, data):
        base_items = []
        special_fields = []
        for date_key, data_points in data.items():
            if type(data_points) == dict:
                data_points = [data_points]
            data = self._parse_item(date_key, data_points)
            base_items += data[0]
            special_fields += data[1]
        return DataParser(
            base_items, self.options, self.data_type
        ).get_data(special_fields)

    def _parse_item(self, date_key, data_points):
        base_items = []
        special_fields = []
        for data_point in data_points:
            start_date = Parser.to_datetime(date_key)

# Path: performanceplatform/collector/piwik/base.py
# class BaseFetcher(object):
#     def __init__(self, credentials, query):
#         self.token_auth = credentials['token_auth']
#         self.url = credentials['url']
#         self.api_method = query['api_method']
#         self.site_id = query['site_id']
#         self.query_format = 'json'
# 
#     def _request_params(self):
#         return {
#             'method': self.api_method,
#             'idSite': self.site_id,
#             'format': self.query_format,
#             'token_auth': self.token_auth
#         }
# 
#     def _make_request(self):
#         return requests_with_backoff.get(
#             url="{url}?module=API".format(url=self.url),
#             params=self._request_params()
#         )
# 
#     def fetch(self):
#         response = self._make_request()
#         response.raise_for_status()
#         return response.json()
# 
# Path: performanceplatform/utils/data_pusher.py
# class Pusher(object):
#     def __init__(self, target_data_set_config, options):
#         self.data_set_client = DataSet.from_config(target_data_set_config)
#         self.chunk_size = options.get('chunk-size', 100)
#         self.empty_data_set = options.get('empty-data-set', False)
# 
#     def push(self, data):
#         if data:
#             if self.empty_data_set:
#                 self.data_set_client.empty_data_set()
#             self.data_set_client.post(
#                 data, chunk_size=self.chunk_size)
#         else:
#             logging.info("Doing nothing - no data to push")
# 
# Path: performanceplatform/utils/data_parser.py
# class DataParser(object):
#     def __init__(self, data, options, data_type):
#         # it would be nice to have some json schemas or something to validate
#         # for now here are some docs
#         """
#         data: Can be any array of dicts
#         options: {
#           mappings: "a dict: converts keys matching key of the mappings dict
#               to the value in the mapping dict",
#           additionalFields: "a dict:
#               key value pairs will be added into the returned data",
#           idMapping: "a list of keys or single key string: for each key
#               the corresponding values will be concatenated in order to create
#               a unique _humanId field on the returned data and base64
#               encoded in order to create an _id field. If no idMapping
#               are provided then the item start_date and any avaiable
#               'dimensions' will be used instead",
#           dataType: "a value to be set on a dataType attribute.
#               Overrides the data_type argument"
#           plugins: "a list of string respresentations of python classes being
#               instantiated. e.g. ComputeIdFrom('_timestamp', '_timespan').
#               To be run on the passed in data to mutate it.
#               See performanceplatform.collector.ga.plugins for more details"
#         }
#         data_type: "a string - the data_type to be set as a data_type
#             attribute on the returned data unless it is overridden in options"
#         """
#         self.data = list(data)
#         if "plugins" in options:
#             self.plugins = options["plugins"]
#         else:
#             self.plugins = None
#         self.data_type = options.get('dataType', data_type)
#         self.mappings = options.get("mappings", {})
#         self.idMapping = options.get("idMapping", None)
#         self.additionalFields = options.get('additionalFields', {})
# 
#     def get_data(self, special_fields=None):
#         """
#         special_fields: "a dict of data specific to collector type
#             that should be added to the data returned by the parser.
#             This will also be and operated on by idMapping, mappings and
#             plugins"
# 
#         This method loops through the data provided to the instance.
#         For each item it will return a dict of the format:
#         {
#             "_timestamp": "the item start_date",
#             "dataType": "self.data_type for the instance",
#             ...
#             "any additional fields": "from self.additionalFields",
#             "any special_fields": "from special fields argument",
#             "any item dimensions": "from item.dimensions",
#             ...
#             mappings changing keys in this dict from self.mappings
#                are then applied on the above
#             ...
#             "_humanId": "derived from either the values corresponding to
#                idMapping concatenated or the data_type, item.start_date,
#                timeSpan and item.dimensions values if any concatenated"
#             "_id": "derived from either the values corresponding to
#                idMapping concatenated or the data_type, item.start_date,
#                timeSpan and item.dimensions values if any concatenated and
#                then base64 encoded"
#         }
#         """
#         docs = build_document_set(self.data, self.data_type, self.mappings,
#                                   special_fields,
#                                   self.idMapping,
#                                   additionalFields=self.additionalFields)
# 
#         if self.plugins:
#             docs = run_plugins(self.plugins, list(docs))
# 
#         return docs


# Path: performanceplatform/collector/piwik/core.py
import re
import pytz
from datetime import datetime
from performanceplatform.collector.piwik.base import BaseFetcher
from performanceplatform.utils.data_pusher import Pusher
from performanceplatform.utils.data_parser import DataParser

        if self.api_method_arguments:
            params.update(self.api_method_arguments)
        return params


class Parser():
    def __init__(self, query, options, data_type):
        self.options = options
        self.data_type = data_type
        self.period = FREQUENCY_TO_PERIOD_MAPPING[
            query.get('frequency', 'weekly')]

    def parse(self, data):
        base_items = []
        special_fields = []
        for date_key, data_points in data.items():
            if type(data_points) == dict:
                data_points = [data_points]
            data = self._parse_item(date_key, data_points)
            base_items += data[0]
            special_fields += data[1]
        return DataParser(
            base_items, self.options, self.data_type
        ).get_data(special_fields)

    def _parse_item(self, date_key, data_points):
        base_items = []
        special_fields = []
        for data_point in data_points:
            start_date = Parser.to_datetime(date_key)

            base_items.append({'start_date': start_date})