<commit_before># -*-coding=utf -*-

"""Google Analytics backend for Cubes

Required packages:

    pyopenssl
    google-api-python-client

"""

from ...errors import *
from ...logging import get_logger
from ...stores import Store
from ...providers import ModelProvider
from ...model import Cube, create_dimension, aggregate_list
from .mapper import ga_id_to_identifier
import pkgutil
import json

from apiclient.errors import HttpError
from apiclient.discovery import build
from oauth2client.client import AccessTokenRefreshError

from collections import OrderedDict

import re

try:
    from oauth2client.client import SignedJwtAssertionCredentials
except ImportError:
    from ...common import MissingPackage
    SignedJwtAssertionCredentials = MissingPackage("oauth2client.crypt",
        "Google Analytics Backend with SignedJwtAssertionCredentials; " +
        "you may need to install pyopenssl and OpenSSL")

try:
    import httplib2
except ImportError:
    from ...common import MissingPackage
    httplib2 = MissingPackage("httplib2", "Google Analytics Backend")


# .js file that contains structure with definition of GA cubes – relation
# between metrics and dimensions that can be used together in a query
# Note: this is kind-of workaround, since GA metadata API does not provide a
# way to get this information
_GA_CUBES_JS = "https://developers.google.com/apps/js/analytics_dm/analytics_dm_js-bundle.js"

GA_TIME_DIM_METADATA = {
    "name": "time",
    "role": "time",
    "levels": [
        { "name": "year", "label": "Year" },
        { "name": "month", "label": "Month", "info": { "aggregation_units": 3 }},
        { "name": "day", "label": "Day", "info": { "aggregation_units": 7 } },
    ],
    "hierarchies": [
        {"name": "ymd", "levels": ["year", "month", "day"]},
    ],
    "default_hierarchy_name": "ymd"
}


class GoogleAnalyticsModelProvider(ModelProvider):
    __identifier__ = "ga"
    def __init__(self, *args, **kwargs):
        super(GoogleAnalyticsModelProvider, self).__init__(*args, **kwargs)

        self.logger = get_logger()
        self.logger.setLevel("DEBUG")

        self.ga_concepts = {};
        self.ga_measures = {};
        self.ga_dimensions = {};
        self.ga_cubes = []

    def requires_store(self):
        return True

    def public_dimensions(self):
        return []

    def initialize_from_store(self):
        self._refresh_metadata()

    def _refresh_metadata(self):
        """Load GA metadata. Group metrics and dimensions by `group`"""
        # TODO: move this request to the store for potential re-authentication
        rq = self.store.service.metadata().columns()
        columns = rq.list(reportType='ga').execute()

        # Note: no Cubes model related logic should be here (such as name
        # mangling)

        self.ga_metrics = OrderedDict()
        self.ga_dimensions = OrderedDict()
        self.ga_concepts = OrderedDict()

        for item in columns["items"]:
            # Get the id from the "outer" dictionary and keep juts the "inner"
            # dictionary
            item_id = item["id"]
            item = item["attributes"]
            item["id"] = item_id

            self.logger.debug("GA concept: %s" % item_id)
            self.ga_concepts[item_id] = item

            if item.get('status') == 'DEPRECATED':
                self.logger.debug("Discarding deprecated item %s" % item_id)
                continue

            if item["type"] == "METRIC":
                self.ga_metrics[item_id] = item<commit_msg>exclude deprecated metrics/dimensions prior to adding to concepts map<commit_after># -*-coding=utf -*-

"""Google Analytics backend for Cubes

Required packages:

    pyopenssl
    google-api-python-client

"""

from ...errors import *
from ...logging import get_logger
from ...stores import Store
from ...providers import ModelProvider
from ...model import Cube, create_dimension, aggregate_list
from .mapper import ga_id_to_identifier
import pkgutil
import json

from apiclient.errors import HttpError
from apiclient.discovery import build
from oauth2client.client import AccessTokenRefreshError

from collections import OrderedDict

import re

try:
    from oauth2client.client import SignedJwtAssertionCredentials
except ImportError:
    from ...common import MissingPackage
    SignedJwtAssertionCredentials = MissingPackage("oauth2client.crypt",
        "Google Analytics Backend with SignedJwtAssertionCredentials; " +
        "you may need to install pyopenssl and OpenSSL")

try:
    import httplib2
except ImportError:
    from ...common import MissingPackage
    httplib2 = MissingPackage("httplib2", "Google Analytics Backend")


# .js file that contains structure with definition of GA cubes – relation
# between metrics and dimensions that can be used together in a query
# Note: this is kind-of workaround, since GA metadata API does not provide a
# way to get this information
_GA_CUBES_JS = "https://developers.google.com/apps/js/analytics_dm/analytics_dm_js-bundle.js"

GA_TIME_DIM_METADATA = {
    "name": "time",
    "role": "time",
    "levels": [
        { "name": "year", "label": "Year" },
        { "name": "month", "label": "Month", "info": { "aggregation_units": 3 }},
        { "name": "day", "label": "Day", "info": { "aggregation_units": 7 } },
    ],
    "hierarchies": [
        {"name": "ymd", "levels": ["year", "month", "day"]},
    ],
    "default_hierarchy_name": "ymd"
}


class GoogleAnalyticsModelProvider(ModelProvider):
    __identifier__ = "ga"
    def __init__(self, *args, **kwargs):
        super(GoogleAnalyticsModelProvider, self).__init__(*args, **kwargs)

        self.logger = get_logger()
        self.logger.setLevel("DEBUG")

        self.ga_concepts = {};
        self.ga_measures = {};
        self.ga_dimensions = {};
        self.ga_cubes = []

    def requires_store(self):
        return True

    def public_dimensions(self):
        return []

    def initialize_from_store(self):
        self._refresh_metadata()

    def _refresh_metadata(self):
        """Load GA metadata. Group metrics and dimensions by `group`"""
        # TODO: move this request to the store for potential re-authentication
        rq = self.store.service.metadata().columns()
        columns = rq.list(reportType='ga').execute()

        # Note: no Cubes model related logic should be here (such as name
        # mangling)

        self.ga_metrics = OrderedDict()
        self.ga_dimensions = OrderedDict()
        self.ga_concepts = OrderedDict()

        for item in columns["items"]:
            # Get the id from the "outer" dictionary and keep juts the "inner"
            # dictionary
            item_id = item["id"]
            item = item["attributes"]
            item["id"] = item_id

            self.logger.debug("GA concept: %s" % item_id)

            if item.get('status') == 'DEPRECATED':
                self.logger.debug("Discarding deprecated item %s" % item_id)
                continue

            self.ga_concepts[item_id] = item

            if item["type"] == "METRIC":
                self.ga_metrics[item_id] = item
{'fraction_non_alphanumeric': 0.11270578303081469, 'fraction_numerical': 0.0022513015336991698, 'mean_word_length': 2.946696279844531, 'pattern_counts': {'":': 36, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6243458', 'n_tokens_mistral': 2115, 'n_tokens_neox': 2025, 'n_words': 587}