<commit_before>from __future__ import division
from __future__ import print_function

# -*- coding: utf-8 -*-
# Copyright 2013 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License")
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

__author__ = '<EMAIL> (<NAME>)'


import datetime
import json
import logging
import os
import sys
import webapp2

# Appengine imports.
from google.appengine.api import files
from google.appengine.api import memcache
from google.appengine.api import urlfetch
from google.appengine.api import users
from google.appengine.ext import blobstore
from google.appengine.ext import db
from google.appengine.ext.webapp import blobstore_handlers

# File imports.
import common
import models
import settings
import uma


# uma.googleplex.com/data/histograms/ids-chrome-histograms.txt
BIGSTORE_BUCKET = '/gs/uma-dashboards/'
BIGSTORE_RESTFUL_URI = 'https://uma-dashboards.storage.googleapis.com/'

CSSPROPERITES_BS_HISTOGRAM_ID = str(0xbfd59b316a6c31f1)
ANIMATIONPROPS_BS_HISTOGRAM_ID = str(0xbee14b73f4fdde73)
FEATURE_OBSERVER_BS_HISTOGRAM_ID = str(0x2e44945129413683)

# For fetching files from the production BigStore during development.
OAUTH2_CREDENTIALS_FILENAME = os.path.join(
    settings.ROOT_DIR, 'scripts', 'oauth2.data')


class YesterdayHandler(blobstore_handlers.BlobstoreDownloadHandler):
  """Loads yesterday's UMA data from BigStore."""

  MODEL_CLASS = {
    CSSPROPERITES_BS_HISTOGRAM_ID: models.StableInstance,
    ANIMATIONPROPS_BS_HISTOGRAM_ID: models.AnimatedProperty,
    FEATURE_OBSERVER_BS_HISTOGRAM_ID: models.FeatureObserver,
  }

  def _SaveData(self, data, yesterday, histogram_id):
    try: 
      model_class = self.MODEL_CLASS[histogram_id]
    except Exception, e:
      logging.error('Invalid CSS property bucket id used: %s' % histogram_id)
      return

    # Response format is "bucket-bucket+1=hits".
    # Example: 10-11=2175995,11-12=56635467,12-13=2432539420
    #values_list = data['kTempHistograms'][CSSPROPERITES_BS_HISTOGRAM_ID]['b'].split(',')
    values_list = data['kTempHistograms'][histogram_id]['b'].split(',')

    #sum_total = int(data['kTempHistograms'][CSSPROPERITES_BS_HISTOGRAM_ID]['s']) # TODO: use this.
    
    # Stores a hit count for each CSS property (properties_dict[bucket] = hits).
    properties_dict = {}

    for val in values_list:
      bucket_range, hits_string = val.split('=') # e.g. "10-11=2175995"

      parts = bucket_range.split('-')

      beginning_range = int(parts[0])
      end_range = int(parts[1])

      # Range > 1 indicates malformed data. Skip it.
      if end_range - beginning_range > 1:
        continue

      # beginning_range is our bucket number; the stable CSSPropertyID.
      properties_dict[beginning_range] = int(hits_string)

    # For CSSPROPERITES_BS_HISTOGRAM_ID, bucket 1 is total pages visited for
    # stank rank histogram. We're guaranteed to have it.
    # For the FEATURE_OBSERVER_BS_HISTOGRAM_ID, the PageVisits bucket_id is 52
    # See uma.py. The actual % is calculated from the count / this number.
    # For ANIMATIONPROPS_BS_HISTOGRAM_ID, we have to calculate the total count.
    if 1 in properties_dict and histogram_id == CSSPROPERITES_BS_HISTOGRAM_ID:
      total_pages = properties_dict.get(1)
    elif (uma.PAGE_VISITS_BUCKET_ID in properties_dict and
          histogram_id == FEATURE_OBSERVER_BS_HISTOGRAM_ID):
      total_pages = properties_dict.get(uma.PAGE_VISITS_BUCKET_ID)

      # Don't include PageVisits results.
      del properties_dict[uma.PAGE_VISITS_BUCKET_ID]
    else:
      total_pages = sum(properties_dict.values())

    for bucket_id, num_hits in properties_dict.items():
      # If the id is not in the map, use 'ERROR' for the name.
      # TODO(ericbidelman): Non-matched bucket ids are likely new properties
      # that have been added and need to be updated in uma.py. Find way to
      # autofix these values with the appropriate property_name later.
      property_map = uma.CSS_PROPERTY_BUCKETS
      if histogram_id == FEATURE_OBSERVER_BS_HISTOGRAM_ID:
        property_map = uma.FEATUREOBSERVER_BUCKETS

      property_name = property_map.get(bucket_id, 'ERROR')

      query = model_class.all()
      query.filter('bucket_id = ', bucket_id)
      query.filter('date =', yesterday)

      # Only add this entity if one doesn't already exist with the same
      # bucket_id and date.
      if query.count() > 0:
        continue

      # TODO(ericbidelman): Calculate a rolling average here
      # This will be done using a GQL query to grab information
      # for the past 6 days.
      # We average those past 6 days with the new day's data
      # and store the result in rolling_percentage

      entity = model_class(
          property_name=property_name,
          bucket_id=bucket_id,
          date=yesterday,
          #hits=num_hits,
          #total_pages=total_pages,
          day_percentage=(num_hits * 1.0 / total_pages)
          #rolling_percentage=
          )
      entity.put()

  def get(self):
    """Loads the data file located at |filename|.

    Args:
      filename: The filename for the data file to be loaded.
    """
    yesterday = datetime.date.today() - datetime.timedelta(1)
    yesterday_formatted = yesterday.strftime("%Y.%m.%d")
<commit_msg>Add logging message for debugging<commit_after>from __future__ import division
from __future__ import print_function

# -*- coding: utf-8 -*-
# Copyright 2013 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License")
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

__author__ = '<EMAIL> (<NAME>)'


import datetime
import json
import logging
import os
import sys
import webapp2

# Appengine imports.
from google.appengine.api import files
from google.appengine.api import memcache
from google.appengine.api import urlfetch
from google.appengine.api import users
from google.appengine.ext import blobstore
from google.appengine.ext import db
from google.appengine.ext.webapp import blobstore_handlers

# File imports.
import common
import models
import settings
import uma


# uma.googleplex.com/data/histograms/ids-chrome-histograms.txt
BIGSTORE_BUCKET = '/gs/uma-dashboards/'
BIGSTORE_RESTFUL_URI = 'https://uma-dashboards.storage.googleapis.com/'

CSSPROPERITES_BS_HISTOGRAM_ID = str(0xbfd59b316a6c31f1)
ANIMATIONPROPS_BS_HISTOGRAM_ID = str(0xbee14b73f4fdde73)
FEATURE_OBSERVER_BS_HISTOGRAM_ID = str(0x2e44945129413683)

# For fetching files from the production BigStore during development.
OAUTH2_CREDENTIALS_FILENAME = os.path.join(
    settings.ROOT_DIR, 'scripts', 'oauth2.data')


class YesterdayHandler(blobstore_handlers.BlobstoreDownloadHandler):
  """Loads yesterday's UMA data from BigStore."""

  MODEL_CLASS = {
    CSSPROPERITES_BS_HISTOGRAM_ID: models.StableInstance,
    ANIMATIONPROPS_BS_HISTOGRAM_ID: models.AnimatedProperty,
    FEATURE_OBSERVER_BS_HISTOGRAM_ID: models.FeatureObserver,
  }

  def _SaveData(self, data, yesterday, histogram_id):
    try: 
      model_class = self.MODEL_CLASS[histogram_id]
    except Exception, e:
      logging.error('Invalid CSS property bucket id used: %s' % histogram_id)
      return

    # Response format is "bucket-bucket+1=hits".
    # Example: 10-11=2175995,11-12=56635467,12-13=2432539420
    #values_list = data['kTempHistograms'][CSSPROPERITES_BS_HISTOGRAM_ID]['b'].split(',')
    values_list = data['kTempHistograms'][histogram_id]['b'].split(',')

    #sum_total = int(data['kTempHistograms'][CSSPROPERITES_BS_HISTOGRAM_ID]['s']) # TODO: use this.
    
    # Stores a hit count for each CSS property (properties_dict[bucket] = hits).
    properties_dict = {}

    for val in values_list:
      bucket_range, hits_string = val.split('=') # e.g. "10-11=2175995"

      parts = bucket_range.split('-')

      beginning_range = int(parts[0])
      end_range = int(parts[1])

      # Range > 1 indicates malformed data. Skip it.
      if end_range - beginning_range > 1:
        continue

      # beginning_range is our bucket number; the stable CSSPropertyID.
      properties_dict[beginning_range] = int(hits_string)

    # For CSSPROPERITES_BS_HISTOGRAM_ID, bucket 1 is total pages visited for
    # stank rank histogram. We're guaranteed to have it.
    # For the FEATURE_OBSERVER_BS_HISTOGRAM_ID, the PageVisits bucket_id is 52
    # See uma.py. The actual % is calculated from the count / this number.
    # For ANIMATIONPROPS_BS_HISTOGRAM_ID, we have to calculate the total count.
    if 1 in properties_dict and histogram_id == CSSPROPERITES_BS_HISTOGRAM_ID:
      total_pages = properties_dict.get(1)
    elif (uma.PAGE_VISITS_BUCKET_ID in properties_dict and
          histogram_id == FEATURE_OBSERVER_BS_HISTOGRAM_ID):
      total_pages = properties_dict.get(uma.PAGE_VISITS_BUCKET_ID)

      # Don't include PageVisits results.
      del properties_dict[uma.PAGE_VISITS_BUCKET_ID]
    else:
      total_pages = sum(properties_dict.values())

    for bucket_id, num_hits in properties_dict.items():
      # If the id is not in the map, use 'ERROR' for the name.
      # TODO(ericbidelman): Non-matched bucket ids are likely new properties
      # that have been added and need to be updated in uma.py. Find way to
      # autofix these values with the appropriate property_name later.
      property_map = uma.CSS_PROPERTY_BUCKETS
      if histogram_id == FEATURE_OBSERVER_BS_HISTOGRAM_ID:
        property_map = uma.FEATUREOBSERVER_BUCKETS

      property_name = property_map.get(bucket_id, 'ERROR')

      query = model_class.all()
      query.filter('bucket_id = ', bucket_id)
      query.filter('date =', yesterday)

      # Only add this entity if one doesn't already exist with the same
      # bucket_id and date.
      if query.count() > 0:
        logging.info('Cron data was already fetched for this date')
        continue

      # TODO(ericbidelman): Calculate a rolling average here
      # This will be done using a GQL query to grab information
      # for the past 6 days.
      # We average those past 6 days with the new day's data
      # and store the result in rolling_percentage

      entity = model_class(
          property_name=property_name,
          bucket_id=bucket_id,
          date=yesterday,
          #hits=num_hits,
          #total_pages=total_pages,
          day_percentage=(num_hits * 1.0 / total_pages)
          #rolling_percentage=
          )
      entity.put()

  def get(self):
    """Loads the data file located at |filename|.

    Args:
      filename: The filename for the data file to be loaded.
    """
    yesterday = datetime.date.today() - datetime.timedelta(1)
    yesterday_formatted = yesterday.strftime("%Y.%m.%d")

{'fraction_non_alphanumeric': 0.08382557331480195, 'fraction_numerical': 0.01911049339819319, 'mean_word_length': 3.6762794476035743, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1473037', 'n_tokens_mistral': 3803, 'n_tokens_neox': 3617, 'n_words': 1014}