<commit_before>#!/usr/bin/env python
#
# Copyright 2007 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import csv
import json

import webapp2
from google.appengine.api import app_identity
from google.appengine.ext import webapp
from google.appengine.ext.webapp.util import run_wsgi_app

from apiclient.discovery import build
import httplib2
from oauth2client import appengine


SCOPE = 'https://www.googleapis.com/auth/bigquery'
PROJECT_ID = app_identity.get_application_id()

credentials = appengine.AppAssertionCredentials(scope=SCOPE)
http = credentials.authorize(httplib2.Http())
service = build('bigquery', 'v2', http=http)


class MainHandler(webapp2.RequestHandler):
  def get(self):
    format = self.request.get('_format', 'csv_text')
    self.response.headers.add_header('Access-Control-Allow-Origin', '*')
    self.response.headers['Content-Type'] = {'csv': 'text/csv'}.get(format, 'text/plain')
  
    query = QUERIES['women_by_country'] % PERIODS['2014/04/12 14:00 UTC']
    query_results = runSyncQuery(service, PROJECT_ID, query)
    if format.startswith('csv'):
      writer = csv.writer(self.response.out)
      for row in query_results:
        writer.writerow(row)
    if format == 'json':
      result = {'query': query}
      result['data'] = query_results
      self.response.out.write(json.dumps(result))


class GetConfigHandler(webapp2.RequestHandler):
  def get(self):
    self.response.headers.add_header('Access-Control-Allow-Origin', '*')
    result = {
      'queries': sorted(QUERIES.keys()),
      'periods': sorted(PERIODS.keys()),
    }
    self.response.out.write(json.dumps(result))
    


app = webapp2.WSGIApplication([
    ('/', MainHandler),
    ('/get_config', GetConfigHandler),
], debug=True)


def runSyncQuery (service, projectId, query, timeout=0):
    jobCollection = service.jobs()
    queryData = {'query':query, 'timeoutMs':timeout}<commit_msg>parameters<commit_after>#!/usr/bin/env python
#
# Copyright 2007 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import csv
import json

import webapp2
from google.appengine.api import app_identity
from google.appengine.ext import webapp
from google.appengine.ext.webapp.util import run_wsgi_app

from apiclient.discovery import build
import httplib2
from oauth2client import appengine


SCOPE = 'https://www.googleapis.com/auth/bigquery'
PROJECT_ID = app_identity.get_application_id()

credentials = appengine.AppAssertionCredentials(scope=SCOPE)
http = credentials.authorize(httplib2.Http())
service = build('bigquery', 'v2', http=http)


class MainHandler(webapp2.RequestHandler):
  def get(self):
    output_format = self.request.get('_format', 'csv_text')
    query_key = self.request.get('query', QUERIES.keys()[0])
    period_key = self.request.get('period', PERIODS.keys()[0])

    self.response.headers.add_header('Access-Control-Allow-Origin', '*')
    self.response.headers['Content-Type'] = {'csv': 'text/csv'}.get(format, 'text/plain')
  
    query = QUERIES[query_key] % PERIODS[period_key]
    query_results = runSyncQuery(service, PROJECT_ID, query)
    if output_format.startswith('csv'):
      writer = csv.writer(self.response.out)
      for row in query_results:
        writer.writerow(row)
    if output_format == 'json':
      result = {'query': query}
      result['data'] = query_results
      self.response.out.write(json.dumps(result))


class GetConfigHandler(webapp2.RequestHandler):
  def get(self):
    self.response.headers.add_header('Access-Control-Allow-Origin', '*')
    result = {
      'queries': sorted(QUERIES.keys()),
      'periods': sorted(PERIODS.keys()),
    }
    self.response.out.write(json.dumps(result))
    


app = webapp2.WSGIApplication([
    ('/', MainHandler),
    ('/get_config', GetConfigHandler),
], debug=True)


def runSyncQuery (service, projectId, query, timeout=0):
    jobCollection = service.jobs()
    queryData = {'query':query, 'timeoutMs':timeout}
{'fraction_non_alphanumeric': 0.12459481361426256, 'fraction_numerical': 0.009724473257698542, 'mean_word_length': 4.413377192982456, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2023439', 'n_tokens_mistral': 1561, 'n_tokens_neox': 1504, 'n_words': 385}