<commit_before>"""
  Copyright 2014 Google Inc. All rights reserved.

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
"""

import webapp2
from os import path, stat
from incremental_fonts_utils import prepare_bundle
import logging
import StringIO
from time import time, sleep

BASE_DIR = path.dirname(__file__)


class IncrementalFonts(webapp2.RequestHandler):

  def get(self):
    self.response.headers['Content-Type'] = 'text/plain'
    self.response.out.write('incremental fonts server under development, try back later')


class GlyphRequest(webapp2.RequestHandler):
  """Service for glyph requests from the server Takes closure of glyphs and returns them as a bundle

  """

  def post(self):
    self.response.headers.add_header('Access-Control-Allow-Origin', '*')

    #HACK 
    #Since GAE is brain dead, it decides using gzip compression only for text 
    #resources and does not allow the application to decide. 
    #Therefore, we set mime_type for binary data as text.
    self.response.headers['Content-Type'] = 'text/richtext'
    f = StringIO.StringIO(prepare_bundle(self.request))
    slow_write(f, self.response.out)


class DoLogging(webapp2.RequestHandler):
  """Dump logging messages into the server logs.

  """
  logger = logging.getLogger()
  logger.setLevel(logging.INFO)

  def post(self):
    self.response.headers.add_header('Access-Control-Allow-Origin', '*')
    logging.info(self.request.body + '')


class WebFont(webapp2.RequestHandler):
  chunk_size = 512

  # 3G (at least according to WebPageTest.org) is 1.6 Mbps / 768 Kbps
  # so download is 200 KBps; 5 mS / KB 
  def get(self, fontname):
    self.response.headers['Content-Type'] = 'application/binary'
    filename = BASE_DIR + '/fonts/' + fontname
    f = open(filename, 'rb')
    slow_write(f, self.response.out)

def slow_write(in_file, out_file):
  chunk_size = 512
  t0 = time()
  chunks_sent = 0
  delay_per_chunk = 0.005 / 1024 * chunk_size
  while True:
    chunk = in_file.read(chunk_size)
    if not chunk:
      break
    chunks_sent += 1
    needed_sleep_time = chunks_sent * delay_per_chunk - (time() - t0)
    if (needed_sleep_time > 0):
      sleep(needed_sleep_time)
    out_file.write(chunk)


app = webapp2.WSGIApplication([
    ('/incremental_fonts/request', GlyphRequest),
    ('/incremental_fonts/logger', DoLogging),
    (r'/incremental_fonts/webfonts/(.*)', WebFont),
    ('/incremental_fonts/.*', IncrementalFonts)
], debug=True)<commit_msg>Make a separate path for the incremental font xfers since they will be compressed.<commit_after>"""
  Copyright 2014 Google Inc. All rights reserved.

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
"""

import webapp2
from os import path, stat
from incremental_fonts_utils import prepare_bundle
import logging
import StringIO
from time import time, sleep

BASE_DIR = path.dirname(__file__)


class IncrementalFonts(webapp2.RequestHandler):

  def get(self):
    self.response.headers['Content-Type'] = 'text/plain'
    self.response.out.write('incremental fonts server under development, try back later')


class GlyphRequest(webapp2.RequestHandler):
  """Service for glyph requests from the server Takes closure of glyphs and returns them as a bundle

  """

  def post(self):
    self.response.headers.add_header('Access-Control-Allow-Origin', '*')

    #HACK 
    #Since GAE is brain dead, it decides using gzip compression only for text 
    #resources and does not allow the application to decide. 
    #Therefore, we set mime_type for binary data as text.
    self.response.headers['Content-Type'] = 'text/richtext'
    f = StringIO.StringIO(prepare_bundle(self.request))
    slow_write(f, self.response.out, 2.5)


class DoLogging(webapp2.RequestHandler):
  """Dump logging messages into the server logs.

  """
  logger = logging.getLogger()
  logger.setLevel(logging.INFO)

  def post(self):
    self.response.headers.add_header('Access-Control-Allow-Origin', '*')
    logging.info(self.request.body + '')


class IncrFont(webapp2.RequestHandler):
  chunk_size = 512

  # 3G (at least according to WebPageTest.org) is 1.6 Mbps / 768 Kbps
  # so download is 200 KBps; 5 mS / KB 
  def get(self, fontname):
    self.response.headers['Content-Type'] = 'text/plain'
    self.response.headers['Content-Type'] = 'text/richtext'
    filename = BASE_DIR + '/fonts/' + fontname
    f = open(filename, 'rb')
    slow_write(f, self.response.out, 2.5)

class WebFont(webapp2.RequestHandler):
  chunk_size = 512

  # 3G (at least according to WebPageTest.org) is 1.6 Mbps / 768 Kbps
  # so download is 200 KBps; 5 mS / KB 
  def get(self, fontname):
    self.response.headers['Content-Type'] = 'application/binary'
    filename = BASE_DIR + '/fonts/' + fontname
    f = open(filename, 'rb')
    slow_write(f, self.response.out, 5)

def slow_write(in_file, out_file, ms_per_k):
  chunk_size = 512
  t0 = time()
  chunks_sent = 0
  delay_per_chunk = (ms_per_k / 1024.0) / 1024 * chunk_size
  while True:
    chunk = in_file.read(chunk_size)
    if not chunk:
      break
    chunks_sent += 1
    needed_sleep_time = chunks_sent * delay_per_chunk - (time() - t0)
    if (needed_sleep_time > 0):
      sleep(needed_sleep_time)
    out_file.write(chunk)


app = webapp2.WSGIApplication([
    ('/incremental_fonts/request', GlyphRequest),
    ('/incremental_fonts/logger', DoLogging),
    (r'/incremental_fonts/webfonts/(.*)', WebFont),
    (r'/incremental_fonts/incrfonts/(.*)', IncrFont),
    ('/incremental_fonts/.*', IncrementalFonts)
], debug=True)
{'fraction_non_alphanumeric': 0.10496300863131935, 'fraction_numerical': 0.016337854500616523, 'mean_word_length': 3.8497757847533634, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4730060', 'n_tokens_mistral': 2150, 'n_tokens_neox': 1971, 'n_words': 616}