<commit_before>#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import binascii
import datetime
import logging
import os
import re

# The app engine headers are located locally, so don't worry about not finding
# them.
# pylint: disable=E0611,F0401
import webapp2
from google.appengine.api import datastore_errors
from google.appengine.api import files
from google.appengine.api import memcache
from google.appengine.api import users
from google.appengine.api import taskqueue
from google.appengine.ext import blobstore
from google.appengine.ext import db
from google.appengine.ext.webapp import blobstore_handlers
# pylint: enable=E0611,F0401

# The maximum number of hash entries that can be queried in a single
# request.
MAX_HASH_DIGESTS_PER_CALL = 1000

# The minimum size, in bytes, a hash entry must be before it gets stored in the
# blobstore, otherwise it is stored as a blob property.
MIN_SIZE_FOR_BLOBSTORE = 20 * 1024

# The number of days a datamodel must go unaccessed for before it is deleted.
DATASTORE_TIME_TO_LIVE_IN_DAYS = 7

# The maximum number of items to delete at a time.
ITEMS_TO_DELETE_ASYNC = 100

# The domains that are allowed to access this application.
VALID_DOMAINS = (
    'chromium.org',
    'google.com',
)

HASH_DIGEST_LENGTH = 20
HASH_HEXDIGEST_LENGTH = 40
# The SHA-1 must be lower case. This simplifies the code, e.g. the keys are
# unambiguous, no need to constantly call .lower().
VALID_SHA1_RE = r'[a-f0-9]{' + str(HASH_HEXDIGEST_LENGTH) + r'}'
VALID_SHA1_RE_COMPILED = re.compile(VALID_SHA1_RE)
VALID_NAMESPACE_RE = r'[a-z0-9A-Z\-]+'


class ContentNamespace(db.Model):
  """Used as an ancestor of HashEntry to create mutiple content-addressed
  "tables".

  There's primarily three tables. The table name is its key name:
  - default:    The default CAD.
  - gzip:       This namespace contains the content in gzip'ed format.
  - temporary*: This family of namespace is a discardable namespace for testing
                purpose only.

  All the tables in the temporary* family should have is_testing==True and the
  others is_testing==False.
  """
  is_testing = db.BooleanProperty()
  creation = db.DateTimeProperty(auto_now=True)


class HashEntry(db.Model):
  """Represents the hash content."""
  # The reference pointing to the hash content, which is stored inside the
  # blobstore. This is only valid if the hash content was at least
  # MIN_SIZE_FOR_BLOBSTORE.
  hash_content_reference = blobstore.BlobReferenceProperty()

  # The hash content. This is only valid if the hash content was smaller than
  # MIN_SIZE_FOR_BLOBSTORE.
  hash_content = db.BlobProperty()

  # The day the content was last accessed. This is used to determine when
  # data is old and should be cleared.
  last_access = db.DateProperty(auto_now_add=True)

  creation = db.DateTimeProperty(auto_now=True)

  # It is an .isolated file.
  is_isolated = db.BooleanProperty(default=False)

  @property
  def is_compressed(self):
    """Is it the raw data or was it modified in any form, e.g. compressed, so
    that the SHA-1 doesn't match.
    """
    self.parent_key().name().endswith(('bzip2', 'gzip'))


class WhitelistedIP(db.Model):
  """Items where the IP address is allowed."""
  # The IP of the machine to whitelist. Can be either IPv4 or IPv6.
  ip = db.StringProperty()

  # Is only for maintenance purpose, not used.
  comment = db.StringProperty()


def GetContentNamespaceKey(namespace):
  """Returns the ContentNamespace key for the namespace value.

  Makes sure the entity exists in the datastore.
  memcache.
  """
  return ContentNamespace.get_or_insert(
      namespace, is_testing=namespace.startswith('temporary')).key()


def GetContentByHash(namespace, hash_key):
  """Returns the HashEntry with the given key, or None if it no HashEntry
  matches."""
  if not VALID_SHA1_RE_COMPILED.match(hash_key):
    logging.error('Given an invalid sha-1 value, %s', hash_key)
    return None

  try:
    namespace_model_key = GetContentNamespaceKey(namespace)
    key = db.Key.from_path('HashEntry', hash_key, parent=namespace_model_key)
    return HashEntry.get(key)
  except (db.BadKeyError, db.KindError):
    pass

  return None


def StoreValueInBlobstore(value):
  """Store the given content in the blobstore, returning the key if it is
  successfully stored, otherwise return None."""
  # TODO(csharp): don't use the experimental API, instead create a blobstore
  # upload url and have the url use that instead.
  try:
    file_name = files.blobstore.create(mime_type='application/octet-stream')
    with files.open(file_name, 'a') as f:
      f.write(value)
    files.finalize(file_name)
    return files.blobstore.get_blob_key(file_name)
  except files.ApiTemporaryUnavailableError:
    logging.warning('An exception while trying to store results in the '
                    'blobstore.')
  return None


def CreateHashEntry(namespace, hash_key):
  """Generates a new hash entry from the request if one doesn't exist.

  Returns None if there is a problem generating the entry or if an entry already
  exists with the given key.
  """
  namespace_model_key = GetContentNamespaceKey(namespace)
  key = db.Key.from_path('HashEntry', hash_key, parent=namespace_model_key)

  if HashEntry.all(keys_only=True).filter('__key__ =', key).get():
    return None
  return HashEntry(key=key)


class ACLRequestHandler(webapp2.RequestHandler):
  """Adds ACL to the request handler to ensure only valid users can use
  the handlers."""
  def dispatch(self):
    """Ensures that only users from valid domains can continue, and that users
    from invalid domains receive an error message."""
    user = users.get_current_user()
    if not user:
      # Verify if its IP is whitelisted.
      query = WhitelistedIP.gql('WHERE ip = :1', self.request.remote_addr)
      if not query.count():
        self.abort(401, detail='Please login first.')
    else:
      domain = user.email().partition('@')[2]
      if domain not in VALID_DOMAINS:
        logging.warning('Disallowing %s, invalid domain' % user.email())
        self.abort(403, detail='Invalid domain, %s' % domain)

    return webapp2.RequestHandler.dispatch(self)


def payload_to_hashes(request):
  """Loads the hash provided on the payload concatenated in binary form."""
  hash_digests = request.request.body
  if len(hash_digests) % HASH_DIGEST_LENGTH:
    msg = (
        'Hash digests must all be of length %d, had %d bytes total, last '
        'digest was of length %d' % (
              HASH_DIGEST_LENGTH,
              len(hash_digests),
              len(hash_digests) % HASH_DIGEST_LENGTH))
    logging.error(msg)<commit_msg>Add log entry when an IP is not whitelisted to make searching easier.<commit_after>#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import binascii
import datetime
import logging
import os
import re

# The app engine headers are located locally, so don't worry about not finding
# them.
# pylint: disable=E0611,F0401
import webapp2
from google.appengine.api import datastore_errors
from google.appengine.api import files
from google.appengine.api import memcache
from google.appengine.api import users
from google.appengine.api import taskqueue
from google.appengine.ext import blobstore
from google.appengine.ext import db
from google.appengine.ext.webapp import blobstore_handlers
# pylint: enable=E0611,F0401

# The maximum number of hash entries that can be queried in a single
# request.
MAX_HASH_DIGESTS_PER_CALL = 1000

# The minimum size, in bytes, a hash entry must be before it gets stored in the
# blobstore, otherwise it is stored as a blob property.
MIN_SIZE_FOR_BLOBSTORE = 20 * 1024

# The number of days a datamodel must go unaccessed for before it is deleted.
DATASTORE_TIME_TO_LIVE_IN_DAYS = 7

# The maximum number of items to delete at a time.
ITEMS_TO_DELETE_ASYNC = 100

# The domains that are allowed to access this application.
VALID_DOMAINS = (
    'chromium.org',
    'google.com',
)

HASH_DIGEST_LENGTH = 20
HASH_HEXDIGEST_LENGTH = 40
# The SHA-1 must be lower case. This simplifies the code, e.g. the keys are
# unambiguous, no need to constantly call .lower().
VALID_SHA1_RE = r'[a-f0-9]{' + str(HASH_HEXDIGEST_LENGTH) + r'}'
VALID_SHA1_RE_COMPILED = re.compile(VALID_SHA1_RE)
VALID_NAMESPACE_RE = r'[a-z0-9A-Z\-]+'


class ContentNamespace(db.Model):
  """Used as an ancestor of HashEntry to create mutiple content-addressed
  "tables".

  There's primarily three tables. The table name is its key name:
  - default:    The default CAD.
  - gzip:       This namespace contains the content in gzip'ed format.
  - temporary*: This family of namespace is a discardable namespace for testing
                purpose only.

  All the tables in the temporary* family should have is_testing==True and the
  others is_testing==False.
  """
  is_testing = db.BooleanProperty()
  creation = db.DateTimeProperty(auto_now=True)


class HashEntry(db.Model):
  """Represents the hash content."""
  # The reference pointing to the hash content, which is stored inside the
  # blobstore. This is only valid if the hash content was at least
  # MIN_SIZE_FOR_BLOBSTORE.
  hash_content_reference = blobstore.BlobReferenceProperty()

  # The hash content. This is only valid if the hash content was smaller than
  # MIN_SIZE_FOR_BLOBSTORE.
  hash_content = db.BlobProperty()

  # The day the content was last accessed. This is used to determine when
  # data is old and should be cleared.
  last_access = db.DateProperty(auto_now_add=True)

  creation = db.DateTimeProperty(auto_now=True)

  # It is an .isolated file.
  is_isolated = db.BooleanProperty(default=False)

  @property
  def is_compressed(self):
    """Is it the raw data or was it modified in any form, e.g. compressed, so
    that the SHA-1 doesn't match.
    """
    self.parent_key().name().endswith(('bzip2', 'gzip'))


class WhitelistedIP(db.Model):
  """Items where the IP address is allowed."""
  # The IP of the machine to whitelist. Can be either IPv4 or IPv6.
  ip = db.StringProperty()

  # Is only for maintenance purpose, not used.
  comment = db.StringProperty()


def GetContentNamespaceKey(namespace):
  """Returns the ContentNamespace key for the namespace value.

  Makes sure the entity exists in the datastore.
  memcache.
  """
  return ContentNamespace.get_or_insert(
      namespace, is_testing=namespace.startswith('temporary')).key()


def GetContentByHash(namespace, hash_key):
  """Returns the HashEntry with the given key, or None if it no HashEntry
  matches."""
  if not VALID_SHA1_RE_COMPILED.match(hash_key):
    logging.error('Given an invalid sha-1 value, %s', hash_key)
    return None

  try:
    namespace_model_key = GetContentNamespaceKey(namespace)
    key = db.Key.from_path('HashEntry', hash_key, parent=namespace_model_key)
    return HashEntry.get(key)
  except (db.BadKeyError, db.KindError):
    pass

  return None


def StoreValueInBlobstore(value):
  """Store the given content in the blobstore, returning the key if it is
  successfully stored, otherwise return None."""
  # TODO(csharp): don't use the experimental API, instead create a blobstore
  # upload url and have the url use that instead.
  try:
    file_name = files.blobstore.create(mime_type='application/octet-stream')
    with files.open(file_name, 'a') as f:
      f.write(value)
    files.finalize(file_name)
    return files.blobstore.get_blob_key(file_name)
  except files.ApiTemporaryUnavailableError:
    logging.warning('An exception while trying to store results in the '
                    'blobstore.')
  return None


def CreateHashEntry(namespace, hash_key):
  """Generates a new hash entry from the request if one doesn't exist.

  Returns None if there is a problem generating the entry or if an entry already
  exists with the given key.
  """
  namespace_model_key = GetContentNamespaceKey(namespace)
  key = db.Key.from_path('HashEntry', hash_key, parent=namespace_model_key)

  if HashEntry.all(keys_only=True).filter('__key__ =', key).get():
    return None
  return HashEntry(key=key)


class ACLRequestHandler(webapp2.RequestHandler):
  """Adds ACL to the request handler to ensure only valid users can use
  the handlers."""
  def dispatch(self):
    """Ensures that only users from valid domains can continue, and that users
    from invalid domains receive an error message."""
    user = users.get_current_user()
    if not user:
      # Verify if its IP is whitelisted.
      query = WhitelistedIP.gql('WHERE ip = :1', self.request.remote_addr)
      if not query.count():
        logging.warning('Blocking IP %s', self.request.remote_addr)
        self.abort(401, detail='Please login first.')
    else:
      domain = user.email().partition('@')[2]
      if domain not in VALID_DOMAINS:
        logging.warning('Disallowing %s, invalid domain' % user.email())
        self.abort(403, detail='Invalid domain, %s' % domain)

    return webapp2.RequestHandler.dispatch(self)


def payload_to_hashes(request):
  """Loads the hash provided on the payload concatenated in binary form."""
  hash_digests = request.request.body
  if len(hash_digests) % HASH_DIGEST_LENGTH:
    msg = (
        'Hash digests must all be of length %d, had %d bytes total, last '
        'digest was of length %d' % (
              HASH_DIGEST_LENGTH,
              len(hash_digests),
              len(hash_digests) % HASH_DIGEST_LENGTH))
    logging.error(msg)
{'fraction_non_alphanumeric': 0.08595694830868356, 'fraction_numerical': 0.00922536242495241, 'mean_word_length': 3.876472688325598, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4339349', 'n_tokens_mistral': 4271, 'n_tokens_neox': 4087, 'n_words': 1453}