<commit_before>#!/usr/bin/env python
# coding=utf-8
#
# Copyright 2011 Friday Film Club. All Rights Reserved.

"""Restore film data from app backup to dev server

To get entities from prod backup:

1. gsutil cp gs://ffcapp.appspot.com/** data/backups/YYYYMMDD/.
2. Run restore_dev_data.py
"""

import os
import sys

APPENGINE_PATH = '/usr/local/google_appengine/'

def fix_appengine_path():
  EXTRA_PATHS = [
    APPENGINE_PATH,
    os.path.join(APPENGINE_PATH, 'lib', 'antlr3'),
    os.path.join(APPENGINE_PATH, 'lib', 'django'),
    os.path.join(APPENGINE_PATH, 'lib', 'fancy_urllib'),
    os.path.join(APPENGINE_PATH, 'lib', 'ipaddr'),
    os.path.join(APPENGINE_PATH, 'lib', 'webapp2'),
    os.path.join(APPENGINE_PATH, 'lib', 'webob-1.1.1'),
    os.path.join(APPENGINE_PATH, 'lib', 'yaml', 'lib'),
    os.path.join(APPENGINE_PATH, 'lib', 'webapp2-2.5.2'),
  ]
  sys.path.extend(EXTRA_PATHS)

fix_appengine_path()

# Add models to path.
sys.path.insert(0, '/Users/adammcgrath/dev/projects/ffc/src/')

from google.appengine.ext import ndb
from google.appengine.ext.remote_api import remote_api_stub
from google.appengine.api.files import records
from google.appengine.datastore import entity_pb


import models

APP_NAME = 'dev~ffcapp'
os.environ['AUTH_DOMAIN'] = 'gmail.com'
os.environ['USER_EMAIL'] = '<EMAIL>'


def auth_func():
  return (os.environ['USER_EMAIL'], '')


def fix_emails():
  # TODO: Batch the datastore writes.
  print 'Fixing emails'
  for u in models.User.query():
    u.email = 'adamjmc<EMAIL>+%<EMAIL>' % u.username
    print u.email
    u.put()
  print 'Fixed emails'


def main():
  # Use 'localhost:8080' for dev server.
  remote_api_stub.ConfigureRemoteDatastore(APP_NAME, '/_ah/remote_api',
      auth_func, servername='localhost:8080')

  year = sys.argv[1]
  base_dir = 'data/backups/%s' % year

  for backup_file in os.listdir(base_dir):
    raw = open('%s/%s' % (base_dir, backup_file), 'r')
    reader = records.RecordsReader(raw)
    entities = []
    for record in reader:
      try:
        entity_proto = entity_pb.EntityProto(contents=record)
      except:
        break

      entity_proto.key().set_app(APP_NAME)
      for p in entity_proto.property_list():
        p.value().referencevalue().set_app(APP_NAME)

      entities.append(ndb.Model._from_pb(entity_proto))

    if len(entities):
      ndb.put_multi(entities)
      print 'restored: %s' % backup_file
  fix_emails()

if __name__ == '__main__':
  main()
<commit_msg>Speed up restore_dev_data by adding cursor to user query<commit_after>#!/usr/bin/env python
# coding=utf-8
#
# Copyright 2011 Friday Film Club. All Rights Reserved.

"""Restore film data from app backup to dev server

To get entities from prod backup:

1. gsutil cp gs://ffcapp.appspot.com/** data/backups/YYYYMMDD/.
2. Run restore_dev_data.py
"""

import os
import sys

APPENGINE_PATH = '/usr/local/google_appengine/'

def fix_appengine_path():
  EXTRA_PATHS = [
    APPENGINE_PATH,
    os.path.join(APPENGINE_PATH, 'lib', 'antlr3'),
    os.path.join(APPENGINE_PATH, 'lib', 'django'),
    os.path.join(APPENGINE_PATH, 'lib', 'fancy_urllib'),
    os.path.join(APPENGINE_PATH, 'lib', 'ipaddr'),
    os.path.join(APPENGINE_PATH, 'lib', 'webapp2'),
    os.path.join(APPENGINE_PATH, 'lib', 'webob-1.1.1'),
    os.path.join(APPENGINE_PATH, 'lib', 'yaml', 'lib'),
    os.path.join(APPENGINE_PATH, 'lib', 'webapp2-2.5.2'),
  ]
  sys.path.extend(EXTRA_PATHS)

fix_appengine_path()

# Add models to path.
sys.path.insert(0, '/Users/adammcgrath/dev/projects/ffc/src/')

from google.appengine.ext import ndb
from google.appengine.ext.remote_api import remote_api_stub
from google.appengine.api.files import records
from google.appengine.datastore import entity_pb


import models

APP_NAME = 'dev~ffcapp'
BATCH_SIZE = 100
os.environ['AUTH_DOMAIN'] = 'gmail.com'
os.environ['USER_EMAIL'] = '<EMAIL>'


def auth_func():
  return (os.environ['USER_EMAIL'], '')


def fix_emails(curs=None):
  print 'Fixing emails'
  users, next_curs, more = models.User.query().fetch_page(BATCH_SIZE,
                                                          start_cursor=curs)
  for u in users:
    u.email = '<EMAIL>+%s<EMAIL>' % u.username
  ndb.put_multi(users)

  if more:
    fix_emails(next_curs)


def main():
  # Use 'localhost:8080' for dev server.
  remote_api_stub.ConfigureRemoteDatastore(APP_NAME, '/_ah/remote_api',
      auth_func, servername='localhost:8080')

  year = sys.argv[1]
  base_dir = 'data/backups/%s' % year

  for backup_file in os.listdir(base_dir):
    raw = open('%s/%s' % (base_dir, backup_file), 'r')
    reader = records.RecordsReader(raw)
    entities = []
    for record in reader:
      try:
        entity_proto = entity_pb.EntityProto(contents=record)
      except:
        break

      entity_proto.key().set_app(APP_NAME)
      for p in entity_proto.property_list():
        p.value().referencevalue().set_app(APP_NAME)

      entities.append(ndb.Model._from_pb(entity_proto))

    if len(entities):
      ndb.put_multi(entities)
      print 'restored: %s' % backup_file
  fix_emails()

if __name__ == '__main__':
  main()

{'fraction_non_alphanumeric': 0.1424373040752351, 'fraction_numerical': 0.010775862068965518, 'mean_word_length': 3.7532588454376166, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2856051', 'n_tokens_mistral': 1929, 'n_tokens_neox': 1879, 'n_words': 376}