<commit_before>#!/usr/bin/python
#
# Copyright 2011 Friday Film Club. All Rights Reserved.
"""Handles the Movie Auto Complete UI."""

__author__ = '<EMAIL> (<NAME>)'

import logging
import os
import re

import webapp2
from google.appengine.api import memcache
from google.appengine.ext import blobstore
from google.appengine.ext import ndb
from google.appengine.ext.webapp import blobstore_handlers

from mapreduce import control, mapreduce_pipeline
from mapreduce.model import MapreduceState

import baserequesthandler
import forms
import map_reduce
import models
import settings



class HomePage(baserequesthandler.RequestHandler):
  """Shows the homepage."""

  def get(self):
    return self.render_template('admin/index.html', {})


class AddFilms(baserequesthandler.RequestHandler):
  """Form for uploading films as a CSV file as '{Year}, {Title}'."""

  def get(self):
    upload_url = blobstore.create_upload_url('/admin/addfilmshandler')

    return self.render_template('admin/addfilms.html', {
        'upload_url': upload_url
    })


class AddEditQuestion(baserequesthandler.RequestHandler):
  """Adds a question to the datastore."""

  def get(self, key=None):
    if key:
      question_entity = ndb.Key('Question', key).get()

      form = forms.Question(obj=question_entity)
    else:
      question_entity = None
      form = forms.Question()
    
    return self.render_template('admin/addquestion.html', {
        'form': form,
        'question': question_entity,
    })

  def post(self, key=None):
    form = forms.Question(formdata=self.request.POST)

    if key:
      question_entity = ndb.Key('Question', key).get()
    else:
      question_entity = models.Question()

    logging.info(key)
    logging.info(question_entity)
    logging.info(models.Question())

    if form.validate():
      # TODO(adamjmcgrath): only put once.
      logging.info('1')
      logging.info(question_entity)
      question_entity.put()
      form.populate_obj(question_entity)
      logging.info('2')
      logging.info(question_entity)
      question_entity.put()
      return webapp2.redirect('/admin/questions')
    else:
      return self.render_template('admin/addquestion.html', {
          'form': form,
          'question': question_entity,
          'debug': self.request.get('debug')
      })



class Questions(baserequesthandler.RequestHandler):
  """Adds a question to the datastore."""

  def get(self):
    return self.render_template('admin/questions.html', {
      'questions': models.Question.query(),
    })


class AddFilmsHandler(blobstore_handlers.BlobstoreUploadHandler):
  """Processes the uploaded films and create's a blob to upoad and index."""

  def post(self):
    upload_files = self.get_uploads('file')
    blob_info = upload_files[0]

    batch = 1
    last_add = models.Film.query().order(-models.Film.batch).get()
    if last_add:
      batch += last_add.batch

    logging.info('Uploading film batch: %d.' % batch)

    mapreduce_parameters = {
      'blob_key': str(blob_info.key()),
      'done_callback': '/admin/addfilmsdone',
    }
    
    mapper_parameters = {
      'blob_keys': str(blob_info.key()),
      'batch': batch
    }

    map_reduce_id = control.start_map(
        'Add films to datastore.', # Name
        'map_reduce.add_film_map', # handler_spec
        'mapreduce.input_readers.BlobstoreLineInputReader', # reader_spec
        mapper_parameters, # mapper_parameters
        mapreduce_parameters=mapreduce_parameters)

    return webapp2.redirect('/admin')


class AddFilmsDone(baserequesthandler.RequestHandler):
  """Delete the blob once the films have been added to the datastore."""

  def post(self):
    logging.info('Add Movies Complete.')
    mr_id = self.request.headers['Mapreduce-Id']
    mr_state = MapreduceState.get_by_key_name(mr_id)
    mr_spec = mr_state.mapreduce_spec
    json_spec = mr_spec.to_json()
    json_params = json_spec['params']
    blob_key = json_params['blob_key']
    blob_info = blobstore.BlobInfo.get(blob_key)
    blob_info.delete()
    logging.info('Temp blob deleted for mapreduce: %s' % mr_id)


class IndexFilms(baserequesthandler.RequestHandler):
  """Index's the films."""

  def post(self):
    logging.info('Start indexing films.')
    pipeline = map_reduce.IndexerPipeline()
    pipeline.start()

    return webapp2.redirect('/admin')
<commit_msg>Remove spurious logging<commit_after>#!/usr/bin/python
#
# Copyright 2011 Friday Film Club. All Rights Reserved.
"""Handles the Movie Auto Complete UI."""

__author__ = '<EMAIL> (<NAME>)'

import logging
import os
import re

import webapp2
from google.appengine.api import memcache
from google.appengine.ext import blobstore
from google.appengine.ext import ndb
from google.appengine.ext.webapp import blobstore_handlers

from mapreduce import control, mapreduce_pipeline
from mapreduce.model import MapreduceState

import baserequesthandler
import forms
import map_reduce
import models
import settings



class HomePage(baserequesthandler.RequestHandler):
  """Shows the homepage."""

  def get(self):
    return self.render_template('admin/index.html', {})


class AddFilms(baserequesthandler.RequestHandler):
  """Form for uploading films as a CSV file as '{Year}, {Title}'."""

  def get(self):
    upload_url = blobstore.create_upload_url('/admin/addfilmshandler')

    return self.render_template('admin/addfilms.html', {
        'upload_url': upload_url
    })


class AddEditQuestion(baserequesthandler.RequestHandler):
  """Adds a question to the datastore."""

  def get(self, key=None):
    if key:
      question_entity = ndb.Key('Question', key).get()

      form = forms.Question(obj=question_entity)
    else:
      question_entity = None
      form = forms.Question()
    
    return self.render_template('admin/addquestion.html', {
        'form': form,
        'question': question_entity,
    })

  def post(self, key=None):
    form = forms.Question(formdata=self.request.POST)

    if key:
      question_entity = ndb.Key('Question', key).get()
    else:
      question_entity = models.Question()

    if form.validate():
      # TODO(adamjmcgrath): only put once.
      question_entity.put()
      form.populate_obj(question_entity)
      question_entity.put()
      return webapp2.redirect('/admin/questions')
    else:
      return self.render_template('admin/addquestion.html', {
          'form': form,
          'question': question_entity,
          'debug': self.request.get('debug')
      })



class Questions(baserequesthandler.RequestHandler):
  """Adds a question to the datastore."""

  def get(self):
    return self.render_template('admin/questions.html', {
      'questions': models.Question.query(),
    })


class AddFilmsHandler(blobstore_handlers.BlobstoreUploadHandler):
  """Processes the uploaded films and create's a blob to upoad and index."""

  def post(self):
    upload_files = self.get_uploads('file')
    blob_info = upload_files[0]

    batch = 1
    last_add = models.Film.query().order(-models.Film.batch).get()
    if last_add:
      batch += last_add.batch

    logging.info('Uploading film batch: %d.' % batch)

    mapreduce_parameters = {
      'blob_key': str(blob_info.key()),
      'done_callback': '/admin/addfilmsdone',
    }
    
    mapper_parameters = {
      'blob_keys': str(blob_info.key()),
      'batch': batch
    }

    map_reduce_id = control.start_map(
        'Add films to datastore.', # Name
        'map_reduce.add_film_map', # handler_spec
        'mapreduce.input_readers.BlobstoreLineInputReader', # reader_spec
        mapper_parameters, # mapper_parameters
        mapreduce_parameters=mapreduce_parameters)

    return webapp2.redirect('/admin')


class AddFilmsDone(baserequesthandler.RequestHandler):
  """Delete the blob once the films have been added to the datastore."""

  def post(self):
    logging.info('Add Movies Complete.')
    mr_id = self.request.headers['Mapreduce-Id']
    mr_state = MapreduceState.get_by_key_name(mr_id)
    mr_spec = mr_state.mapreduce_spec
    json_spec = mr_spec.to_json()
    json_params = json_spec['params']
    blob_key = json_params['blob_key']
    blob_info = blobstore.BlobInfo.get(blob_key)
    blob_info.delete()
    logging.info('Temp blob deleted for mapreduce: %s' % mr_id)


class IndexFilms(baserequesthandler.RequestHandler):
  """Index's the films."""

  def post(self):
    logging.info('Start indexing films.')
    pipeline = map_reduce.IndexerPipeline()
    pipeline.start()

    return webapp2.redirect('/admin')

{'fraction_non_alphanumeric': 0.11785420824040381, 'fraction_numerical': 0.002582462730367414, 'mean_word_length': 3.6430517711171664, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6290373', 'n_tokens_mistral': 2699, 'n_tokens_neox': 2546, 'n_words': 573}