<commit_before>#!/usr/bin/env python

import sys
import os
import logging
import flask
import schema
import webpages
from storage import get_db


def create_app():
    app = flask.Flask(__name__)

    webpages.register(app)

    app.config.from_pyfile('default_config.py')
    if 'APP_SETTINGS' in os.environ:
        app.config.from_envvar('APP_SETTINGS')

    return app


def accessdb_mjson(args):
    logging.getLogger('migrations.from_access').setLevel(logging.INFO)

    from migrations.from_access import load_from_sql, verify_data
    kwargs = {'indent': 2} if args.indent else {}
    for doc in verify_data(load_from_sql()):
        flask.json.dump(doc, sys.stdout, **kwargs)
        sys.stdout.write('\n')


def import_mjson(args):
    logging.getLogger('storage').setLevel(logging.INFO)

    def batched(iterator, count=10):
        batch = []
        for value in iterator:
            batch.append(value)
            if len(batch) >= count:
                yield batch
                batch = []
        if batch:
            yield batch

    def read_json_lines(stream):
        for line in stream:
            yield flask.json.loads(line)

    def load_document(data):
        doc = schema.SpaDoc(data)
        assert doc.validate(), '%s does not validate' % data['section1']['code']
        assert doc.value == data, 'failed round-tripping the json data'
        return doc

    db = get_db(create_app())

    for batch in batched(load_document(d) for d in read_json_lines(sys.stdin)):
        db.save_document_batch(batch)
        sys.stdout.write('.')
        sys.stdout.flush()

    print ' done'


def runserver(args):
    app = create_app()

    if args.proxied:
        from revproxy import ReverseProxied
        app.wsgi_app = ReverseProxied(app.wsgi_app)

    if args.cherrypy:
        from cherrypy import wsgiserver
        server = wsgiserver.CherryPyWSGIServer(('localhost', 5000), app)
        try:
            server.start()
        except KeyboardInterrupt:
            server.stop()

    else:
        app.run(debug=True)


def create_argument_parser():
    import argparse
    parser = argparse.ArgumentParser()
    subparsers = parser.add_subparsers()

    parser_runserver = subparsers.add_parser('runserver')
    parser_runserver.set_defaults(func=runserver)
    parser_runserver.add_argument('--cherrypy', action='store_true')
    parser_runserver.add_argument('--proxied', action='store_true')

    parser_accessdb_mjson = subparsers.add_parser('accessdb_mjson')<commit_msg>optionally listen on all interfaces<commit_after>#!/usr/bin/env python

import sys
import os
import logging
import flask
import schema
import webpages
from storage import get_db


def create_app():
    app = flask.Flask(__name__)

    webpages.register(app)

    app.config.from_pyfile('default_config.py')
    if 'APP_SETTINGS' in os.environ:
        app.config.from_envvar('APP_SETTINGS')

    return app


def accessdb_mjson(args):
    logging.getLogger('migrations.from_access').setLevel(logging.INFO)

    from migrations.from_access import load_from_sql, verify_data
    kwargs = {'indent': 2} if args.indent else {}
    for doc in verify_data(load_from_sql()):
        flask.json.dump(doc, sys.stdout, **kwargs)
        sys.stdout.write('\n')


def import_mjson(args):
    logging.getLogger('storage').setLevel(logging.INFO)

    def batched(iterator, count=10):
        batch = []
        for value in iterator:
            batch.append(value)
            if len(batch) >= count:
                yield batch
                batch = []
        if batch:
            yield batch

    def read_json_lines(stream):
        for line in stream:
            yield flask.json.loads(line)

    def load_document(data):
        doc = schema.SpaDoc(data)
        assert doc.validate(), '%s does not validate' % data['section1']['code']
        assert doc.value == data, 'failed round-tripping the json data'
        return doc

    db = get_db(create_app())

    for batch in batched(load_document(d) for d in read_json_lines(sys.stdin)):
        db.save_document_batch(batch)
        sys.stdout.write('.')
        sys.stdout.flush()

    print ' done'


def runserver(args):
    app = create_app()

    host = '0.0.0.0' if args.listen_all else '127.0.0.1'

    if args.proxied:
        from revproxy import ReverseProxied
        app.wsgi_app = ReverseProxied(app.wsgi_app)

    if args.cherrypy:
        from cherrypy import wsgiserver
        server = wsgiserver.CherryPyWSGIServer((host, 5000), app)
        try:
            server.start()
        except KeyboardInterrupt:
            server.stop()

    else:
        app.run(host, debug=True)


def create_argument_parser():
    import argparse
    parser = argparse.ArgumentParser()
    subparsers = parser.add_subparsers()

    parser_runserver = subparsers.add_parser('runserver')
    parser_runserver.set_defaults(func=runserver)
    parser_runserver.add_argument('--cherrypy', action='store_true')
    parser_runserver.add_argument('--proxied', action='store_true')
    parser_runserver.add_argument('--listen-all', action='store_true')

    parser_accessdb_mjson = subparsers.add_parser('accessdb_mjson')
{'fraction_non_alphanumeric': 0.10761205564142194, 'fraction_numerical': 0.005023183925811438, 'mean_word_length': 2.86055182699478, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4760587', 'n_tokens_mistral': 1664, 'n_tokens_neox': 1593, 'n_words': 342}