<commit_before>#!/usr/bin/env python
import click
import logging
import os
import re
import sys


@click.group()
def cli():
    pass


@cli.command()
@click.option('--open', 'open_server', is_flag=True,
              help='Open the server for communication from outside', default=False)
@click.option('--debug-js', is_flag=True, help="Don't minify the JavaScript files")
def testserver(open_server, debug_js):
    from anyway import app
    from anyway.parsers import united
    from apscheduler.scheduler import Scheduler

    sched = Scheduler()

    @sched.interval_schedule(hours=12)
    def scheduled_import(): # pylint: disable=unused-variable
        united.main()
    sched.start()

    logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(message)s')

    if debug_js:
        app.config['ASSETS_DEBUG'] = True

    default_host = '0.0.0.0' if open_server else '127.0.0.1'
    app.run(debug=True, host=os.getenv('IP', default_host),
            port=int(os.getenv('PORT', 5000)))


@cli.group()
def process():
    pass


@process.command()
@click.option('--specific_folder', is_flag=True, default=False)
@click.option('--delete_all', is_flag=True)
@click.option('--path', type=str, default="static/data/cbs")
@click.option('--batch_size', type=int, default=10000)
@click.option('--delete_start_date', type=str, default=None)

def cbs(specific_folder, delete_all, path, batch_size, delete_start_date):
    from anyway.parsers.cbs import main

    return main(specific_folder=specific_folder, delete_all=delete_all, path=path,
                batch_size=batch_size, delete_start_date=delete_start_date)

@process.command()
@click.option('--specific_folder', is_flag=True, default=False)
@click.option('--delete_all', is_flag=True)
@click.option('--path', type=str, default="static/data/cbs_vehicles_registered")
def registered_vehicles(specific_folder, delete_all, path):
    from anyway.parsers.registered import main

    return main(specific_folder=specific_folder, delete_all=delete_all, path=path)


@process.command()
@click.option('--light', is_flag=True, help='Import without downloading any new files')
@click.option('--username', default='')
@click.option('--password', default='')<commit_msg>change batch_size to 5000<commit_after>#!/usr/bin/env python
import click
import logging
import os
import re
import sys


@click.group()
def cli():
    pass


@cli.command()
@click.option('--open', 'open_server', is_flag=True,
              help='Open the server for communication from outside', default=False)
@click.option('--debug-js', is_flag=True, help="Don't minify the JavaScript files")
def testserver(open_server, debug_js):
    from anyway import app
    from anyway.parsers import united
    from apscheduler.scheduler import Scheduler

    sched = Scheduler()

    @sched.interval_schedule(hours=12)
    def scheduled_import(): # pylint: disable=unused-variable
        united.main()
    sched.start()

    logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(message)s')

    if debug_js:
        app.config['ASSETS_DEBUG'] = True

    default_host = '0.0.0.0' if open_server else '127.0.0.1'
    app.run(debug=True, host=os.getenv('IP', default_host),
            port=int(os.getenv('PORT', 5000)))


@cli.group()
def process():
    pass


@process.command()
@click.option('--specific_folder', is_flag=True, default=False)
@click.option('--delete_all', is_flag=True)
@click.option('--path', type=str, default="static/data/cbs")
@click.option('--batch_size', type=int, default=5000)
@click.option('--delete_start_date', type=str, default=None)

def cbs(specific_folder, delete_all, path, batch_size, delete_start_date):
    from anyway.parsers.cbs import main

    return main(specific_folder=specific_folder, delete_all=delete_all, path=path,
                batch_size=batch_size, delete_start_date=delete_start_date)

@process.command()
@click.option('--specific_folder', is_flag=True, default=False)
@click.option('--delete_all', is_flag=True)
@click.option('--path', type=str, default="static/data/cbs_vehicles_registered")
def registered_vehicles(specific_folder, delete_all, path):
    from anyway.parsers.registered import main

    return main(specific_folder=specific_folder, delete_all=delete_all, path=path)


@process.command()
@click.option('--light', is_flag=True, help='Import without downloading any new files')
@click.option('--username', default='')
@click.option('--password', default='')
{'fraction_non_alphanumeric': 0.14693327342170298, 'fraction_numerical': 0.01011008762075938, 'mean_word_length': 5.0, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3530731', 'n_tokens_mistral': 1483, 'n_tokens_neox': 1419, 'n_words': 294}