<commit_before># -*- coding: utf-8 -*-

from optparse import make_option

from django.core.management import BaseCommand

from real_content.models import Language, Content

"""
python manage.py drc_loaddata -d titles_hr -l hr -c title
python manage.py drc_loaddata -d titles_en -l en -c title
python manage.py drc_loaddata -d paragraphs_hr -l hr -c paragraph
python manage.py drc_loaddata -d paragraphs_en -l en -c paragraph
"""

class Command(BaseCommand):
    help = u""" go """
    option_list = BaseCommand.option_list + (
        make_option('-d', '--doc', action='store', dest='doc',
            help=u'file to parse'),
        make_option('-l', '--language', action='store', dest='language',
            help=u'language of url'),
        make_option('-c', '--cat', action='store', dest='cat',
            help=u'category of content'),
        )<commit_msg>- drc_loaddata command help<commit_after># -*- coding: utf-8 -*-

from optparse import make_option

from django.core.management import BaseCommand

from real_content.models import Language, Content

"""
python manage.py drc_loaddata -d titles_hr -l hr -c title
python manage.py drc_loaddata -d titles_en -l en -c title
python manage.py drc_loaddata -d paragraphs_hr -l hr -c paragraph
python manage.py drc_loaddata -d paragraphs_en -l en -c paragraph
"""

class Command(BaseCommand):
    help = u"""
    Loads data from local files. File should contain one title/paragraph per line.

    python manage.py drc_loaddata -d local_file -l en -c title
    """
    option_list = BaseCommand.option_list + (
        make_option('-d', '--doc', action='store', dest='doc',
            help=u'file to parse'),
        make_option('-l', '--language', action='store', dest='language',
            help=u'language of url'),
        make_option('-c', '--cat', action='store', dest='cat',
            help=u'category of content'),
        )
{'fraction_non_alphanumeric': 0.13347457627118645, 'fraction_numerical': 0.001059322033898305, 'mean_word_length': 3.1699779249448126, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2644222', 'n_tokens_mistral': 591, 'n_tokens_neox': 573, 'n_words': 183}