<commit_before>from __future__ import unicode_literals
from __future__ import print_function
from __future__ import absolute_import

import json
import os
import shutil
import subprocess
import sys
import tempfile

from datetime import datetime
from optparse import make_option

from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from django.template.loader import render_to_string
from django.utils.translation import ugettext as _

from kalite.topic_tools import settings as topic_tools_settings, \
    get_content_cache, get_exercise_cache
from kalite.settings.base import CONTENT_ROOT
from kalite import i18n

from kalite_zim.utils import download_video, logger

from fle_utils.general import softload_json

from submarine.parser import parser as submarine_parser
from kalite_zim.anythumbnailer.thumbnail_ import create_thumbnail
from distutils.spawn import find_executable

def compressor_init(input_dir):

    settings.COMPRESS_PRECOMPILERS = (
        ('text/x-scss', 'django_libsass.SassCompiler'),
    )

    settings.STATICFILES_FINDERS = (
        'django.contrib.staticfiles.finders.FileSystemFinder',
        'django.contrib.staticfiles.finders.AppDirectoriesFinder',
        'compressor.finders.CompressorFinder',
    )

    settings.COMPRESS_ROOT = input_dir
    settings.COMPRESS_OUTPUT_DIR = ''
    settings.COMPRESS_CSS_FILTERS = []


class Command(BaseCommand):
    args = ('zimfile')
    help = 'Export video and meta data of your KA Lite installation to OpenZim'  # @ReservedAssignment
    option_list = BaseCommand.option_list + (
        make_option(
            '--language', '-l',
            action='store',
            dest='language',
            default='en',
            help='Select which language (videos and meta data) to export'
        ),
        make_option(
            '--tmp-dir', '-t',
            action='store',
            dest='tmp_dir',
            default='',
            help='Directory for the temporary zim filesystem'
        ),
        make_option(
            '--test',
            action='store_true',
            dest='test',
            help='Use test data'
        ),
        make_option(
            '--clear', '-c',
            action='store_true',
            dest='clear',
            default=False,
            help='Force clearing temporary fs and output destinations before write'
        ),
        make_option(
            '--resume', '-r',
            action='store_true',
            dest='clear',
            default=False,
            help='Resume writing into a dirty tmp-dir'
        ),
        make_option(
            '--download', '-d',
            action='store_true',
            dest='download',
            default=False,
            help='Instead of skipping videos that are not available, download them to KA Lite.'
        ),
        make_option(
            '--zimwriterfs', '-z',
            action='store',
            dest='zimwriterfs',
            default=None,
            help="Path to zimwriterfs if it's not on the shell path"
        ),
        make_option(
            '--publisher', '-p',
            action='store',
            dest='publisher',
            default="Learning Equality",
            help="Name of publisher"
        ),
        make_option(
            '--transcode2webm',
            action='store_true',
            dest='transcode2webm',
            default=False,
            help="Name of publisher"
        ),
    )

    def handle(self, *args, **options):
        if len(args) != 1:
            raise CommandError("Takes exactly 1 argument")

        dest_file = os.path.abspath(args[0])

        logger.info("Starting up KA Lite export2zim command")
        beginning = datetime.now()
        logger.info("Begin: {}".format(beginning))

        language = options.get('language')
        if not language:
            raise CommandError("Must specify a language!")

        if not options.get('tmp_dir'):
            tmp_dir = os.path.join(tempfile.gettempdir(), 'ka-lite-zim_{}'.format(language))
        else:
            tmp_dir = options.get('tmp_dir')

        tmp_dir = os.path.abspath(tmp_dir)

        if os.path.exists(tmp_dir) and os.listdir(tmp_dir):
            if options['clear']:
                logger.info("Clearing directory {}".format(tmp_dir))
                shutil.rmtree(tmp_dir)
            elif options['clear']:
                logger.info("Resuming in dirty tmp directory {}".format(tmp_dir))
            else:
                raise CommandError(
                    "{} not empty, use the -c option to clean it or use an empty destination directory.".format(
                        tmp_dir
                    )
                )

        zimwriterfs = options.get("zimwriterfs", None)
        publisher = options.get("publisher")
        transcode2webm = options.get("transcode2webm")
        ffmpeg = find_executable("ffmpeg")

        if not ffmpeg:
            logger.warning("FFMpeg not found in your path, you won't be able to create missing thumbnails or transcode to webm.")

        if not zimwriterfs:
            zimwriterfs = find_executable("zimwriterfs")
            if not zimwriterfs:
                raise CommandError("Could not find zimwriterfs in your path, try specifying --zimwriterfs=/path")

        if not os.path.exists(zimwriterfs):
            raise CommandError("Invalid --zimwriterfs")

        from kalite_zim import __name__ as base_path
        base_path = os.path.abspath(base_path)
        data_path = os.path.join(base_path, 'data')

        # Where subtitles are found in KA Lite
        subtitle_src_dir = i18n.get_srt_path(language)
<commit_msg>fix a typo<commit_after>from __future__ import unicode_literals
from __future__ import print_function
from __future__ import absolute_import

import json
import os
import shutil
import subprocess
import sys
import tempfile

from datetime import datetime
from optparse import make_option

from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from django.template.loader import render_to_string
from django.utils.translation import ugettext as _

from kalite.topic_tools import settings as topic_tools_settings, \
    get_content_cache, get_exercise_cache
from kalite.settings.base import CONTENT_ROOT
from kalite import i18n

from kalite_zim.utils import download_video, logger

from fle_utils.general import softload_json

from submarine.parser import parser as submarine_parser
from kalite_zim.anythumbnailer.thumbnail_ import create_thumbnail
from distutils.spawn import find_executable

def compressor_init(input_dir):

    settings.COMPRESS_PRECOMPILERS = (
        ('text/x-scss', 'django_libsass.SassCompiler'),
    )

    settings.STATICFILES_FINDERS = (
        'django.contrib.staticfiles.finders.FileSystemFinder',
        'django.contrib.staticfiles.finders.AppDirectoriesFinder',
        'compressor.finders.CompressorFinder',
    )

    settings.COMPRESS_ROOT = input_dir
    settings.COMPRESS_OUTPUT_DIR = ''
    settings.COMPRESS_CSS_FILTERS = []


class Command(BaseCommand):
    args = ('zimfile')
    help = 'Export video and meta data of your KA Lite installation to OpenZim'  # @ReservedAssignment
    option_list = BaseCommand.option_list + (
        make_option(
            '--language', '-l',
            action='store',
            dest='language',
            default='en',
            help='Select which language (videos and meta data) to export'
        ),
        make_option(
            '--tmp-dir', '-t',
            action='store',
            dest='tmp_dir',
            default='',
            help='Directory for the temporary zim filesystem'
        ),
        make_option(
            '--test',
            action='store_true',
            dest='test',
            help='Use test data'
        ),
        make_option(
            '--clear', '-c',
            action='store_true',
            dest='clear',
            default=False,
            help='Force clearing temporary fs and output destinations before write'
        ),
        make_option(
            '--resume', '-r',
            action='store_true',
            dest='resume',
            default=False,
            help='Resume writing into a dirty tmp-dir'
        ),
        make_option(
            '--download', '-d',
            action='store_true',
            dest='download',
            default=False,
            help='Instead of skipping videos that are not available, download them to KA Lite.'
        ),
        make_option(
            '--zimwriterfs', '-z',
            action='store',
            dest='zimwriterfs',
            default=None,
            help="Path to zimwriterfs if it's not on the shell path"
        ),
        make_option(
            '--publisher', '-p',
            action='store',
            dest='publisher',
            default="Learning Equality",
            help="Name of publisher"
        ),
        make_option(
            '--transcode2webm',
            action='store_true',
            dest='transcode2webm',
            default=False,
            help="Name of publisher"
        ),
    )

    def handle(self, *args, **options):
        if len(args) != 1:
            raise CommandError("Takes exactly 1 argument")

        dest_file = os.path.abspath(args[0])

        logger.info("Starting up KA Lite export2zim command")
        beginning = datetime.now()
        logger.info("Begin: {}".format(beginning))

        language = options.get('language')
        if not language:
            raise CommandError("Must specify a language!")

        if not options.get('tmp_dir'):
            tmp_dir = os.path.join(tempfile.gettempdir(), 'ka-lite-zim_{}'.format(language))
        else:
            tmp_dir = options.get('tmp_dir')

        tmp_dir = os.path.abspath(tmp_dir)

        if os.path.exists(tmp_dir) and os.listdir(tmp_dir):
            if options['clear']:
                logger.info("Clearing directory {}".format(tmp_dir))
                shutil.rmtree(tmp_dir)
            elif options['resume']:
                logger.info("Resuming in dirty tmp directory {}".format(tmp_dir))
            else:
                raise CommandError(
                    "{} not empty, use the -c option to clean it or use an empty destination directory.".format(
                        tmp_dir
                    )
                )

        zimwriterfs = options.get("zimwriterfs", None)
        publisher = options.get("publisher")
        transcode2webm = options.get("transcode2webm")
        ffmpeg = find_executable("ffmpeg")

        if not ffmpeg:
            logger.warning("FFMpeg not found in your path, you won't be able to create missing thumbnails or transcode to webm.")

        if not zimwriterfs:
            zimwriterfs = find_executable("zimwriterfs")
            if not zimwriterfs:
                raise CommandError("Could not find zimwriterfs in your path, try specifying --zimwriterfs=/path")

        if not os.path.exists(zimwriterfs):
            raise CommandError("Invalid --zimwriterfs")

        from kalite_zim import __name__ as base_path
        base_path = os.path.abspath(base_path)
        data_path = os.path.join(base_path, 'data')

        # Where subtitles are found in KA Lite
        subtitle_src_dir = i18n.get_srt_path(language)

{'fraction_non_alphanumeric': 0.09616908850726552, 'fraction_numerical': 0.0021136063408190224, 'mean_word_length': 2.134418989787469, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3860510', 'n_tokens_mistral': 3204, 'n_tokens_neox': 3111, 'n_words': 788}