<commit_before>import os
import sys
import json
import time
import click
import pkg_resources

from .i18n import get_default_lang, is_valid_language
from .utils import secure_url
from .project import Project


version = pkg_resources.get_distribution('Lektor').version


def echo_json(data):
    click.echo(json.dumps(data, indent=2).rstrip())


def buildflag(cli):
    return click.option(
        '-f', '--build-flag', 'build_flags', multiple=True,
        help='Defines an arbitrary build flag.  These can be used by plugins '
        'to customize the build process.  More information can be found in '
        'the documentation of affected plugins.')(cli)


class Context(object):

    def __init__(self):
        self._project_path = os.environ.get('LEKTOR_PROJECT') or None
        self._project = None
        self._env = None
        self._ui_lang = None

    def _get_ui_lang(self):
        rv = self._ui_lang
        if rv is None:
            rv = self._ui_lang = get_default_lang()
        return rv

    def _set_ui_lang(self, value):
        self._ui_lang = value

    ui_lang = property(_get_ui_lang, _set_ui_lang)
    del _get_ui_lang, _set_ui_lang

    def set_project_path(self, value):
        self._project_path = value
        self._project = None

    def get_project(self, silent=False):
        if self._project is not None:
            return self._project
        if self._project_path is not None:
            rv = Project.from_path(self._project_path)
        else:
            rv = Project.discover()
        if rv is None:
            if silent:
                return None
            if self._project_path is None:
                raise click.UsageError('Could not automatically discover '
                                       'project.  A Lektor project must '
                                       'exist in the working directory or '
                                       'any of the parent directories.')
            raise click.UsageError('Could not find project "%s"' %
                                   self._project_path)
        self._project = rv
        return rv

    def get_default_output_path(self):
        rv = os.environ.get('LEKTOR_OUTPUT_PATH')
        if rv is not None:
            return rv
        return self.get_project().get_output_path()

    def get_env(self):
        if self._env is not None:
            return self._env
        from lektor.environment import Environment
        env = Environment(self.get_project(), load_plugins=False)
        self._env = env
        return env

    def load_plugins(self, reinstall=False):
        from .packages import load_packages
        from .pluginsystem import initialize_plugins
        load_packages(self.get_env(), reinstall=reinstall)
        initialize_plugins(self.get_env())


pass_context = click.make_pass_decorator(Context, ensure=True)


def validate_language(ctx, param, value):
    if value is not None and not is_valid_language(value):
        raise click.BadParameter('Unsupported language "%s".' % value)
    return value


@click.group()
@click.option('--project', type=click.Path(),
              help='The path to the lektor project to work with.')
@click.option('--language', default=None, callback=validate_language,
              help='The UI language to use (overrides autodetection).')
@click.version_option(prog_name='Lektor', version=version)
@pass_context
def cli(ctx, project=None, language=None):
    """The lektor management application.

    This command can invoke lektor locally and serve up the website.  It's
    intended for local development of websites.
    """
    if language is not None:
        ctx.ui_lang = language
    if project is not None:
        ctx.set_project_path(project)


@cli.command('build')
@click.option('-O', '--output-path', type=click.Path(), default=None,
              help='The output path.')
@click.option('--watch', is_flag=True, help='If this is enabled the build '
              'process goes into an automatic loop where it watches the '
              'file system for changes and rebuilds.')
@click.option('--prune/--no-prune', default=True, help='Controls if old '
              'artifacts should be pruned.  This is the default.')
@click.option('-v', '--verbose', 'verbosity', count=True,
              help='Increases the verbosity of the logging.')
@click.option('--source-info-only', is_flag=True,
              help='Instead of building only updates the source infos.  The '
              'source info is used by the web admin panel to quickly find '
              'information about the source files (for instance jump to '
              'files).')
@buildflag
@click.option('--profile', is_flag=True,
              help='Enable build profiler.')
@pass_context
def build_cmd(ctx, output_path, watch, prune, verbosity,
              source_info_only, profile, build_flags):
    """Builds the entire project into the final artifacts.

    The default behavior is to build the project into the default build
    output path which can be discovered with the `project-info` command
    but an alternative output folder can be provided with the `--output-path`
    option.

    The default behavior is to perform a build followed by a pruning step
    which removes no longer referenced artifacts from the output folder.
    Lektor will only build the files that require rebuilding if the output
    folder is reused.

    To enforce a clean build you have to issue a `clean` command first.

    If the build fails the exit code will be `1` otherwise `0`.  This can be
    used by external scripts to only deploy on successful build for instance.
    """
    from lektor.builder import Builder
    from lektor.reporter import CliReporter

    if output_path is None:
        output_path = ctx.get_default_output_path()

    ctx.load_plugins()

    env = ctx.get_env()

    def _build():
        builder = Builder(env.new_pad(), output_path,
                          build_flags=build_flags)
        if source_info_only:
            builder.update_all_source_infos()
            return True

        if profile:
            from .utils import profile_func
            failures = profile_func(builder.build_all)
        else:
            failures = builder.build_all()
        if prune:
            builder.prune()
        return failures == 0

    reporter = CliReporter(env, verbosity=verbosity)
    with reporter:
        success = _build()
        if not watch:
            return sys.exit(0 if success else 1)

        from lektor.watcher import watch
        click.secho('Watching for file system changes', fg='cyan')
        last_build = time.time()
        for ts, _, _ in watch(env):
            if ts > last_build:
                _build()
                last_build = time.time()


@cli.command('clean')
@click.option('-O', '--output-path', type=click.Path(), default=None,
              help='The output path.')
@click.option('-v', '--verbose', 'verbosity', count=True,
              help='Increases the verbosity of the logging.')
@click.confirmation_option(help='Confirms the cleaning.')
@pass_context
def clean_cmd(ctx, output_path, verbosity):
    """Cleans the entire build folder.

    If not build folder is provided, the default build folder of the project
    in the Lektor cache is used.
    """
    from lektor.builder import Builder
    from lektor.reporter import CliReporter

    if output_path is None:
        output_path = ctx.get_default_output_path()

    env = ctx.get_env()

    reporter = CliReporter(env, verbosity=verbosity)
    with reporter:
        builder = Builder(env.new_pad(), output_path)
        builder.prune(all=True)


@cli.command('deploy', short_help='Deploy the website.')
@click.argument('server', required=False)
@click.option('-O', '--output-path', type=click.Path(), default=None,
              help='The output path.')
@click.option('--username', envvar='LEKTOR_DEPLOY_USERNAME',
              help='An optional username to override the URL.')
@click.option('--password', envvar='LEKTOR_DEPLOY_PASSWORD',
              help='An optional password to override the URL or the '
              'default prompt.')
@click.option('--key-file', envvar='LEKTOR_DEPLOY_KEY_FILE',
              help='The path to a key file that should be used for the '
              'authentication of the deployment.')
@click.option('--key', envvar='LEKTOR_DEPLOY_KEY',
              help='The contents of a key file directly a string that should '
              'be used for authentication of the deployment.')<commit_msg>Load plugins before "lektor clean".<commit_after>import os
import sys
import json
import time
import click
import pkg_resources

from .i18n import get_default_lang, is_valid_language
from .utils import secure_url
from .project import Project


version = pkg_resources.get_distribution('Lektor').version


def echo_json(data):
    click.echo(json.dumps(data, indent=2).rstrip())


def buildflag(cli):
    return click.option(
        '-f', '--build-flag', 'build_flags', multiple=True,
        help='Defines an arbitrary build flag.  These can be used by plugins '
        'to customize the build process.  More information can be found in '
        'the documentation of affected plugins.')(cli)


class Context(object):

    def __init__(self):
        self._project_path = os.environ.get('LEKTOR_PROJECT') or None
        self._project = None
        self._env = None
        self._ui_lang = None

    def _get_ui_lang(self):
        rv = self._ui_lang
        if rv is None:
            rv = self._ui_lang = get_default_lang()
        return rv

    def _set_ui_lang(self, value):
        self._ui_lang = value

    ui_lang = property(_get_ui_lang, _set_ui_lang)
    del _get_ui_lang, _set_ui_lang

    def set_project_path(self, value):
        self._project_path = value
        self._project = None

    def get_project(self, silent=False):
        if self._project is not None:
            return self._project
        if self._project_path is not None:
            rv = Project.from_path(self._project_path)
        else:
            rv = Project.discover()
        if rv is None:
            if silent:
                return None
            if self._project_path is None:
                raise click.UsageError('Could not automatically discover '
                                       'project.  A Lektor project must '
                                       'exist in the working directory or '
                                       'any of the parent directories.')
            raise click.UsageError('Could not find project "%s"' %
                                   self._project_path)
        self._project = rv
        return rv

    def get_default_output_path(self):
        rv = os.environ.get('LEKTOR_OUTPUT_PATH')
        if rv is not None:
            return rv
        return self.get_project().get_output_path()

    def get_env(self):
        if self._env is not None:
            return self._env
        from lektor.environment import Environment
        env = Environment(self.get_project(), load_plugins=False)
        self._env = env
        return env

    def load_plugins(self, reinstall=False):
        from .packages import load_packages
        from .pluginsystem import initialize_plugins
        load_packages(self.get_env(), reinstall=reinstall)
        initialize_plugins(self.get_env())


pass_context = click.make_pass_decorator(Context, ensure=True)


def validate_language(ctx, param, value):
    if value is not None and not is_valid_language(value):
        raise click.BadParameter('Unsupported language "%s".' % value)
    return value


@click.group()
@click.option('--project', type=click.Path(),
              help='The path to the lektor project to work with.')
@click.option('--language', default=None, callback=validate_language,
              help='The UI language to use (overrides autodetection).')
@click.version_option(prog_name='Lektor', version=version)
@pass_context
def cli(ctx, project=None, language=None):
    """The lektor management application.

    This command can invoke lektor locally and serve up the website.  It's
    intended for local development of websites.
    """
    if language is not None:
        ctx.ui_lang = language
    if project is not None:
        ctx.set_project_path(project)


@cli.command('build')
@click.option('-O', '--output-path', type=click.Path(), default=None,
              help='The output path.')
@click.option('--watch', is_flag=True, help='If this is enabled the build '
              'process goes into an automatic loop where it watches the '
              'file system for changes and rebuilds.')
@click.option('--prune/--no-prune', default=True, help='Controls if old '
              'artifacts should be pruned.  This is the default.')
@click.option('-v', '--verbose', 'verbosity', count=True,
              help='Increases the verbosity of the logging.')
@click.option('--source-info-only', is_flag=True,
              help='Instead of building only updates the source infos.  The '
              'source info is used by the web admin panel to quickly find '
              'information about the source files (for instance jump to '
              'files).')
@buildflag
@click.option('--profile', is_flag=True,
              help='Enable build profiler.')
@pass_context
def build_cmd(ctx, output_path, watch, prune, verbosity,
              source_info_only, profile, build_flags):
    """Builds the entire project into the final artifacts.

    The default behavior is to build the project into the default build
    output path which can be discovered with the `project-info` command
    but an alternative output folder can be provided with the `--output-path`
    option.

    The default behavior is to perform a build followed by a pruning step
    which removes no longer referenced artifacts from the output folder.
    Lektor will only build the files that require rebuilding if the output
    folder is reused.

    To enforce a clean build you have to issue a `clean` command first.

    If the build fails the exit code will be `1` otherwise `0`.  This can be
    used by external scripts to only deploy on successful build for instance.
    """
    from lektor.builder import Builder
    from lektor.reporter import CliReporter

    if output_path is None:
        output_path = ctx.get_default_output_path()

    ctx.load_plugins()

    env = ctx.get_env()

    def _build():
        builder = Builder(env.new_pad(), output_path,
                          build_flags=build_flags)
        if source_info_only:
            builder.update_all_source_infos()
            return True

        if profile:
            from .utils import profile_func
            failures = profile_func(builder.build_all)
        else:
            failures = builder.build_all()
        if prune:
            builder.prune()
        return failures == 0

    reporter = CliReporter(env, verbosity=verbosity)
    with reporter:
        success = _build()
        if not watch:
            return sys.exit(0 if success else 1)

        from lektor.watcher import watch
        click.secho('Watching for file system changes', fg='cyan')
        last_build = time.time()
        for ts, _, _ in watch(env):
            if ts > last_build:
                _build()
                last_build = time.time()


@cli.command('clean')
@click.option('-O', '--output-path', type=click.Path(), default=None,
              help='The output path.')
@click.option('-v', '--verbose', 'verbosity', count=True,
              help='Increases the verbosity of the logging.')
@click.confirmation_option(help='Confirms the cleaning.')
@pass_context
def clean_cmd(ctx, output_path, verbosity):
    """Cleans the entire build folder.

    If not build folder is provided, the default build folder of the project
    in the Lektor cache is used.
    """
    from lektor.builder import Builder
    from lektor.reporter import CliReporter

    if output_path is None:
        output_path = ctx.get_default_output_path()

    ctx.load_plugins()
    env = ctx.get_env()

    reporter = CliReporter(env, verbosity=verbosity)
    with reporter:
        builder = Builder(env.new_pad(), output_path)
        builder.prune(all=True)


@cli.command('deploy', short_help='Deploy the website.')
@click.argument('server', required=False)
@click.option('-O', '--output-path', type=click.Path(), default=None,
              help='The output path.')
@click.option('--username', envvar='LEKTOR_DEPLOY_USERNAME',
              help='An optional username to override the URL.')
@click.option('--password', envvar='LEKTOR_DEPLOY_PASSWORD',
              help='An optional password to override the URL or the '
              'default prompt.')
@click.option('--key-file', envvar='LEKTOR_DEPLOY_KEY_FILE',
              help='The path to a key file that should be used for the '
              'authentication of the deployment.')
@click.option('--key', envvar='LEKTOR_DEPLOY_KEY',
              help='The contents of a key file directly a string that should '
              'be used for authentication of the deployment.')
{'fraction_non_alphanumeric': 0.09331386861313869, 'fraction_numerical': 0.0009343065693430657, 'mean_word_length': 2.5051166598444534, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6834785', 'n_tokens_mistral': 4810, 'n_tokens_neox': 4687, 'n_words': 1530}