<commit_before>import os
import sys
import json
from paver.easy import *
from paver.path import path

HOME_DIR = path(__file__).dirname().abspath()

options(
    home_dir = HOME_DIR,
    build_dir = HOME_DIR / 'build',
    src_dir = HOME_DIR / 'src',
    test_dir = HOME_DIR / 'test',
    tmp_dir = HOME_DIR / '.tmp',
    closure_library_dir = HOME_DIR / '../closure-library-read-only',
    closure_compiler = HOME_DIR / '../closure-compiler-read-only/build/compiler.jar',
    closure_lint = 'gjslint',
    closure_fix_lint = 'fixjsstyle',
)

# Run options again to use variables from previous declaration
options(
    modules_file = options.src_dir / 'modules.json',
    externs_dir = options.src_dir / 'externs',
    calcdeps = options.closure_library_dir / 'closure/bin/calcdeps.py',
    depswriter = options.closure_library_dir / 'closure/bin/build/depswriter.py',
)

# Closure compiler options
options(
    compiler_errors = [
        '', # First one blank for join()
        'accessControls',
        'checkRegExp',
        'checkTypes',
        'checkVars',
        'deprecated',
        'fileoverviewTags',
        'invalidCasts',
        'missingProperties',
        'undefinedVars',
        'visibility'
    ],
    compiler_warnings = [
        '', # First one blank for join()
        'nonStandardJsDocs',
        'strictModuleDepCheck',
        'unknownDefines',
    ],
    compiler_flags = [
        '--warning_level=VERBOSE',
        '--summary_detail_level=3',
    ]
)

def check_requirements():
    """Make sure all system requirements are met"""
    if not options.closure_library_dir.isdir():
        raise BuildFailure("Closure library not found")

    if not options.closure_compiler.isfile():
        raise BuildFailure("Closure compiler not found")

# Run requirements test no matter what
check_requirements()

@task
def clean():
    """Cleans up the project directory"""
    options.build_dir.rmtree()
    options.tmp_dir.rmtree()

@task
@consume_args
def default(args = []):
    """Write out dependency file in order to test non-compiled scripts"""
    js_files = options.src_dir.files('*.js')

    output_mode = 'deps'

    sh('python %s -i %s -p % s -d %s -o %s --output_file %s' % (
        options.calcdeps,
        ' -i '.join(js_files),
        options.src_dir,
        options.closure_library_dir,
        output_mode,
        options.test_dir / 'deps.js'
    ))

@task
@consume_args
def debug(args):
    """Create debug versions for testing"""
    output_mode = 'script'

    for jsfile in options.src_dir.files('*.js'):
        outfile = options.build_dir / jsfile.basename()
        js = sh('python %s -i %s -p % s -p %s -o %s --output_file %s' % (
            options.calcdeps,
            jsfile,
            options.src_dir,
            options.closure_library_dir,
            output_mode,
            outfile
        ), capture=True)

        # Need to modify the output in order to set COMPILED to true, since
        # this normally happens via the compiler
        contents = outfile.text().replace('COMPILED = false', 'COMPILED = true', 1)
        outfile.write_text(contents)

@task
def lint():
    """Run Google's style checker on source files"""
    if not options.tmp_dir.isdir():
        options.tmp_dir.makedirs()

    last_run = None
    lint_flag = options.tmp_dir / 'lint'
    if lint_flag.exists():
        last_run = (options.tmp_dir / 'lint').mtime

    lint_flag.touch()

    for file in options.src_dir.walkfiles('*.js'):<commit_msg>Don't do lint checks on extern files<commit_after>import os
import sys
import json
from paver.easy import *
from paver.path import path

HOME_DIR = path(__file__).dirname().abspath()

options(
    home_dir = HOME_DIR,
    build_dir = HOME_DIR / 'build',
    src_dir = HOME_DIR / 'src',
    test_dir = HOME_DIR / 'test',
    tmp_dir = HOME_DIR / '.tmp',
    closure_library_dir = HOME_DIR / '../closure-library-read-only',
    closure_compiler = HOME_DIR / '../closure-compiler-read-only/build/compiler.jar',
    closure_lint = 'gjslint',
    closure_fix_lint = 'fixjsstyle',
)

# Run options again to use variables from previous declaration
options(
    modules_file = options.src_dir / 'modules.json',
    externs_dir = options.src_dir / 'externs',
    calcdeps = options.closure_library_dir / 'closure/bin/calcdeps.py',
    depswriter = options.closure_library_dir / 'closure/bin/build/depswriter.py',
)

# Closure compiler options
options(
    compiler_errors = [
        '', # First one blank for join()
        'accessControls',
        'checkRegExp',
        'checkTypes',
        'checkVars',
        'deprecated',
        'fileoverviewTags',
        'invalidCasts',
        'missingProperties',
        'undefinedVars',
        'visibility'
    ],
    compiler_warnings = [
        '', # First one blank for join()
        'nonStandardJsDocs',
        'strictModuleDepCheck',
        'unknownDefines',
    ],
    compiler_flags = [
        '--warning_level=VERBOSE',
        '--summary_detail_level=3',
    ]
)

def check_requirements():
    """Make sure all system requirements are met"""
    if not options.closure_library_dir.isdir():
        raise BuildFailure("Closure library not found")

    if not options.closure_compiler.isfile():
        raise BuildFailure("Closure compiler not found")

# Run requirements test no matter what
check_requirements()

@task
def clean():
    """Cleans up the project directory"""
    options.build_dir.rmtree()
    options.tmp_dir.rmtree()

@task
@consume_args
def default(args = []):
    """Write out dependency file in order to test non-compiled scripts"""
    js_files = options.src_dir.files('*.js')

    output_mode = 'deps'

    sh('python %s -i %s -p % s -d %s -o %s --output_file %s' % (
        options.calcdeps,
        ' -i '.join(js_files),
        options.src_dir,
        options.closure_library_dir,
        output_mode,
        options.test_dir / 'deps.js'
    ))

@task
@consume_args
def debug(args):
    """Create debug versions for testing"""
    output_mode = 'script'

    for jsfile in options.src_dir.files('*.js'):
        outfile = options.build_dir / jsfile.basename()
        js = sh('python %s -i %s -p % s -p %s -o %s --output_file %s' % (
            options.calcdeps,
            jsfile,
            options.src_dir,
            options.closure_library_dir,
            output_mode,
            outfile
        ), capture=True)

        # Need to modify the output in order to set COMPILED to true, since
        # this normally happens via the compiler
        contents = outfile.text().replace('COMPILED = false', 'COMPILED = true', 1)
        outfile.write_text(contents)

@task
def lint():
    """Run Google's style checker on source files"""
    if not options.tmp_dir.isdir():
        options.tmp_dir.makedirs()

    last_run = None
    lint_flag = options.tmp_dir / 'lint'
    if lint_flag.exists():
        last_run = (options.tmp_dir / 'lint').mtime

    lint_flag.touch()

    for file in options.src_dir.walkfiles('*.js'):
        # Skip extern files
        if file.parent == options.externs_dir:
            continue

{'fraction_non_alphanumeric': 0.12126760563380282, 'fraction_numerical': 0.0005633802816901409, 'mean_word_length': 2.5954430379746833, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2164336', 'n_tokens_mistral': 2243, 'n_tokens_neox': 2206, 'n_words': 546}