<commit_before>import logging
import os
import subprocess
import yaml
from flask import json # require Flask > 0.10
from flask.ext.rq import job
import plistlib
from .decorators import cached
import checker.runner

ROOT = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..'))
DATA_ROOT = os.path.join(ROOT, 'data')

CLONE_PREPARE_SH = """mkdir -p %(login)s/%(project_id)s.in/ && mkdir -p %(login)s/%(project_id)s.out/src/"""
CLONE_SH = """git clone --depth=100 --quiet --branch=master %(clone)s ."""
CLEAN_SH = '' #"""cd %(root)s && rm -rf %(login)s/%(project_id)s.in/ && rm -rf %(login)s/%(project_id)s.out/"""

logger = logging.getLogger('bakery.tasks')
logger.setLevel(logging.INFO)

def run(*args, **kw):
    logger.info("%s %s" % (str(args), str(kw)))
    out = subprocess.check_output(*args, **kw)
    logger.info(out)

def prun(*args, **kw):
    logger.info("%s %s" % (str(args), str(kw)))
    return subprocess.Popen(*args, **kw)

def add_logger(login, project_id):
    fh = logging.FileHandler(os.path.join(DATA_ROOT, '%(login)s/process.%(project_id)s.log' % locals()))
    fh.setLevel(logging.INFO)
    logger.addHandler(fh)
    logger.info('='*80)
    return(fh)

def remove_logger(fh):
    logger.removeHandler(fh)

@job
def clone_and_process(project):
    git_clone(login = project.login, project_id = project.id, clone = project.clone)
    process_project(login = project.login, project_id = project.id)


@job
def logged_process(project):
    # fh = add_logger(login = project.login, project_id = project.id)
    # # push check before project process
    project_state_push(login = project.login, project_id = project.id)
    process_project(login = project.login, project_id = project.id)
    # remove_logger(fh)

@job
def git_clone(login, project_id, clone):
    git_clean(login, project_id)
    params = {'login': login,
        'project_id': project_id,
        }
    project_dir = os.path.join(DATA_ROOT, '%(login)s/%(project_id)s.in/' % params)
    if os.path.exists(project_dir):
        run('git pull origin master', shell= True, cwd=project_dir)
    else:
        run(CLONE_PREPARE_SH % params, shell=True, cwd=DATA_ROOT)
        # clone variable considered unsafe
        run(str(CLONE_SH % {'clone': clone}).split(), shell=False,
            cwd=os.path.join(DATA_ROOT, login, str(project_id)+'.in/'))

def git_clean(login, project_id):
    params = locals()
    params['root'] = DATA_ROOT
    run(CLEAN_SH % params, shell=True)

def check_yaml(login, project_id):
    yml = os.path.join(DATA_ROOT, '%(login)s/%(project_id)s.in/' % locals(), 'bakery.yaml')
    if not os.path.exists(yml):
        return 0
    return 1

def rwalk(path):
    h = {}
    cd = os.path.abspath(path)
    fs = os.listdir(path)
    for f in fs:
        cf = os.path.join(cd, f)
        if os.path.isfile(cf):
            h[f] = {}
        elif os.path.isdir(cf) and not cf.endswith('.git'):
            h[f] = rwalk(cf)
    return h

# @cached
def project_state_get(login, project_id, full=False):
    yml = os.path.join(DATA_ROOT, '%(login)s/%(project_id)s.in/bakery.yaml' % locals())
    dir_in = os.path.join(DATA_ROOT, '%(login)s/%(project_id)s.in/' % locals())
    if os.path.exists(yml):
        state = yaml.load(open(os.path.join(ROOT, 'bakery', 'bakery.defaults.yaml'), 'r').read()) # the same dir
        state.update(yaml.load(open(yml, 'r').read()))
    else:
        state = yaml.load(open(os.path.join(ROOT, 'bakery', 'bakery.defaults.yaml'), 'r').read()) # the same dir

    if not full:
        # don't need to walk over all usef folders
        return state

    txt_files = []
    ufo_dirs = []
    l = len(dir_in)
    for root, dirs, files in os.walk(dir_in):
        for f in files:
            fullpath = os.path.join(root, f)
            if os.path.splitext(fullpath)[1].lower() in ['.txt', '.md', '.markdown', 'LICENSE']:
                txt_files.append(fullpath[l:])
        for d in dirs:
            fullpath = os.path.join(root, d)
            if os.path.splitext(fullpath)[1].lower() == '.ufo':
                ufo_dirs.append(fullpath[l:])

    state['txt_files'] = txt_files
    state['ufo_dirs'] = ufo_dirs

    if os.path.exists(state['license_file']):
        state['license_file_found'] = True
<commit_msg>#90 auto select license file<commit_after>import logging
import os
import subprocess
import yaml
from flask import json # require Flask > 0.10
from flask.ext.rq import job
import plistlib
from .decorators import cached
import checker.runner

ROOT = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..'))
DATA_ROOT = os.path.join(ROOT, 'data')

CLONE_PREPARE_SH = """mkdir -p %(login)s/%(project_id)s.in/ && mkdir -p %(login)s/%(project_id)s.out/src/"""
CLONE_SH = """git clone --depth=100 --quiet --branch=master %(clone)s ."""
CLEAN_SH = '' #"""cd %(root)s && rm -rf %(login)s/%(project_id)s.in/ && rm -rf %(login)s/%(project_id)s.out/"""

logger = logging.getLogger('bakery.tasks')
logger.setLevel(logging.INFO)

def run(*args, **kw):
    logger.info("%s %s" % (str(args), str(kw)))
    out = subprocess.check_output(*args, **kw)
    logger.info(out)

def prun(*args, **kw):
    logger.info("%s %s" % (str(args), str(kw)))
    return subprocess.Popen(*args, **kw)

def add_logger(login, project_id):
    fh = logging.FileHandler(os.path.join(DATA_ROOT, '%(login)s/process.%(project_id)s.log' % locals()))
    fh.setLevel(logging.INFO)
    logger.addHandler(fh)
    logger.info('='*80)
    return(fh)

def remove_logger(fh):
    logger.removeHandler(fh)

@job
def clone_and_process(project):
    git_clone(login = project.login, project_id = project.id, clone = project.clone)
    process_project(login = project.login, project_id = project.id)


@job
def logged_process(project):
    # fh = add_logger(login = project.login, project_id = project.id)
    # # push check before project process
    project_state_push(login = project.login, project_id = project.id)
    process_project(login = project.login, project_id = project.id)
    # remove_logger(fh)

@job
def git_clone(login, project_id, clone):
    git_clean(login, project_id)
    params = {'login': login,
        'project_id': project_id,
        }
    project_dir = os.path.join(DATA_ROOT, '%(login)s/%(project_id)s.in/' % params)
    if os.path.exists(project_dir):
        run('git pull origin master', shell= True, cwd=project_dir)
    else:
        run(CLONE_PREPARE_SH % params, shell=True, cwd=DATA_ROOT)
        # clone variable considered unsafe
        run(str(CLONE_SH % {'clone': clone}).split(), shell=False,
            cwd=os.path.join(DATA_ROOT, login, str(project_id)+'.in/'))

def git_clean(login, project_id):
    params = locals()
    params['root'] = DATA_ROOT
    run(CLEAN_SH % params, shell=True)

def check_yaml(login, project_id):
    yml = os.path.join(DATA_ROOT, '%(login)s/%(project_id)s.in/' % locals(), 'bakery.yaml')
    if not os.path.exists(yml):
        return 0
    return 1

def rwalk(path):
    h = {}
    cd = os.path.abspath(path)
    fs = os.listdir(path)
    for f in fs:
        cf = os.path.join(cd, f)
        if os.path.isfile(cf):
            h[f] = {}
        elif os.path.isdir(cf) and not cf.endswith('.git'):
            h[f] = rwalk(cf)
    return h

# @cached
def project_state_get(login, project_id, full=False):
    yml = os.path.join(DATA_ROOT, '%(login)s/%(project_id)s.in/bakery.yaml' % locals())
    dir_in = os.path.join(DATA_ROOT, '%(login)s/%(project_id)s.in/' % locals())
    if os.path.exists(yml):
        state = yaml.load(open(os.path.join(ROOT, 'bakery', 'bakery.defaults.yaml'), 'r').read()) # the same dir
        state.update(yaml.load(open(yml, 'r').read()))
    else:
        state = yaml.load(open(os.path.join(ROOT, 'bakery', 'bakery.defaults.yaml'), 'r').read()) # the same dir

    if not full:
        # don't need to walk over all usef folders
        return state

    txt_files = []
    ufo_dirs = []
    l = len(dir_in)
    for root, dirs, files in os.walk(dir_in):
        for f in files:
            fullpath = os.path.join(root, f)
            if os.path.splitext(fullpath)[1].lower() in ['.txt', '.md', '.markdown', 'LICENSE']:
                txt_files.append(fullpath[l:])
        for d in dirs:
            fullpath = os.path.join(root, d)
            if os.path.splitext(fullpath)[1].lower() == '.ufo':
                ufo_dirs.append(fullpath[l:])

    state['txt_files'] = txt_files
    state['ufo_dirs'] = ufo_dirs

    # if lincense_file not defined then choose OFL.txt or LICENSE.txt from the root of repo
    if not state['license_file']:
        for fn in ['OFL.txt', 'LICENSE.txt']: # order means priority
            if os.path.exists(os.path.join(dir_in, fn)):
                state['license_file'] = fn
                break

    if os.path.exists(state['license_file']):
        state['license_file_found'] = True


{'fraction_non_alphanumeric': 0.16550247859396125, 'fraction_numerical': 0.0029292474087426767, 'mean_word_length': 3.209103840682788, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6910058', 'n_tokens_mistral': 3251, 'n_tokens_neox': 3169, 'n_words': 657}