<commit_before>#!/usr/bin/env python
__author__ = 'mrx'

import sys

import utils.read_configs as cfg
import lexers.utils.common as lex
from gitignore_gen import gen as git_gen
from utils.pyutils import relative_path, snd, fst
from model import compute_neurons


KEYWORDS_PATHS = ['lexers/utils/c_keywords.txt',
                  'lexers/utils/java_keywords.txt',
                  'lexers/utils/python_keywords.txt']


KEYWORDS_DIR = 'lexers/utils'


MODEL_CONFIG = 'neurons.csv'


def inputs_dict(filename, kws, langs):
    with open(filename, "r") as in_file:
        contents = in_file.read()
        inp_list_raw = lex.keywordsAndSymbolsFrequencies(kws, contents)
        inp_list = {}
        for k, v in inp_list_raw.iteritems():
            inp_list[str(k)] = v
        inp_list.update({'intercept' : 1})
    inp_dict = {}
    for l in langs:
        inp_dict.update({l: inp_list})
    return inp_dict, map(lambda x: str(x), inp_list.keys())


def detect(filename):
    langs, kws = cfg.read_kws(relative_path(KEYWORDS_DIR, __file__))
    inputs, _ = inputs_dict(filename, kws, langs)
    model_coeffs = cfg.read_config(relative_path(MODEL_CONFIG, __file__))
    responses_dict = compute_neurons(inputs, model_coeffs)
    min_response = min(responses_dict.iteritems(), key=snd)
    return fst(min_response).capitalize()
<commit_msg>Max now<commit_after>#!/usr/bin/env python
__author__ = 'mrx'

import sys

import utils.read_configs as cfg
import lexers.utils.common as lex
from gitignore_gen import gen as git_gen
from utils.pyutils import relative_path, snd, fst
from model import compute_neurons


KEYWORDS_PATHS = ['lexers/utils/c_keywords.txt',
                  'lexers/utils/java_keywords.txt',
                  'lexers/utils/python_keywords.txt']


KEYWORDS_DIR = 'lexers/utils'


MODEL_CONFIG = 'neurons.csv'


def inputs_dict(filename, kws, langs):
    with open(filename, "r") as in_file:
        contents = in_file.read()
        inp_list_raw = lex.keywordsAndSymbolsFrequencies(kws, contents)
        inp_list = {}
        for k, v in inp_list_raw.iteritems():
            inp_list[str(k)] = v
        inp_list.update({'intercept' : 1})
    inp_dict = {}
    for l in langs:
        inp_dict.update({l: inp_list})
    return inp_dict, map(lambda x: str(x), inp_list.keys())


def detect(filename):
    langs, kws = cfg.read_kws(relative_path(KEYWORDS_DIR, __file__))
    inputs, _ = inputs_dict(filename, kws, langs)
    model_coeffs = cfg.read_config(relative_path(MODEL_CONFIG, __file__))
    responses_dict = compute_neurons(inputs, model_coeffs)
    print(responses_dict)
    min_response = max(responses_dict.iteritems(), key=snd)
    return fst(min_response).capitalize()

{'fraction_non_alphanumeric': 0.10962962962962963, 'fraction_numerical': 0.0007407407407407407, 'mean_word_length': 3.6013628620102214, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3334464', 'n_tokens_mistral': 983, 'n_tokens_neox': 954, 'n_words': 194}