<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import argparse
import json


parser = argparse.ArgumentParser()
parser.add_argument("--text", required=True, help="Dirty input text")
parser.add_argument("--short", action='store_true')
args = parser.parse_args()


def chars_score(text):
  known_chars = u'qwertzuiopasdfghjklyxcvbnm1234567890QWERTZUIOPASDFGHJKLYXCVBNM ?:.,;-=/+ěščřžýáíéĚŠČŘŽÝÁÍÉůúŇ'
  known_cnt = 0

  for char in text:
    if char in known_chars:
      known_cnt += 1

  return float(known_cnt)/len(text)

# TODO: add another functions here
# def xyz_score(text):
#    text magic here
#    return 0.5


def compute(scores):
    total = 0
    cnt = len(scores)
    for score in scores:
        total = (total * (cnt-1) + score['value']) / cnt;
    return {'score': total, 'parts': scores}


scores = []
scores.append({'name': 'chars_score', 'value': chars_score(args.text)})
# TODO: add another functions here
# scores.append({'name': 'xyz_score', 'value': xyz_score(args.text)})

total_score = compute(scores)<commit_msg>adding length score<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import argparse
import json
import math

parser = argparse.ArgumentParser()
parser.add_argument("--text", required=True, help="Input text")
parser.add_argument("--short", action='store_true')
args = parser.parse_args()

VALID_CHARS = u'qwertzuiopasdfghjklyxcvbnm1234567890QWERTZUIOPASDFGHJKLYXCVBNM ?:.,;-=/+ěščřžýáíéĚŠČŘŽÝÁÍÉůúŇ'

def chars_score(text):
    known_cnt = 0
    for char in text:
        if char in VALID_CHARS:
            known_cnt += 1
    return float(known_cnt)/len(text)

def length_score(text):
    cnt = 0
    for char in text:
        if char in VALID_CHARS:
            cnt += 1
    return min(math.log(cnt, 1000), 1.0)

# TODO: add another functions here
# def xyz_score(text):
#    text magic here
#    return 0.5


def compute(scores):
    total = 0
    cnt = 0
    for score in scores:
        cnt += 1
        total = (total * (cnt-1) + score['value']) / cnt;
    return {'score': total, 'parts': scores}


scores = []
scores.append({'name': 'chars_score', 'value': chars_score(args.text)})
scores.append({'name': 'length_score', 'value': length_score(args.text)})
# TODO: add another functions here
# scores.append({'name': 'xyz_score', 'value': xyz_score(args.text)})

total_score = compute(scores)
{'fraction_non_alphanumeric': 0.14684722809987305, 'fraction_numerical': 0.018620397799407534, 'mean_word_length': 3.9352818371607516, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1303543', 'n_tokens_mistral': 915, 'n_tokens_neox': 853, 'n_words': 198}