<commit_before>import io
import csv
import logging
from normality import normalize, collapse_spaces
from servicelayer.cache import make_key

from aleph.core import settings, kv

log = logging.getLogger(__name__)
PLACE_KEY = 'ner*gns'

TAG_PERSON = 'peopleMentioned'
TAG_COMPANY = 'companiesMentioned'
TAG_LANGUAGE = 'detectedLanguage'
TAG_COUNTRY = 'detectedCountry'
TAG_EMAIL = 'emailMentioned'
TAG_PHONE = 'phoneMentioned'
TAG_IBAN = 'ibanMentioned'
TAG_LOCATION = 'location'

TEXT_MIN_LENGTH = 60
TEXT_MAX_LENGTH = 100000


def text_chunks(texts, sep=' '):
    chunk, total = [], 0
    for text in texts:
        text = collapse_spaces(text)
        if text is None or len(text) < 5:
            continue
        chunk.append(text)
        total += len(text)
        if total > TEXT_MAX_LENGTH:
            yield sep.join(chunk)
            chunk, total = [], 0
    # if total > TEXT_MIN_LENGTH:
    if len(chunk):
        yield sep.join(chunk)


def tag_key(label):
    return normalize(label, lowercase=True, ascii=True)


def place_key(name):
    return make_key(PLACE_KEY, name)


def load_places():
    if kv.get(PLACE_KEY) or settings.TESTING:
        return
    total = 0
    pipe = kv.pipeline(transaction=False)
    log.info("Loading geonames...")
    with io.open(settings.GEONAMES_DATA, 'r', encoding='utf-8') as fh:
        for row in csv.reader(fh, delimiter='\t'):<commit_msg>limit max text size<commit_after>import io
import csv
import logging
from normality import normalize, collapse_spaces
from servicelayer.cache import make_key

from aleph.core import settings, kv

log = logging.getLogger(__name__)
PLACE_KEY = 'ner*gns'

TAG_PERSON = 'peopleMentioned'
TAG_COMPANY = 'companiesMentioned'
TAG_LANGUAGE = 'detectedLanguage'
TAG_COUNTRY = 'detectedCountry'
TAG_EMAIL = 'emailMentioned'
TAG_PHONE = 'phoneMentioned'
TAG_IBAN = 'ibanMentioned'
TAG_LOCATION = 'location'

TEXT_MIN_LENGTH = 60
TEXT_MAX_LENGTH = 50000


def text_chunks(texts, sep=' ', step=5000):
    chunk, total = [], 0
    for text in texts:
        text = collapse_spaces(text)
        if text is None or len(text) < 5:
            continue
        text = text[:TEXT_MAX_LENGTH]
        chunk.append(text)
        total += len(text)
        if total > step:
            yield sep.join(chunk)
            chunk, total = [], 0
    # if total > TEXT_MIN_LENGTH:
    if len(chunk):
        yield sep.join(chunk)


def tag_key(label):
    return normalize(label, lowercase=True, ascii=True)


def place_key(name):
    return make_key(PLACE_KEY, name)


def load_places():
    if kv.get(PLACE_KEY) or settings.TESTING:
        return
    total = 0
    pipe = kv.pipeline(transaction=False)
    log.info("Loading geonames...")
    with io.open(settings.GEONAMES_DATA, 'r', encoding='utf-8') as fh:
        for row in csv.reader(fh, delimiter='\t'):
{'fraction_non_alphanumeric': 0.10533757511488158, 'fraction_numerical': 0.010250972074938141, 'mean_word_length': 3.3808049535603715, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4000098', 'n_tokens_mistral': 1001, 'n_tokens_neox': 943, 'n_words': 239}