<commit_before>#!/usr/bin/env python

import sys
PY3 = sys.version_info > (3,)
import re
import csv
import os
if PY3:
    from queue import Queue
else:
    from Queue import Queue
from threading import Thread
from progressbar import ProgressBar, Percentage, Bar, RotatingMarker, ETA, SimpleProgress

DEBUG = False
# DEBUG = True
NUM_WORKERS = 30
MAX_FILES = -1
FIELD_NAMES = ['avg_word_length', 'avg_words_per_line', 'nr_lines',
    'nr_different_words', 'nr_unique_words', 'has_consecutive_pagebreaks', 'id',
    'nr_words']

def _re_matches(regex, line):
    m = re.search(regex, line)
    return m == None

def is_numeric(line):
    return re.match('^[0-9\.,]+$', line.strip())

def has_pagebreak(line):
    return _re_matches('\x0c+', line)

def has_consecutive_pagebreaks(line):<commit_msg>reduce number of workers to 10<commit_after>#!/usr/bin/env python

import sys
PY3 = sys.version_info > (3,)
import re
import csv
import os
if PY3:
    from queue import Queue
else:
    from Queue import Queue
from threading import Thread
from progressbar import ProgressBar, Percentage, Bar, RotatingMarker, ETA, SimpleProgress

DEBUG = False
# DEBUG = True
NUM_WORKERS = 10
MAX_FILES = -1
FIELD_NAMES = ['avg_word_length', 'avg_words_per_line', 'nr_lines',
    'nr_different_words', 'nr_unique_words', 'has_consecutive_pagebreaks', 'id',
    'nr_words']

def _re_matches(regex, line):
    m = re.search(regex, line)
    return m == None

def is_numeric(line):
    return re.match('^[0-9\.,]+$', line.strip())

def has_pagebreak(line):
    return _re_matches('\x0c+', line)

def has_consecutive_pagebreaks(line):
{'fraction_non_alphanumeric': 0.12196639701306783, 'fraction_numerical': 0.012445550715619166, 'mean_word_length': 4.868613138686132, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '57773', 'n_tokens_mistral': 588, 'n_tokens_neox': 567, 'n_words': 154}