def process(self, lines, percent_max_numeric=0.7, percent_max_non_ascii=0.4, min_line_length=20, verbose=False, disable_pbar=True):
    skipped_because_min_length = np.array([0, 0], dtype=np.uint64)
    skipped_alpha_count = np.array([0, 0], dtype=np.uint64)
    skipped_because_max_numeric = np.array([0, 0], dtype=np.uint64)
    skipped_because_max_non_ascii = np.array([0, 0], dtype=np.uint64)
    skipped_because_forbidden_chars = np.array([0, 0], dtype=np.uint64)
    total_original_length = 0
    total_clean_length = 0
    output = []
    for line in tqdm(lines, disable=disable_pbar):
        line = line.strip()
        length = len(line)
        total_original_length += length
        if (length < min_line_length):
            skipped_because_min_length += np.array([1, length], dtype=np.uint64)
            continue
        line = bytes(line, 'utf-8').decode('utf-8', 'ignore')
        digit_count = 0
        alpha_count = 0
        ascii_count = 0
        forbidden_char = False
        for char in line:
            if (char in self.forbidden_chars):
                forbidden_char = True
                break
            if char.isnumeric():
                digit_count += 1
            if char.isalpha():
                alpha_count += 1
            if char.isascii():
                ascii_count += 1
        if forbidden_char:
            skipped_because_forbidden_chars += np.array([1, length], dtype=np.uint64)
            continue
        if ((alpha_count == 0) or ((alpha_count / length) < 0.5)):
            skipped_alpha_count += np.array([1, length], dtype=np.uint64)
            if verbose:
                print('Skipping alpha={:.3f}: [{}]'.format((alpha_count / length), line))
            continue
        if (((digit_count / alpha_count) >= percent_max_numeric) and (digit_count > 6)):
            skipped_because_max_numeric += np.array([1, length], dtype=np.uint64)
            if verbose:
                print('Skipping digit={:.3f}: [{}]'.format((digit_count / alpha_count), line))
            continue
        if (((ascii_count / alpha_count) < percent_max_non_ascii) and (length > 15)):
            skipped_because_max_non_ascii += np.array([1, length], dtype=np.uint64)
            if verbose:
                print('Skipping ascii={:.3f}: [{}]'.format((digit_count / alpha_count), line))
            continue
        if (((line.strip()[0] == '|') and (line.count('|') > 2)) or ((line.strip()[0] == '│') and (line.count('│') > 2))):
            skipped_because_forbidden_chars += np.array([1, length], dtype=np.uint64)
            if verbose:
                print('Skipping table line: [{}]'.format(line))
            continue
        line = self.r1.sub('\\1\\2', line)
        line = self.r2.sub('\\1\\2', line)
        line = self.r3.sub('-', line)
        line = self.r4.sub('\\1\\2', line)
        line = self.r5.sub('', line)
        line = self.r6.sub('', line)
        line = self.r7.sub('', line)
        line = self.r8.sub('', line)
        line = self.r9.sub('', line)
        line = line.replace('( ă)', '(ă)')
        line = line.replace('ţ', 'ț')
        line = line.replace('ş', 'ș')
        line = line.replace('Ţ', 'Ț')
        line = line.replace('Ş', 'Ș')
        line = line.replace('Ã¢', 'â')
        line = self.space.sub(' ', line).strip()
        if (len(line) < min_line_length):
            skipped_because_min_length += np.array([1, length], dtype=np.uint64)
            continue
        total_clean_length += len(line)
        output.append((line + '\n'))
    stats = {}
    stats['skipped_because_min_length'] = skipped_because_min_length
    stats['skipped_alpha_count'] = skipped_alpha_count
    stats['skipped_because_max_numeric'] = skipped_because_max_numeric
    stats['skipped_because_max_non_ascii'] = skipped_because_max_non_ascii
    stats['skipped_because_forbidden_chars'] = skipped_because_forbidden_chars
    stats['total_original_length'] = total_original_length
    stats['total_clean_length'] = total_clean_length
    return (output, stats)
def process(self, lines, percent_max_numeric=0.7, percent_max_non_ascii=0.4, min_line_length=20, verbose=False, disable_pbar=True):
    skipped_because_min_length = np.array([0, 0], dtype=np.uint64)
    skipped_alpha_count = np.array([0, 0], dtype=np.uint64)
    skipped_because_max_numeric = np.array([0, 0], dtype=np.uint64)
    skipped_because_max_non_ascii = np.array([0, 0], dtype=np.uint64)
    skipped_because_forbidden_chars = np.array([0, 0], dtype=np.uint64)
    total_original_length = 0
    total_clean_length = 0
    output = []
    for line in tqdm(lines, disable=disable_pbar):
        line = line.strip()
        length = len(line)
        total_original_length += length
        if (length < min_line_length):
            skipped_because_min_length += np.array([1, length], dtype=np.uint64)
            continue
        line = bytes(line, 'utf-8').decode('utf-8', 'ignore')
        digit_count = 0
        alpha_count = 0
        ascii_count = 0
        forbidden_char = False
        for char in line:
            if (char in self.forbidden_chars):
                forbidden_char = True
                break
            if char.isnumeric():
                digit_count += 1
            if char.isalpha():
                alpha_count += 1
            if char.isascii():
                ascii_count += 1
        if forbidden_char:
            skipped_because_forbidden_chars += np.array([1, length], dtype=np.uint64)
            continue
        if ((alpha_count == 0) or ((alpha_count / length) < 0.5)):
            skipped_alpha_count += np.array([1, length], dtype=np.uint64)
            if verbose:
                print('Skipping alpha={:.3f}: [{}]'.format((alpha_count / length), line))
            continue
        if (((digit_count / alpha_count) >= percent_max_numeric) and (digit_count > 6)):
            skipped_because_max_numeric += np.array([1, length], dtype=np.uint64)
            if verbose:
                print('Skipping digit={:.3f}: [{}]'.format((digit_count / alpha_count), line))
            continue
        if (((ascii_count / alpha_count) < percent_max_non_ascii) and (length > 15)):
            skipped_because_max_non_ascii += np.array([1, length], dtype=np.uint64)
            if verbose:
                print('Skipping ascii={:.3f}: [{}]'.format((digit_count / alpha_count), line))
            continue
        if (((line.strip()[0] == '|') and (line.count('|') > 2)) or ((line.strip()[0] == '│') and (line.count('│') > 2))):
            skipped_because_forbidden_chars += np.array([1, length], dtype=np.uint64)
            if verbose:
                print('Skipping table line: [{}]'.format(line))
            continue
        line = self.r1.sub('\\1\\2', line)
        line = self.r2.sub('\\1\\2', line)
        line = self.r3.sub('-', line)
        line = self.r4.sub('\\1\\2', line)
        line = self.r5.sub(, line)
        line = self.r6.sub(, line)
        line = self.r7.sub(, line)
        line = self.r8.sub(, line)
        line = self.r9.sub(, line)
        line = line.replace('( ă)', '(ă)')
        line = line.replace('ţ', 'ț')
        line = line.replace('ş', 'ș')
        line = line.replace('Ţ', 'Ț')
        line = line.replace('Ş', 'Ș')
        line = line.replace('Ã¢', 'â')
        line = self.space.sub(' ', line).strip()
        if (len(line) < min_line_length):
            skipped_because_min_length += np.array([1, length], dtype=np.uint64)
            continue
        total_clean_length += len(line)
        output.append((line + '\n'))
    stats = {}
    stats['skipped_because_min_length'] = skipped_because_min_length
    stats['skipped_alpha_count'] = skipped_alpha_count
    stats['skipped_because_max_numeric'] = skipped_because_max_numeric
    stats['skipped_because_max_non_ascii'] = skipped_because_max_non_ascii
    stats['skipped_because_forbidden_chars'] = skipped_because_forbidden_chars
    stats['total_original_length'] = total_original_length
    stats['total_clean_length'] = total_clean_length
    return (output, stats)