<commit_before>package plagiarism;

import plagiarism.filter.CharacterFilter;
import plagiarism.filter.IrrelevantWordsFilter;
import plagiarism.filter.WordSeparatorFilter;

import java.io.*;
import java.util.HashMap;
import java.util.Map;

/**
 * Plagiarism builds a filter chain to read and filter an input text.
 * <p>
 * The filtered data will be stored in this object and the object provides a method to calculate a simple checksum of the input text.
 * <p>
 * This checksum gives information, if the input text might be a plagiarism or not.
 *
 * @author <NAME>
 */
public class Plagiarism {

    private Map<String, Integer> map;
    private File file;

    public Plagiarism(File file) throws IOException {
        this.file = file;
        this.map = new HashMap<>();
        init();
    }

    private void init() throws IOException {

        BufferedReader bufferedReader = new BufferedReader(new FileReader(file));
        FilterReader filterChain = new IrrelevantWordsFilter(new WordSeparatorFilter(new CharacterFilter(bufferedReader)));

        char[] buffer = new char[256];
        int readChars;

        while ((readChars = filterChain.read(buffer)) != -1) {
            if (readChars != 0) {
                String s = new String(buffer, 0, readChars);
                map.compute(s, (k, v) -> (v == null) ? 1 : ++v);
            }
        }
    }

    public int checksum() {
        return map.keySet()
                .stream()
                .mapToInt(s -> digitSum(s) * map.get(s))
                .sum();
    }

    private int digitSum(String s) {
        return s.chars().sum();
    }

    @Override
    public String toString() {
        return map.toString();
    }
}<commit_msg>rename vars<commit_after>package plagiarism;

import plagiarism.filter.CharacterFilter;
import plagiarism.filter.IrrelevantWordsFilter;
import plagiarism.filter.WordSeparatorFilter;

import java.io.*;
import java.util.HashMap;
import java.util.Map;

/**
 * Plagiarism builds a filter chain to read and filter an input text.
 * <p>
 * The filtered data will be stored in this object and the object provides a method to calculate a simple checksum of the input text.
 * <p>
 * This checksum gives information, if the input text might be a plagiarism or not.
 *
 * @author <NAME>
 */
public class Plagiarism {

    private Map<String, Integer> map;
    private File file;

    public Plagiarism(File file) throws IOException {
        this.file = file;
        this.map = new HashMap<>();
        init();
    }

    private void init() throws IOException {

        BufferedReader bufferedReader = new BufferedReader(new FileReader(file));
        FilterReader filterChain = new IrrelevantWordsFilter(new WordSeparatorFilter(new CharacterFilter(bufferedReader)));

        char[] buffer = new char[256];
        int readChars;

        while ((readChars = filterChain.read(buffer)) != -1) {
            if (readChars != 0) {
                String word = new String(buffer, 0, readChars);
                map.compute(word, (k, v) -> (v == null) ? 1 : ++v);
            }
        }
    }

    public int checksum() {
        return map.keySet()
                .stream()
                .mapToInt(word -> digitSum(word) * map.get(word))
                .sum();
    }

    private int digitSum(String word) {
        return word.chars().sum();
    }

    @Override
    public String toString() {
        return map.toString();
    }
}
{'fraction_non_alphanumeric': 0.10642628671125327, 'fraction_numerical': 0.004070950857807502, 'mean_word_length': 2.460764587525151, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3493199', 'n_tokens_mistral': 1012, 'n_tokens_neox': 951, 'n_words': 307}