<commit_before>import edu.cmu.lti.jawjaw.pobj.POS;
import edu.cmu.lti.jawjaw.util.WordNetUtil;
import edu.cmu.lti.lexical_db.ILexicalDatabase;
import edu.cmu.lti.lexical_db.NictWordNet;
import edu.cmu.lti.lexical_db.data.Concept;
import edu.cmu.lti.ws4j.Relatedness;
import edu.cmu.lti.ws4j.RelatednessCalculator;
import edu.cmu.lti.ws4j.impl.HirstStOnge;
import edu.cmu.lti.ws4j.impl.Lin;
import edu.cmu.lti.ws4j.impl.Path;
import edu.cmu.lti.ws4j.impl.WuPalmer;
import edu.cmu.lti.ws4j.util.PorterStemmer;
import edu.cmu.lti.ws4j.util.WS4JConfiguration;
import org.apache.commons.lang3.StringUtils;

import java.util.Arrays;
import java.util.List;
import java.util.StringTokenizer;
import java.util.stream.Collector;

class NLPUtils {
    private static ILexicalDatabase db = new NictWordNet();

    static double checkStrings(String oldArg, String newArg) {
        Pair<double[]> stringVectors = stringsToVectors(oldArg, newArg);

        // Use cosineSimilarity to analyse these vectors
        double[] one = stringVectors.getOne();
        double[] two = stringVectors.getTwo();

        if (allZero(one) || allZero(two)) return 0;

        return cosineSimilarity(one, two);
    }

    private static boolean allZero(double[] arr) {
        return Arrays.stream(arr).allMatch(value -> value == 0);
    }

    private static String removeStopWordsAndStem(String string) {
        StringBuilder result = new StringBuilder();
        StringTokenizer st = new StringTokenizer(string);

        PorterStemmer ps = new PorterStemmer();

        while (st.hasMoreTokens()) {
            String next = st.nextToken();
            if (!isStopWord(next)) {

                result.append(next);
                result.append(' ');<commit_msg>Debugging a heroku only problem to do with the lib/*.jar dependencies so I need to push<commit_after>import edu.cmu.lti.jawjaw.pobj.POS;
import edu.cmu.lti.lexical_db.NictWordNet;
import edu.cmu.lti.lexical_db.data.Concept;
import edu.cmu.lti.ws4j.RelatednessCalculator;
import edu.cmu.lti.ws4j.impl.Lin;
import edu.cmu.lti.ws4j.impl.Path;
import edu.cmu.lti.ws4j.impl.WuPalmer;
import edu.cmu.lti.ws4j.util.PorterStemmer;
import edu.cmu.lti.ws4j.util.WS4JConfiguration;

import java.util.Arrays;
import java.util.List;
import java.util.StringTokenizer;

class NLPUtils {
    private static NictWordNet db = new NictWordNet();

    static double checkStrings(String oldArg, String newArg) {
        Pair<double[]> stringVectors = stringsToVectors(oldArg, newArg);

        // Use cosineSimilarity to analyse these vectors
        double[] one = stringVectors.getOne();
        double[] two = stringVectors.getTwo();

        if (allZero(one) || allZero(two)) return 0;

        return cosineSimilarity(one, two);
    }

    private static boolean allZero(double[] arr) {
        return Arrays.stream(arr).allMatch(value -> value == 0);
    }

    private static String removeStopWordsAndStem(String string) {
        StringBuilder result = new StringBuilder();
        StringTokenizer st = new StringTokenizer(string);

        PorterStemmer ps = new PorterStemmer();

        while (st.hasMoreTokens()) {
            String next = st.nextToken();
            if (!isStopWord(next)) {

                result.append(next);
                result.append(' ');
{'fraction_non_alphanumeric': 0.11400848999393572, 'fraction_numerical': 0.006367495451788963, 'mean_word_length': 3.301173402868318, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4199008', 'n_tokens_mistral': 1161, 'n_tokens_neox': 1123, 'n_words': 221}