<commit_before>/*
* Copyright [2016] [<NAME> (<EMAIL>)]
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*    http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
 */
package Utilities.TextModels;

import Utilities.Enumerations.RepresentationModel;
import Utilities.Enumerations.SimilarityMetric;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;

/**
 *
 * @author <NAME>
 */
public class TokenNGramsWithGlobalWeights extends TokenNGrams {

    private static final Logger LOGGER = Logger.getLogger(TokenNGramsWithGlobalWeights.class.getName());

    protected final static Map<String, IncrementalCounter>[] DOC_FREQ = new HashMap[2];

    public TokenNGramsWithGlobalWeights(int did, int n, RepresentationModel model, SimilarityMetric simMetric, String iName) {
        super(did, n, model, simMetric, iName);
    }

    @Override
    public void finalizeModel() {
        if (DOC_FREQ[datasetId] == null) {
            DOC_FREQ[datasetId] = new HashMap<>();
        }

        for (String keyValue : itemsFrequency.keySet()) {
            IncrementalCounter frequency = DOC_FREQ[datasetId].get(keyValue);
            if (frequency == null) {
                frequency = new IncrementalCounter();
                DOC_FREQ[datasetId].put(keyValue, frequency);
            }
            frequency.incrementCounter();
        }
    }

    protected double getARCSSimilarity(TokenNGramsWithGlobalWeights oModel) {
        Set<String> commonKeys = new HashSet(itemsFrequency.keySet());
        commonKeys.retainAll(oModel.getItemsFrequency().keySet());

        double similarity = 0;
        if (datasetId == 0 && datasetId == oModel.getDatasetId()) { // Dirty ER
            for (String key : commonKeys) {
                double frequency = DOC_FREQ[0].get(key).getCounter();
                similarity += 1.0 / (Math.log1p(frequency * (frequency - 1) / 2.0) / Math.log(2));
            }
        } else if (datasetId != oModel.getDatasetId()) { // Clean-Clean ER
            for (String key : commonKeys) {
                similarity += 1.0 / (Math.log1p(DOC_FREQ[0].get(key).getCounter() * DOC_FREQ[1].get(key).getCounter()) / Math.log(2));
            }
        } else {
            LOGGER.log(Level.SEVERE, "Both models come from dataset 1!");
            System.exit(-1);
        }

        return similarity;
    }

    protected double getIdfWeight(String keyValue) {
        IncrementalCounter frequency = DOC_FREQ[datasetId].get(keyValue);
        if (frequency == null) {
            return 0;
        }

        double weight = -0;
        if (NO_OF_DOCUMENTS[datasetId] < frequency.getCounter()) {<commit_msg>Updated ARCS similarity.<commit_after>/*
* Copyright [2016] [<NAME> (<EMAIL>)]
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*    http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
 */
package Utilities.TextModels;

import Utilities.Constants;
import Utilities.Enumerations.RepresentationModel;
import Utilities.Enumerations.SimilarityMetric;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;

/**
 *
 * @author <NAME>
 */
public class TokenNGramsWithGlobalWeights extends TokenNGrams implements Constants {

    private static final Logger LOGGER = Logger.getLogger(TokenNGramsWithGlobalWeights.class.getName());

    protected final static Map<String, IncrementalCounter>[] DOC_FREQ = new HashMap[2];

    public TokenNGramsWithGlobalWeights(int did, int n, RepresentationModel model, SimilarityMetric simMetric, String iName) {
        super(did, n, model, simMetric, iName);
    }

    @Override
    public void finalizeModel() {
        if (DOC_FREQ[datasetId] == null) {
            DOC_FREQ[datasetId] = new HashMap<>();
        }

        for (String keyValue : itemsFrequency.keySet()) {
            IncrementalCounter frequency = DOC_FREQ[datasetId].get(keyValue);
            if (frequency == null) {
                frequency = new IncrementalCounter();
                DOC_FREQ[datasetId].put(keyValue, frequency);
            }
            frequency.incrementCounter();
        }
    }

    protected double getARCSSimilarity(TokenNGramsWithGlobalWeights oModel) {
        Set<String> commonKeys = new HashSet(itemsFrequency.keySet());
        commonKeys.retainAll(oModel.getItemsFrequency().keySet());

        double similarity = 0;
        if (datasetId == DATASET_1 && datasetId == oModel.getDatasetId()) { // Dirty ER
            for (String key : commonKeys) {
                double frequency = DOC_FREQ[DATASET_1].get(key).getCounter();
                similarity += 1.0 / (Math.log1p(frequency * (frequency - 1) / 2.0) / Math.log(2));
            }
        } else if (datasetId != oModel.getDatasetId()) { // Clean-Clean ER
            for (String key : commonKeys) {
                similarity += 1.0 / (Math.log1p(DOC_FREQ[DATASET_1].get(key).getCounter() * DOC_FREQ[DATASET_2].get(key).getCounter()) / Math.log(2));
            }
        } else {
            LOGGER.log(Level.SEVERE, "Both models come from dataset 1!");
            System.exit(-1);
        }

        return similarity;
    }

    protected double getIdfWeight(String keyValue) {
        IncrementalCounter frequency = DOC_FREQ[datasetId].get(keyValue);
        if (frequency == null) {
            return 0;
        }

        double weight = -0;
        if (NO_OF_DOCUMENTS[datasetId] < frequency.getCounter()) {
{'fraction_non_alphanumeric': 0.11020345252774352, 'fraction_numerical': 0.008939580764488286, 'mean_word_length': 2.892621475704859, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3021240', 'n_tokens_mistral': 1906, 'n_tokens_neox': 1831, 'n_words': 492}