<commit_before>/*
 * Copyright 2014 Rackspace
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */
package com.rackspacecloud.blueflood.CloudFilesBackfiller.rollup.handlers;

import com.codahale.metrics.Counter;
import com.codahale.metrics.Meter;
import com.google.gson.Gson;
import com.rackspacecloud.blueflood.CloudFilesBackfiller.download.CloudFilesManager;
import com.rackspacecloud.blueflood.CloudFilesBackfiller.exceptions.OutOFBandException;
import com.rackspacecloud.blueflood.CloudFilesBackfiller.gson.CheckFromJson;
import com.rackspacecloud.blueflood.CloudFilesBackfiller.gson.MetricPoint;
import com.rackspacecloud.blueflood.CloudFilesBackfiller.service.BackFillerConfig;
import com.rackspacecloud.blueflood.CloudFilesBackfiller.service.OutOFBandRollup;
import com.rackspacecloud.blueflood.rollup.Granularity;
import com.rackspacecloud.blueflood.service.Configuration;
import com.rackspacecloud.blueflood.types.*;
import com.rackspacecloud.blueflood.utils.Metrics;
import com.rackspacecloud.blueflood.utils.Util;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentSkipListMap;

public class BuildStore {
    private static final BuildStore buildStore = new BuildStore();
    private static final Logger log = LoggerFactory.getLogger(BuildStore.class);
    // number of metrics coming in for ranges which we have already rolled up that can be tolerated
    private static final int OUT_OF_RANGE_TOLERATION_THRESHOLD = 10;
    // please getEligibleData for better understanding of what is this parameter
    private static final int RANGE_BUFFER = Configuration.getInstance().getIntegerProperty(BackFillerConfig.NUMBER_OF_BUFFERRED_SLOTS);
    private static int outOfRangeToleration = 0;
    // Specifies the sorting order in TreeMap
    private static Comparator<Range> rangeComparator = new Comparator<Range>() {
        @Override public int compare(Range r1, Range r2) {
            return (int) (r1.getStart() - r2.getStart());
        }
    };
    /*
     * This is a concurrent version of TreeMap.
     * Reason behind using a TreeMap : We want to keep the mapping from ranges to metrics sorted
     * Reason for using concurrent version : We do not want to block on simultaneous read(happens in RollupGenerator), write(happens here). Both these operations should not overlap<commit_msg>Fix the typo in config<commit_after>/*
 * Copyright 2014 Rackspace
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */
package com.rackspacecloud.blueflood.CloudFilesBackfiller.rollup.handlers;

import com.codahale.metrics.Counter;
import com.codahale.metrics.Meter;
import com.google.gson.Gson;
import com.rackspacecloud.blueflood.CloudFilesBackfiller.download.CloudFilesManager;
import com.rackspacecloud.blueflood.CloudFilesBackfiller.exceptions.OutOFBandException;
import com.rackspacecloud.blueflood.CloudFilesBackfiller.gson.CheckFromJson;
import com.rackspacecloud.blueflood.CloudFilesBackfiller.gson.MetricPoint;
import com.rackspacecloud.blueflood.CloudFilesBackfiller.service.BackFillerConfig;
import com.rackspacecloud.blueflood.CloudFilesBackfiller.service.OutOFBandRollup;
import com.rackspacecloud.blueflood.rollup.Granularity;
import com.rackspacecloud.blueflood.service.Configuration;
import com.rackspacecloud.blueflood.types.*;
import com.rackspacecloud.blueflood.utils.Metrics;
import com.rackspacecloud.blueflood.utils.Util;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentSkipListMap;

public class BuildStore {
    private static final BuildStore buildStore = new BuildStore();
    private static final Logger log = LoggerFactory.getLogger(BuildStore.class);
    // number of metrics coming in for ranges which we have already rolled up that can be tolerated
    private static final int OUT_OF_RANGE_TOLERATION_THRESHOLD = 10;
    // please getEligibleData for better understanding of what is this parameter
    private static final int RANGE_BUFFER = Configuration.getInstance().getIntegerProperty(BackFillerConfig.NUMBER_OF_BUFFERED_SLOTS);
    private static int outOfRangeToleration = 0;
    // Specifies the sorting order in TreeMap
    private static Comparator<Range> rangeComparator = new Comparator<Range>() {
        @Override public int compare(Range r1, Range r2) {
            return (int) (r1.getStart() - r2.getStart());
        }
    };
    /*
     * This is a concurrent version of TreeMap.
     * Reason behind using a TreeMap : We want to keep the mapping from ranges to metrics sorted
     * Reason for using concurrent version : We do not want to block on simultaneous read(happens in RollupGenerator), write(happens here). Both these operations should not overlap
{'fraction_non_alphanumeric': 0.0734481637959051, 'fraction_numerical': 0.0055248618784530384, 'mean_word_length': 4.631290027447393, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7065608', 'n_tokens_mistral': 1768, 'n_tokens_neox': 1697, 'n_words': 523}