<commit_before>/*
 * Copyright (C) 2014 Indeed Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the
 * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
 * express or implied. See the License for the specific language governing permissions and
 * limitations under the License.
 */
 package com.indeed.imhotep.web;

import com.google.common.base.Strings;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.indeed.util.core.io.Closeables2;
import com.indeed.imhotep.DatasetInfo;
import com.indeed.imhotep.client.ImhotepClient;
import com.indeed.imhotep.metadata.DatasetMetadata;
import com.indeed.imhotep.metadata.FieldMetadata;
import com.indeed.imhotep.metadata.FieldType;
import com.indeed.imhotep.metadata.MetricMetadata;
import org.apache.log4j.Logger;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.type.TypeReference;
import org.springframework.beans.factory.xml.XmlBeanFactory;
import org.springframework.core.io.FileSystemResource;
import org.springframework.scheduling.annotation.Scheduled;

import javax.annotation.Nonnull;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.*;
import java.util.regex.Pattern;

/**
 * @author vladimir
 */

public class ImhotepMetadataCache {
    private static final Logger log = Logger.getLogger(ImhotepMetadataCache.class);

    private LinkedHashMap<String, DatasetMetadata> datasets = Maps.newLinkedHashMap();
    // TODO: integrate into the metadata above?
    private volatile Map<String, Set<String>> datasetToKeywordAnaylzerWhitelist = Maps.newHashMap();
    private final ImhotepClient imhotepClient;
    private String ramsesMetadataPath;
    private final List<Pattern> disabledFields = Lists.newArrayList();


    public ImhotepMetadataCache(ImhotepClient client, String ramsesMetadataPath, String disabledFields) {
        imhotepClient = client;
        this.ramsesMetadataPath = ramsesMetadataPath;
        if(disabledFields != null) {
            for(String field : disabledFields.split(",")) {
                try {
                    this.disabledFields.add(Pattern.compile(field.trim()));
                } catch (Exception e) {
                    log.warn("Failed to compile regex pattern for disabled field: " + field);
                }
            }
        }
    }

    // updated every 60s and actual shards in ImhotepClient are reloaded every 60s
    @Scheduled(fixedRate = 60000)
    public void updateDatasets() {
        Map<String, DatasetInfo> datasetToShardList = imhotepClient.getDatasetToShardList();
        List<String> datasetNames = new ArrayList<String>(datasetToShardList.keySet());
        Collections.sort(datasetNames);

        if(datasetNames.size() == 0) {   // if we get no data, just keep what we already have
            log.warn("Imhotep returns no datasets");
            return;
        }

        // First make empty DatasetMetadata instances
        final LinkedHashMap<String, DatasetMetadata> newDatasets = Maps.newLinkedHashMap();
        for(String datasetName : datasetNames) {
            final DatasetMetadata datasetMetadata = new DatasetMetadata(datasetName);
            newDatasets.put(datasetName, datasetMetadata);
        }

        // Now pre-fill the metadata with fields from Imhotep
        for(DatasetInfo datasetInfo : datasetToShardList.values()) {
            List<String> dsIntFields = Lists.newArrayList(datasetInfo.getIntFields());
            List<String> dsStringFields = Lists.newArrayList(datasetInfo.getStringFields());
            removeDisabledFields(dsIntFields);
            removeDisabledFields(dsStringFields);
            Collections.sort(dsIntFields);
            Collections.sort(dsStringFields);

            final String datasetName = datasetInfo.getDataset();
            final DatasetMetadata datasetMetadata = newDatasets.get(datasetName);
            final LinkedHashMap<String, FieldMetadata> fieldMetadatas = datasetMetadata.getFields();

            for(String intField : dsIntFields) {
                fieldMetadatas.put(intField, new FieldMetadata(intField, FieldType.Integer));
            }
            for(String stringField : dsStringFields) {
                fieldMetadatas.put(stringField, new FieldMetadata(stringField, FieldType.String));
            }
        }

        // now load the metadata from files
        loadMetadataFromFiles(newDatasets);
        for(final DatasetMetadata datasetMetadata : newDatasets.values()) {
            addStandardAliases(datasetMetadata);

            datasetMetadata.finishLoading();
        }<commit_msg>IQL-102: Prefer Int fields to String fields when the field type is ambiguous<commit_after>/*
 * Copyright (C) 2014 Indeed Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the
 * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
 * express or implied. See the License for the specific language governing permissions and
 * limitations under the License.
 */
 package com.indeed.imhotep.web;

import com.google.common.base.Strings;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.indeed.util.core.io.Closeables2;
import com.indeed.imhotep.DatasetInfo;
import com.indeed.imhotep.client.ImhotepClient;
import com.indeed.imhotep.metadata.DatasetMetadata;
import com.indeed.imhotep.metadata.FieldMetadata;
import com.indeed.imhotep.metadata.FieldType;
import com.indeed.imhotep.metadata.MetricMetadata;
import org.apache.log4j.Logger;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.type.TypeReference;
import org.springframework.beans.factory.xml.XmlBeanFactory;
import org.springframework.core.io.FileSystemResource;
import org.springframework.scheduling.annotation.Scheduled;

import javax.annotation.Nonnull;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.*;
import java.util.regex.Pattern;

/**
 * @author vladimir
 */

public class ImhotepMetadataCache {
    private static final Logger log = Logger.getLogger(ImhotepMetadataCache.class);

    private LinkedHashMap<String, DatasetMetadata> datasets = Maps.newLinkedHashMap();
    // TODO: integrate into the metadata above?
    private volatile Map<String, Set<String>> datasetToKeywordAnaylzerWhitelist = Maps.newHashMap();
    private final ImhotepClient imhotepClient;
    private String ramsesMetadataPath;
    private final List<Pattern> disabledFields = Lists.newArrayList();


    public ImhotepMetadataCache(ImhotepClient client, String ramsesMetadataPath, String disabledFields) {
        imhotepClient = client;
        this.ramsesMetadataPath = ramsesMetadataPath;
        if(disabledFields != null) {
            for(String field : disabledFields.split(",")) {
                try {
                    this.disabledFields.add(Pattern.compile(field.trim()));
                } catch (Exception e) {
                    log.warn("Failed to compile regex pattern for disabled field: " + field);
                }
            }
        }
    }

    // updated every 60s and actual shards in ImhotepClient are reloaded every 60s
    @Scheduled(fixedRate = 60000)
    public void updateDatasets() {
        Map<String, DatasetInfo> datasetToShardList = imhotepClient.getDatasetToShardList();
        List<String> datasetNames = new ArrayList<String>(datasetToShardList.keySet());
        Collections.sort(datasetNames);

        if(datasetNames.size() == 0) {   // if we get no data, just keep what we already have
            log.warn("Imhotep returns no datasets");
            return;
        }

        // First make empty DatasetMetadata instances
        final LinkedHashMap<String, DatasetMetadata> newDatasets = Maps.newLinkedHashMap();
        for(String datasetName : datasetNames) {
            final DatasetMetadata datasetMetadata = new DatasetMetadata(datasetName);
            newDatasets.put(datasetName, datasetMetadata);
        }

        // Now pre-fill the metadata with fields from Imhotep
        for(DatasetInfo datasetInfo : datasetToShardList.values()) {
            List<String> dsIntFields = Lists.newArrayList(datasetInfo.getIntFields());
            List<String> dsStringFields = Lists.newArrayList(datasetInfo.getStringFields());
            removeDisabledFields(dsIntFields);
            removeDisabledFields(dsStringFields);
            Collections.sort(dsIntFields);
            Collections.sort(dsStringFields);

            final String datasetName = datasetInfo.getDataset();
            final DatasetMetadata datasetMetadata = newDatasets.get(datasetName);
            final LinkedHashMap<String, FieldMetadata> fieldMetadatas = datasetMetadata.getFields();

            for(String stringField : dsStringFields) {
                fieldMetadatas.put(stringField, new FieldMetadata(stringField, FieldType.String));
            }

            for(String intField : dsIntFields) {
                fieldMetadatas.put(intField, new FieldMetadata(intField, FieldType.Integer));
            }
        }

        // now load the metadata from files
        loadMetadataFromFiles(newDatasets);
        for(final DatasetMetadata datasetMetadata : newDatasets.values()) {
            addStandardAliases(datasetMetadata);

            datasetMetadata.finishLoading();
        }
{'fraction_non_alphanumeric': 0.08782473827064753, 'fraction_numerical': 0.004168282279953471, 'mean_word_length': 3.358681875792142, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5904769', 'n_tokens_mistral': 2744, 'n_tokens_neox': 2692, 'n_words': 682}