<commit_before>/**
 * Copyright (C) 2014-2015 LinkedIn Corp. (<EMAIL>)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.linkedin.pinot.core.segment.creator.impl.stats;

import java.util.HashMap;
import java.util.Map;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.linkedin.pinot.common.data.FieldSpec;
import com.linkedin.pinot.common.data.Schema;
import com.linkedin.pinot.core.data.GenericRow;
import com.linkedin.pinot.core.segment.creator.AbstractColumnStatisticsCollector;
import com.linkedin.pinot.core.segment.creator.SegmentPreIndexStatsCollector;


/**
 * Nov 7, 2014
 */

public class SegmentPreIndexStatsCollectorImpl implements SegmentPreIndexStatsCollector {
  private static final Logger LOGGER = LoggerFactory.getLogger(SegmentPreIndexStatsCollectorImpl.class);

  private final Schema dataSchema;
  Map<String, AbstractColumnStatisticsCollector> columnStatsCollectorMap;

  public SegmentPreIndexStatsCollectorImpl(Schema dataSchema) {
    this.dataSchema = dataSchema;
  }

  @Override
  public void init() {
    columnStatsCollectorMap = new HashMap<String, AbstractColumnStatisticsCollector>();

    for (final FieldSpec spec : dataSchema.getAllFieldSpecs()) {
      switch (spec.getDataType()) {
        case BOOLEAN:
        case STRING:
          columnStatsCollectorMap.put(spec.getName(), new StringColumnPreIndexStatsCollector(spec));
          break;
        case INT:
          columnStatsCollectorMap.put(spec.getName(), new IntColumnPreIndexStatsCollector(spec));
          break;
        case LONG:
          columnStatsCollectorMap.put(spec.getName(), new LongColumnPreIndexStatsCollector(spec));
          break;
        case FLOAT:
          columnStatsCollectorMap.put(spec.getName(), new FloatColumnPreIndexStatsCollector(spec));
          break;
        case DOUBLE:
          columnStatsCollectorMap.put(spec.getName(), new DoubleColumnPreIndexStatsCollector(spec));
          break;
        default:
          break;
      }
    }
  }

  @Override
  public void build() throws Exception {
    for (final String column : columnStatsCollectorMap.keySet()) {
      columnStatsCollectorMap.get(column).seal();
    }
  }

  @Override
  public AbstractColumnStatisticsCollector getColumnProfileFor(String column) throws Exception {
    return columnStatsCollectorMap.get(column);
  }

  @Override
  public void collectRow(GenericRow row) throws Exception {
    for (final String column : row.getFieldNames()) {
      columnStatsCollectorMap.get(column).collect(row.getValue(column));
    }
  }

  public static <T> T convertInstanceOfObject(Object o, Class<T> clazz) {
    try {
      return clazz.cast(o);
    } catch (final ClassCastException e) {
      LOGGER.warn("Caught exception while converting instance", e);
      return null;
    }
  }

  @Override
  public void logStats() {
    try {
      for (final String column : columnStatsCollectorMap.keySet()) {
        LOGGER.info("********** logging for column : " + column + " ********************* ");
        LOGGER.info("min value : " + columnStatsCollectorMap.get(column).getMinValue());
        LOGGER.info("max value : " + columnStatsCollectorMap.get(column).getMaxValue());
        LOGGER.info("cardinality : " + columnStatsCollectorMap.get(column).getCardinality());
        LOGGER.info("length of largest column : " + columnStatsCollectorMap.get(column).getLengthOfLargestElement());
        LOGGER.info("is sorted : " + columnStatsCollectorMap.get(column).isSorted());
        LOGGER.info("column type : " + dataSchema.getFieldSpecFor(column).getDataType());
        LOGGER.info("***********************************************");
      }
    } catch (final Exception e) {
      LOGGER.error("Caught exception while logging column stats", e);
    }

  }
}
<commit_msg>For TimeSpec, we will use outgoing column to init the SegmentPreIndexStatsCollectorImpl<commit_after>/**
 * Copyright (C) 2014-2015 LinkedIn Corp. (<EMAIL>)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.linkedin.pinot.core.segment.creator.impl.stats;

import java.util.HashMap;
import java.util.Map;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.linkedin.pinot.common.data.FieldSpec;
import com.linkedin.pinot.common.data.FieldSpec.DataType;
import com.linkedin.pinot.common.data.Schema;
import com.linkedin.pinot.common.data.TimeFieldSpec;
import com.linkedin.pinot.core.data.GenericRow;
import com.linkedin.pinot.core.segment.creator.AbstractColumnStatisticsCollector;
import com.linkedin.pinot.core.segment.creator.SegmentPreIndexStatsCollector;


/**
 * Nov 7, 2014
 */

public class SegmentPreIndexStatsCollectorImpl implements SegmentPreIndexStatsCollector {
  private static final Logger LOGGER = LoggerFactory.getLogger(SegmentPreIndexStatsCollectorImpl.class);

  private final Schema dataSchema;
  Map<String, AbstractColumnStatisticsCollector> columnStatsCollectorMap;

  public SegmentPreIndexStatsCollectorImpl(Schema dataSchema) {
    this.dataSchema = dataSchema;
  }

  @Override
  public void init() {
    columnStatsCollectorMap = new HashMap<String, AbstractColumnStatisticsCollector>();

    for (final FieldSpec spec : dataSchema.getAllFieldSpecs()) {
      DataType dataType = spec.getDataType();
      String specName = spec.getName();
      if (spec instanceof TimeFieldSpec) {
        TimeFieldSpec timeFieldSpec = (TimeFieldSpec) spec;
        specName = timeFieldSpec.getOutGoingTimeColumnName();
        dataType = timeFieldSpec.getOutgoingGranularitySpec().getDataType();
      }
      switch (dataType) {
        case BOOLEAN:
        case STRING:
          columnStatsCollectorMap.put(specName, new StringColumnPreIndexStatsCollector(spec));
          break;
        case INT:
          columnStatsCollectorMap.put(specName, new IntColumnPreIndexStatsCollector(spec));
          break;
        case LONG:
          columnStatsCollectorMap.put(specName, new LongColumnPreIndexStatsCollector(spec));
          break;
        case FLOAT:
          columnStatsCollectorMap.put(specName, new FloatColumnPreIndexStatsCollector(spec));
          break;
        case DOUBLE:
          columnStatsCollectorMap.put(specName, new DoubleColumnPreIndexStatsCollector(spec));
          break;
        default:
          LOGGER.warn("Failed to initialize FieldSpec: {}", spec);
          break;
      }
    }
  }

  @Override
  public void build() throws Exception {
    for (final String column : columnStatsCollectorMap.keySet()) {
      columnStatsCollectorMap.get(column).seal();
    }
  }

  @Override
  public AbstractColumnStatisticsCollector getColumnProfileFor(String column) throws Exception {
    return columnStatsCollectorMap.get(column);
  }

  @Override
  public void collectRow(GenericRow row) throws Exception {
    for (final String column : row.getFieldNames()) {
      columnStatsCollectorMap.get(column).collect(row.getValue(column));
    }
  }

  public static <T> T convertInstanceOfObject(Object o, Class<T> clazz) {
    try {
      return clazz.cast(o);
    } catch (final ClassCastException e) {
      LOGGER.warn("Caught exception while converting instance", e);
      return null;
    }
  }

  @Override
  public void logStats() {
    try {
      for (final String column : columnStatsCollectorMap.keySet()) {
        LOGGER.info("********** logging for column : " + column + " ********************* ");
        LOGGER.info("min value : " + columnStatsCollectorMap.get(column).getMinValue());
        LOGGER.info("max value : " + columnStatsCollectorMap.get(column).getMaxValue());
        LOGGER.info("cardinality : " + columnStatsCollectorMap.get(column).getCardinality());
        LOGGER.info("length of largest column : " + columnStatsCollectorMap.get(column).getLengthOfLargestElement());
        LOGGER.info("is sorted : " + columnStatsCollectorMap.get(column).isSorted());
        LOGGER.info("column type : " + dataSchema.getFieldSpecFor(column).getDataType());
        LOGGER.info("***********************************************");
      }
    } catch (final Exception e) {
      LOGGER.error("Caught exception while logging column stats", e);
    }

  }
}

{'fraction_non_alphanumeric': 0.11419551491399957, 'fraction_numerical': 0.004136729806226867, 'mean_word_length': 3.762571280456195, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3147121', 'n_tokens_mistral': 2587, 'n_tokens_neox': 2558, 'n_words': 603}