<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.carbondata.integration.spark.load;

import java.nio.charset.Charset;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;

import org.carbondata.core.carbon.metadata.datatype.DataType;
import org.carbondata.core.constants.CarbonCommonConstants;
import org.carbondata.core.util.ByteUtil;
import org.carbondata.core.util.CarbonProperties;

/**
 * Dictionary sort model class holds the member byte value and corresponding key value.
 */
public class CarbonDictionarySortModel implements Comparable<CarbonDictionarySortModel> {

  /**
   * Charset const
   */
  public static final Charset CHARSET_CONST =
      Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET);
  /**
   * Surrogate key
   */
  private int key;

  /**
   * member value in bytes
   */
  private String memberValue;

  /**
   * member dataType
   */
  private DataType dataType;

  /**
   * Constructor to init the dictionary sort model
   *
   * @param key
   * @param dataType
   * @param memberValue
   */
  public CarbonDictionarySortModel(int key, DataType dataType, String memberValue) {
    this.key = key;
    this.dataType = dataType;
    this.memberValue = memberValue;
  }

  /**
   * Compare
   */
  @Override public int compareTo(CarbonDictionarySortModel o) {
    switch (dataType) {
      case INT:
      case LONG:
      case DOUBLE:

        Double d1 = null;
        Double d2 = null;
        try {
          d1 = new Double(memberValue);
        } catch (NumberFormatException e) {
          return 1;
        }
        try {
          d2 = new Double(o.memberValue);
        } catch (NumberFormatException e) {
          return -1;
        }
        return d1.compareTo(d2);
      case DECIMAL:
        java.math.BigDecimal val1 = null;
        java.math.BigDecimal val2 = null;
        try {
          val1 = new java.math.BigDecimal(memberValue);
        } catch (NumberFormatException e) {
          return 1;
        }
        try {
          val2 = new java.math.BigDecimal(o.memberValue);
        } catch (NumberFormatException e) {
          return -1;
        }
        return val1.compareTo(val2);
      case TIMESTAMP:
        SimpleDateFormat parser = new SimpleDateFormat(CarbonProperties.getInstance()
            .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
                CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT));
        Date date1 = null;
        Date date2 = null;
        try {
          date1 = parser.parse(memberValue);
        } catch (ParseException e) {
          return 1;
        }
        try {
          date2 = parser.parse(o.memberValue);
        } catch (ParseException e) {
          return -1;
        }
        return date1.compareTo(date2);
      case STRING:
      default:
        return ByteUtil.UnsafeComparer.INSTANCE.compareTo(this.memberValue.getBytes(CHARSET_CONST),
            o.memberValue.getBytes(CHARSET_CONST));
    }
  }

  /**
   * @see Object#hashCode()
   */
  @Override public int hashCode() {
    int result = 1;
    result = result + ((memberValue == null) ? 0 : memberValue.hashCode());
    return result;
  }

  /**
   * @see Object#equals(Object)
   */
  @Override public boolean equals(Object obj) {
    if (obj instanceof CarbonDictionarySortModel) {
      if (this == obj) {
        return true;
      }
      CarbonDictionarySortModel other = (CarbonDictionarySortModel) obj;
      if (memberValue == null) {
        if (other.memberValue != null) {
          return false;
        }
      } else if (!ByteUtil.UnsafeComparer.INSTANCE.equals(this.memberValue.getBytes(CHARSET_CONST),
          other.memberValue.getBytes(CHARSET_CONST))) {
        return false;
      }
      return true;
    } else {
      return false;
    }
  }

  /**
   * return the surrogate of the member
   *
   * @return
   */
  public int getKey() {
    return key;
  }

  /**
   * Returns member buye
   *
   * @return
   */
  public String getMemberValue() {
    return memberValue;
  }

}<commit_msg>[Issue - 271] code review fix of PR 272<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.carbondata.integration.spark.load;

import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;

import org.carbondata.core.carbon.metadata.datatype.DataType;
import org.carbondata.core.constants.CarbonCommonConstants;
import org.carbondata.core.util.CarbonProperties;

/**
 * Dictionary sort model class holds the member byte value and corresponding key value.
 */
public class CarbonDictionarySortModel implements Comparable<CarbonDictionarySortModel> {

  /**
   * Surrogate key
   */
  private int key;

  /**
   * member value in bytes
   */
  private String memberValue;

  /**
   * member dataType
   */
  private DataType dataType;

  /**
   * Constructor to init the dictionary sort model
   *
   * @param key
   * @param dataType
   * @param memberValue
   */
  public CarbonDictionarySortModel(int key, DataType dataType, String memberValue) {
    this.key = key;
    this.dataType = dataType;
    this.memberValue = memberValue;
  }

  /**
   * Compare
   */
  @Override public int compareTo(CarbonDictionarySortModel o) {
    switch (dataType) {
      case INT:
      case LONG:
      case DOUBLE:

        Double d1 = null;
        Double d2 = null;
        try {
          d1 = new Double(memberValue);
        } catch (NumberFormatException e) {
          return 1;
        }
        try {
          d2 = new Double(o.memberValue);
        } catch (NumberFormatException e) {
          return -1;
        }
        return d1.compareTo(d2);
      case DECIMAL:
        java.math.BigDecimal val1 = null;
        java.math.BigDecimal val2 = null;
        try {
          val1 = new java.math.BigDecimal(memberValue);
        } catch (NumberFormatException e) {
          return 1;
        }
        try {
          val2 = new java.math.BigDecimal(o.memberValue);
        } catch (NumberFormatException e) {
          return -1;
        }
        return val1.compareTo(val2);
      case TIMESTAMP:
        SimpleDateFormat parser = new SimpleDateFormat(CarbonProperties.getInstance()
            .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
                CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT));
        Date date1 = null;
        Date date2 = null;
        try {
          date1 = parser.parse(memberValue);
        } catch (ParseException e) {
          return 1;
        }
        try {
          date2 = parser.parse(o.memberValue);
        } catch (ParseException e) {
          return -1;
        }
        return date1.compareTo(date2);
      case STRING:
      default:
        return this.memberValue.compareTo(o.memberValue);
    }
  }

  /**
   * @see Object#hashCode()
   */
  @Override public int hashCode() {
    int result = ((memberValue == null) ? 0 : memberValue.hashCode());
    return result;
  }

  /**
   * @see Object#equals(Object)
   */
  @Override public boolean equals(Object obj) {
    if (obj instanceof CarbonDictionarySortModel) {
      if (this == obj) {
        return true;
      }
      CarbonDictionarySortModel other = (CarbonDictionarySortModel) obj;
      if (memberValue == null) {
        if (other.memberValue != null) {
          return false;
        }
      } else if (!this.memberValue.equals(other.memberValue)) {
        return false;
      }
      return true;
    } else {
      return false;
    }
  }

  /**
   * return the surrogate of the member
   *
   * @return
   */
  public int getKey() {
    return key;
  }

  /**
   * Returns member buye
   *
   * @return
   */
  public String getMemberValue() {
    return memberValue;
  }

}
{'fraction_non_alphanumeric': 0.09216341489705723, 'fraction_numerical': 0.007006575401530667, 'mean_word_length': 2.4010263929618767, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '145801', 'n_tokens_mistral': 2685, 'n_tokens_neox': 2626, 'n_words': 756}