<commit_before>package tech.tablesaw.api;

import com.google.common.base.CharMatcher;
import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import it.unimi.dsi.fastutil.ints.Int2IntMap;
import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap;
import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.ints.IntArrays;
import it.unimi.dsi.fastutil.ints.IntComparator;
import it.unimi.dsi.fastutil.ints.IntListIterator;
import tech.tablesaw.columns.AbstractColumn;
import tech.tablesaw.columns.CategoryColumnUtils;
import tech.tablesaw.columns.Column;
import tech.tablesaw.filtering.StringBiPredicate;
import tech.tablesaw.filtering.StringPredicate;
import tech.tablesaw.filtering.text.CategoryFilters;
import tech.tablesaw.io.TypeUtils;
import tech.tablesaw.store.ColumnMetadata;
import tech.tablesaw.util.BitmapBackedSelection;
import tech.tablesaw.util.DictionaryMap;
import tech.tablesaw.util.Selection;

import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;

/**
 * A column that contains String values. They are assumed to be 'categorical' rather than free-form text, so are
 * stored in an encoding that takes advantage of the expected repetition of string values.
 * <p>
 * Because the MISSING_VALUE for this column type is an empty string, there is little or no need for special handling
 * of missing values in this class's methods.
 */
public class CategoryColumn extends AbstractColumn
        implements CategoryFilters, CategoryColumnUtils, Iterable<String> {

    public static final String MISSING_VALUE = (String) ColumnType.CATEGORY.getMissingValue();
    private static final int BYTE_SIZE = 4;
    private static int DEFAULT_ARRAY_SIZE = 128;

    private int id = 0;

    // holds a key for each row in the table. the key can be used to lookup the backing string value
    private IntArrayList values;

    // a bidirectional map of keys to backing string values.
    private DictionaryMap lookupTable = new DictionaryMap();

    public final IntComparator rowComparator = new IntComparator() {

        @Override
        public int compare(int i, int i1) {
            String f1 = get(i);
            String f2 = get(i1);
            return f1.compareTo(f2);
        }

        @Override
        public int compare(Integer i, Integer i1) {
            return compare((int) i, (int) i1);
        }
    };

    private IntComparator dictionarySortComparator = new IntComparator() {
        @Override
        public int compare(int i, int i1) {
            return lookupTable.get(i).compareTo(lookupTable.get(i1));
        }

        @Override
        public int compare(Integer o1, Integer o2) {
            return compare((int) o1, (int) o2);
        }
    };

    private IntComparator reverseDictionarySortComparator = new IntComparator() {
        @Override
        public int compare(int i, int i1) {
            return -lookupTable.get(i).compareTo(lookupTable.get(i1));
        }

        @Override
        public int compare(Integer o1, Integer o2) {
            return compare((int) o1, (int) o2);
        }
    };

    public CategoryColumn(String name) {
        super(name);
        values = new IntArrayList(DEFAULT_ARRAY_SIZE);
    }

    public CategoryColumn(String name, List<String> categories) {
        super(name);
        values = new IntArrayList(categories.size());
        for (String string : categories) {
            add(string);
        }
    }

    public CategoryColumn(String name, String[] categories) {
      super(name);
      values = new IntArrayList(categories.length);
      for (String string : categories) {
          add(string);
      }
    }

    public CategoryColumn(ColumnMetadata metadata) {
        super(metadata);
        values = new IntArrayList(DEFAULT_ARRAY_SIZE);
    }

    public CategoryColumn(String name, int size) {
        super(name);
        values = new IntArrayList(size);
    }

    public static String convert(String stringValue) {
        if (Strings.isNullOrEmpty(stringValue) || TypeUtils.MISSING_INDICATORS.contains(stringValue)) {
            return MISSING_VALUE;
        }
        return stringValue;
    }

    @Override
    public ColumnType type() {
        return ColumnType.CATEGORY;
    }

    @Override
    public String getString(int row) {
        return get(row);
    }

    @Override
    public CategoryColumn emptyCopy() {
        CategoryColumn copy = new CategoryColumn(name());
        copy.setComment(comment());
        return copy;
    }

    @Override
    public CategoryColumn emptyCopy(int rowSize) {
        CategoryColumn copy = new CategoryColumn(name(), rowSize);
        copy.setComment(comment());
        return copy;
    }

    @Override
    public void sortAscending() {
        IntArrays.parallelQuickSort(values.elements(), dictionarySortComparator);
    }

    @Override
    public void sortDescending() {
        IntArrays.parallelQuickSort(values.elements(), reverseDictionarySortComparator);
    }

    /**
     * Returns the number of elements (a.k.a. rows or cells) in the column
     */
    @Override
    public int size() {
        return values.size();
    }

    /**
     * Returns the value at rowIndex in this column. The index is zero-based.
     *
     * @throws IndexOutOfBoundsException if the given rowIndex is not in the column
     */
    public String get(int rowIndex) {
        int k = values.getInt(rowIndex);
        return lookupTable.get(k);
    }

    public List<String> toList() {
        return Lists.newArrayList(dictionaryMap().categoryArray());
    }

    @Override
    public Table summary() {
        return countByCategory();
    }
<commit_msg>fix for issue 158<commit_after>package tech.tablesaw.api;

import com.google.common.base.CharMatcher;
import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
import com.google.common.base.Strings;
import it.unimi.dsi.fastutil.ints.Int2IntMap;
import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap;
import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.ints.IntArrays;
import it.unimi.dsi.fastutil.ints.IntComparator;
import it.unimi.dsi.fastutil.ints.IntListIterator;
import tech.tablesaw.columns.AbstractColumn;
import tech.tablesaw.columns.CategoryColumnUtils;
import tech.tablesaw.columns.Column;
import tech.tablesaw.filtering.StringBiPredicate;
import tech.tablesaw.filtering.StringPredicate;
import tech.tablesaw.filtering.text.CategoryFilters;
import tech.tablesaw.io.TypeUtils;
import tech.tablesaw.store.ColumnMetadata;
import tech.tablesaw.util.BitmapBackedSelection;
import tech.tablesaw.util.DictionaryMap;
import tech.tablesaw.util.Selection;

import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;

/**
 * A column that contains String values. They are assumed to be 'categorical' rather than free-form text, so are
 * stored in an encoding that takes advantage of the expected repetition of string values.
 * <p>
 * Because the MISSING_VALUE for this column type is an empty string, there is little or no need for special handling
 * of missing values in this class's methods.
 */
public class CategoryColumn extends AbstractColumn
        implements CategoryFilters, CategoryColumnUtils, Iterable<String> {

    public static final String MISSING_VALUE = (String) ColumnType.CATEGORY.getMissingValue();
    private static final int BYTE_SIZE = 4;
    private static int DEFAULT_ARRAY_SIZE = 128;

    private int id = 0;

    // holds a key for each row in the table. the key can be used to lookup the backing string value
    private IntArrayList values;

    // a bidirectional map of keys to backing string values.
    private DictionaryMap lookupTable = new DictionaryMap();

    public final IntComparator rowComparator = new IntComparator() {

        @Override
        public int compare(int i, int i1) {
            String f1 = get(i);
            String f2 = get(i1);
            return f1.compareTo(f2);
        }

        @Override
        public int compare(Integer i, Integer i1) {
            return compare((int) i, (int) i1);
        }
    };

    private IntComparator dictionarySortComparator = new IntComparator() {
        @Override
        public int compare(int i, int i1) {
            return lookupTable.get(i).compareTo(lookupTable.get(i1));
        }

        @Override
        public int compare(Integer o1, Integer o2) {
            return compare((int) o1, (int) o2);
        }
    };

    private IntComparator reverseDictionarySortComparator = new IntComparator() {
        @Override
        public int compare(int i, int i1) {
            return -lookupTable.get(i).compareTo(lookupTable.get(i1));
        }

        @Override
        public int compare(Integer o1, Integer o2) {
            return compare((int) o1, (int) o2);
        }
    };

    public CategoryColumn(String name) {
        super(name);
        values = new IntArrayList(DEFAULT_ARRAY_SIZE);
    }

    public CategoryColumn(String name, List<String> categories) {
        super(name);
        values = new IntArrayList(categories.size());
        for (String string : categories) {
            add(string);
        }
    }

    public CategoryColumn(String name, String[] categories) {
      super(name);
      values = new IntArrayList(categories.length);
      for (String string : categories) {
          add(string);
      }
    }

    public CategoryColumn(ColumnMetadata metadata) {
        super(metadata);
        values = new IntArrayList(DEFAULT_ARRAY_SIZE);
    }

    public CategoryColumn(String name, int size) {
        super(name);
        values = new IntArrayList(size);
    }

    public static String convert(String stringValue) {
        if (Strings.isNullOrEmpty(stringValue) || TypeUtils.MISSING_INDICATORS.contains(stringValue)) {
            return MISSING_VALUE;
        }
        return stringValue;
    }

    @Override
    public ColumnType type() {
        return ColumnType.CATEGORY;
    }

    @Override
    public String getString(int row) {
        return get(row);
    }

    @Override
    public CategoryColumn emptyCopy() {
        CategoryColumn copy = new CategoryColumn(name());
        copy.setComment(comment());
        return copy;
    }

    @Override
    public CategoryColumn emptyCopy(int rowSize) {
        CategoryColumn copy = new CategoryColumn(name(), rowSize);
        copy.setComment(comment());
        return copy;
    }

    @Override
    public void sortAscending() {
        IntArrays.parallelQuickSort(values.elements(), dictionarySortComparator);
    }

    @Override
    public void sortDescending() {
        IntArrays.parallelQuickSort(values.elements(), reverseDictionarySortComparator);
    }

    /**
     * Returns the number of elements (a.k.a. rows or cells) in the column
     */
    @Override
    public int size() {
        return values.size();
    }

    /**
     * Returns the value at rowIndex in this column. The index is zero-based.
     *
     * @throws IndexOutOfBoundsException if the given rowIndex is not in the column
     */
    public String get(int rowIndex) {
        int k = values.getInt(rowIndex);
        return lookupTable.get(k);
    }

    /**
     * Returns a List<String> representation of all the values in this column
     *
     * NOTE: Unless you really need a string consider using the column itself for large datasets as it uses much less memory
     */
    public List<String> toList() {
        List<String> strings = new ArrayList<>();
        for(String category : this) {
            strings.add(category);
        }
        return strings;
    }

    @Override
    public Table summary() {
        return countByCategory();
    }

{'fraction_non_alphanumeric': 0.09202655084815209, 'fraction_numerical': 0.004834876669671392, 'mean_word_length': 2.864471184293857, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5625414', 'n_tokens_mistral': 3416, 'n_tokens_neox': 3376, 'n_words': 945}