<commit_before>package com.dexvis.dex

import com.dexvis.dex.exception.DexException
import com.dexvis.util.DateUtil


/**
 * 
 * This is the main structure to hold Dex Data.
 * 
 * @author <NAME>
 * 
 */
public class DexData {
  // Header and data:
  def List<String>       header
  def List<List<String>> data
  def List<String> types;
  /**
   * 
   * Construct an empty instance of DexData.
   * 
   */
  public DexData()
  {
    this(new ArrayList<String>())
  }
  
  /**
   * 
   * Given a header, construct a DexData structure with no data.
   * 
   * @param header
   *          The initial header.
   * 
   */
  def DexData(List<String> header)
  {
    this(header, new ArrayList<List<String>>())
  }
  
  /**
   *
   * Given a header and some data, construct a DexData structure to match.
   *
   * @param header The header.
   * @param data The data.
   *
   */
  def DexData(List<String> header, List<List<String>> data)
  {
    this(header, data, new ArrayList<List<String>>())
  }
  
  /**
   * 
   * Given a header, some data, and a list of types, construct a DexData structure to match.
   * 
   * @param header The header.
   * @param data The data.
   * @param types The type of each header.
   * 
   */
  def DexData(List<String> header, List<List<String>> data, List<String> types)
  {
    this.header = header.collect()
    this.data = new ArrayList<List<String>>()
    this.types = new ArrayList<List<String>>()
    
    data.each { row ->
      this.data << row.collect()
    }
    
    types.each { row ->
      this.types << row
    }
  }
  
  /**
   *
   * Given a header and some data, construct a DexData structure to match.
   *
   * @param header The header.
   * @param data The data.
   *
   */
  def DexData(DexData dex)
  {
    this(dex.header, dex.data)
  }
  
  /**
   * 
   * Return an alternate structure map where each column is contained in the
   * key, and the value of the map is the list of row entries for that column.
   * 
   * @return A map where column names are the keys into the map and the values
   *         are the rows for that each column name.
   * 
   */
  public Map<String, List<String>> getColumnMap()
  {
    Map<String, List<String>> columnMap = new HashMap<String, List<String>>()
    int numRows = data.size()
    int numCols = header.size()
    for (String colName : header)
    {
      columnMap.put(colName, new ArrayList<String>())
    }
    
    for (int rowNum = 0; rowNum < numRows; rowNum++)
    {
      for (int colNum = 0; colNum < numCols; colNum++)
      {
        try
        {
          columnMap.get(header.get(colNum)).add(data.get(rowNum).get(colNum))
        }
        catch (Exception ex)
        {
          throw new DexException("Accessing: COLNAME: ${header.get(colNum)}, COL=$colNum, ROW: $rowNum, HEADER=$header")
        }
      }
    }
    
    return columnMap
  }
  
  public List<String> getUniques(int colIndex) {
    return getUniques(getColumnName(colIndex))
  }
  
  public List<String> getUniques(String colName) {    
    def valueMap = [:]
    getColumn(colName).each { valueMap[it] = true }
    return valueMap.keySet().sort()
  }
  
  public List<Integer> categorize(String colName) {
    return categorize(getColumnNumber(colName))
  }
  
  public List<Integer> categorize(int colIndex) {
    def uniques = getUniques(colIndex);
    def catMap = [:]
    
    uniques.eachWithIndex {
      cat, i ->
      catMap[cat] = i;
    }
    
    def categories = new ArrayList<Integer>()
    data.eachWithIndex {
      row, ri ->
      categories.add(catMap[row[colIndex]])
    }
    
    return categories;
  }
  
  public List<Map<String, String>> getMapList()
  {
    List<Map<String, String>> list = new ArrayList<Map<String, String>>();
    
    data.eachWithIndex { row, ri ->
      
      Map<String, String> rowMap = new HashMap<String, String>();
      
      header.eachWithIndex { hdr, hi ->
        rowMap[hdr] = row[hi]
      }
      
      list << rowMap
    }
    
    return list;
  }
  
  /**<commit_msg>bugfix<commit_after>package com.dexvis.dex

import com.dexvis.dex.exception.DexException
import com.dexvis.util.DateUtil


/**
 * 
 * This is the main structure to hold Dex Data.
 * 
 * @author <NAME>
 * 
 */
public class DexData {
  // Header and data:
  def List<String>       header
  def List<List<String>> data
  def List<String> types;
  /**
   * 
   * Construct an empty instance of DexData.
   * 
   */
  public DexData()
  {
    this(new ArrayList<String>())
  }
  
  /**
   * 
   * Given a header, construct a DexData structure with no data.
   * 
   * @param header
   *          The initial header.
   * 
   */
  def DexData(List<String> header)
  {
    this(header, new ArrayList<List<String>>())
  }
  
  /**
   *
   * Given a header and some data, construct a DexData structure to match.
   *
   * @param header The header.
   * @param data The data.
   *
   */
  def DexData(List<String> header, List<List<String>> data)
  {
    this(header, data, new ArrayList<List<String>>())
  }
  
  /**
   * 
   * Given a header, some data, and a list of types, construct a DexData structure to match.
   * 
   * @param header The header.
   * @param data The data.
   * @param types The type of each header.
   * 
   */
  def DexData(List<String> header, List<List<String>> data, List<String> types)
  {
    this.header = header.collect()
    this.data = new ArrayList<List<String>>()
    this.types = new ArrayList<List<String>>()
    
    data.each { row ->
      this.data << row.collect()
    }
    
    types.each { row ->
      this.types << row
    }
  }
  
  /**
   *
   * Given a header and some data, construct a DexData structure to match.
   *
   * @param header The header.
   * @param data The data.
   *
   */
  def DexData(DexData dex)
  {
    this(dex.header, dex.data)
  }
  
  /**
   * 
   * Return an alternate structure map where each column is contained in the
   * key, and the value of the map is the list of row entries for that column.
   * 
   * @return A map where column names are the keys into the map and the values
   *         are the rows for that each column name.
   * 
   */
  public Map<String, List<String>> getColumnMap()
  {
    Map<String, List<String>> columnMap = new HashMap<String, List<String>>()
    int numRows = data.size()
    int numCols = header.size()
    for (String colName : header)
    {
      columnMap.put(colName, new ArrayList<String>())
    }
    
    for (int rowNum = 0; rowNum < numRows; rowNum++)
    {
      for (int colNum = 0; colNum < numCols; colNum++)
      {
        try
        {
          columnMap.get(header.get(colNum)).add(data.get(rowNum).get(colNum))
        }
        catch (Exception ex)
        {
          throw new DexException("Accessing: COLNAME: ${header.get(colNum)}, COL=$colNum, ROW: $rowNum, HEADER=$header")
        }
      }
    }
    
    return columnMap
  }
  
  public List<String> getUniques(int colIndex) {
    return getUniques(getColumnName(colIndex))
  }
  
  public List<String> getUniques(String colName) {    
    def valueMap = [:]
    getColumn(colName).each { valueMap[it] = true }
    return valueMap.keySet().sort()
  }
  
  public List<Integer> categorize(String colName) {
    return categorize(getColumnNumber(colName))
  }
  
  public List<Integer> categorize(int colIndex) {
    def uniques = getUniques(colIndex);
    def catMap = [:]
    
    uniques.eachWithIndex {
      cat, i ->
      catMap[cat] = i;
    }

    def categories = new ArrayList<Integer>()
    getColumn(colIndex).each { categories.add(catMap[it]) }

    return categories;
  }
  
  public List<Map<String, String>> getMapList()
  {
    List<Map<String, String>> list = new ArrayList<Map<String, String>>();
    
    data.eachWithIndex { row, ri ->
      
      Map<String, String> rowMap = new HashMap<String, String>();
      
      header.eachWithIndex { hdr, hi ->
        rowMap[hdr] = row[hi]
      }
      
      list << rowMap
    }
    
    return list;
  }
  
  /**
{'fraction_non_alphanumeric': 0.12282458995868285, 'fraction_numerical': 0.0005008138224615, 'mean_word_length': 2.302190988011575, 'pattern_counts': {'":': 0, '<': 103, '<?xml version=': 0, '>': 98, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4420069', 'n_tokens_mistral': 2671, 'n_tokens_neox': 2601, 'n_words': 698}