<commit_before>/*

  Spec for a dataset management jquery plugin.
  The plugin will provide an object that can be queried.
  

  Formats that should be supported:
  tabular - delimited (specified delimiter)
  json    - array of objects
  dom table - A table object

  Any other types? I don't want to play with XML but if needbe I can be persuaded


  TODO:
    Interpolation 
    
*/

// Creating a new dataset:
// -----------------------

// Constructor Parameters:
{
  url : "String - url to fetch data from",
  jsonp : "boolean - true if this is a jsonp request",
  delimiter : "String - a delimiter string that is used in a tabular datafile",
  data : "Object - an actual javascript object that already contains the data",
  table : "Element - a DOM table that contains the data",
  format : "String - optional file format specification, otherwise we'll try to guess",
  recursive: "Boolean - if true build nested arrays of objects as datasets",
  strict: "Whether to expect the json in our format or whether to interpret as raw array of objects, default false",
  transform: "function to apply to JSON before internal interpretation, optional"
}

// Using a url - json:
var dataset = $.dataset({url : 'http://mydata.com/stuff.json'});

// jsonp
var dataset = $.dataset({url : 'http://mydata.com/stuff.json', jsonp: true});

// Using a url - tabular (note delimiter specified):
var dataset = $.dataset({url : 'http://mydata.com/stuff.csv', delimiter: ","}); 

// From existing object:
var dataObj = [{ id: 1, count: 12 }, { id: 2, count: 14 }],
    dataset = $.dataset({ data: dataObj });

// From dom table
var datatable = $('table#datatable'),
    dataset = $.dataset({ table: datatable });

// dataset metadata
// ---------------

// what metadata about columns do we want here?
dataset.column('GER.1.M').metadata
=> {"definition": "Gross Enrolment Ratio for male students, see http://..."}

// dataset types
// -------------

// json
// file endings supporting auto detection: .json
$.dataset.types.JSON

// tabular
// file ending supporting auto detection:
// .csv - "," delimiter unless otherwise specified
// .tsv - "\t" delimiter unless otherwise specified
// any other common formats?
$.dataset.types.TABULAR
$.dataset.types.DOMTABLE
?

// dataset delimiter regex
// TODO: add what delimiters we are going to check for by default:
// columns: , \t : | 
// rows: \n 
// anything else?

// event types:
// change - value change
// 1. On Entire Dataset
// TODO: what should the callback take here? I feel like it needs old value and new value but it also
// needs to know what the position of that was and I don't feel like it needs to also take a column and row
// what do you think alex?
dataset.change(function(??) {
  // do things here on dataset change
});

// 2. On Column
data.column(3).change(function(??) {
  
});

// 3. On Row
data.row(12).change(function(??) {
  
});

// 4. On Field
dataset.column(3).row(4).change(function(oldValue, newValue) {
  // do things here on field value change
});

// columns
// --------

// data types
$.dataset.datatypes.NUMBER
$.dataset.datatypes.STRING
$.dataset.datatypes.BOOLEAN
$.dataset.datatypes.ARRAY
$.dataset.datatypes.OBJECT
$.dataset.datatypes.DATASET
//explicit support for nested datasets could allow for a variety of smarter operations
//with a touch more magic than simple OBJECT or ARRAY would allow. Nested arrays
//can be recursively be built as datasets when passed in.

// getting all columns:
dataset.columns();

=> [{ name : "Column1", type: "number|string|boolean|array|object", position: 0 ... }, ... ] //any other types?

// getting specific column:
dataset.column(3);
dataset.column("name");

=> { name: "Column5", type: "number", position: 3 ... };

// typechecking
dataset.column(3).isNumber();
dataset.column(3).isString();
dataset.column(3).isBoolean();
dataset.column(3).isArray();
dataset.column(3).isObject();

// type setting
dataset.column(3).setType($.dataset.datatypes.NUMBER);
// This will result in the column being iterated over and converted appropriately.

// math:
// -----------------------
// Note: when calling a math function on a non numeric column, the result will be NaN.

dataset.column(3).min();

=> 3.21 

dataset.column(3).max();

=> 4.2

dataset.column(3).mean();

=> 4

dataset.column(3).mode();

// frequency table
dataset.column(3).freq();

=>[{ 
    value : 12,
    count : 100
  }, ...];

// standard deviation  
dataset.column(3).std();

=> 0.4

// what other math operations might be worth while here?

// column transform:
// -----------------

// allows one to transform a column based on the value it has.
// Note: not sure if transform is the proper function name. Alex pointed out this is just a "map" and
// it may make more sense to have a map.
// Note2: Transformation function needs to be stored on column so that inserted data can be transformed appropriatly - OR SHOULD IT?! < TODO
dataset.column(3).transform(function(value) {
  
  // this modifier accesses column. Should it access field?
  this.setType($.dataset.datatypes.BOOLEAN);

  // return value becomes the new row value
  if (value > 1) {
    return true;
  } else {
    return false;
  }
});


//Sorting - allow the dataset to be sorted by a function
dataset.sortBy(function(a,b) {
  return (a['total_cost'] > b['total_cost']);
});

//Filtering / Querying<commit_msg>transform to map for columns<commit_after>/*

  Spec for a dataset management jquery plugin.
  The plugin will provide an object that can be queried.
  

  Formats that should be supported:
  tabular - delimited (specified delimiter)
  json    - array of objects
  dom table - A table object

  Any other types? I don't want to play with XML but if needbe I can be persuaded


  TODO:
    Interpolation 
    
*/

// Creating a new dataset:
// -----------------------

// Constructor Parameters:
{
  url : "String - url to fetch data from",
  jsonp : "boolean - true if this is a jsonp request",
  delimiter : "String - a delimiter string that is used in a tabular datafile",
  data : "Object - an actual javascript object that already contains the data",
  table : "Element - a DOM table that contains the data",
  format : "String - optional file format specification, otherwise we'll try to guess",
  recursive: "Boolean - if true build nested arrays of objects as datasets",
  strict: "Whether to expect the json in our format or whether to interpret as raw array of objects, default false",
  transform: "function to apply to JSON before internal interpretation, optional"
}

// Using a url - json:
var dataset = $.dataset({url : 'http://mydata.com/stuff.json'});

// jsonp
var dataset = $.dataset({url : 'http://mydata.com/stuff.json', jsonp: true});

// Using a url - tabular (note delimiter specified):
var dataset = $.dataset({url : 'http://mydata.com/stuff.csv', delimiter: ","}); 

// From existing object:
var dataObj = [{ id: 1, count: 12 }, { id: 2, count: 14 }],
    dataset = $.dataset({ data: dataObj });

// From dom table
var datatable = $('table#datatable'),
    dataset = $.dataset({ table: datatable });

// dataset metadata
// ---------------

// what metadata about columns do we want here?
dataset.column('GER.1.M').metadata
=> {"definition": "Gross Enrolment Ratio for male students, see http://..."}

// dataset types
// -------------

// json
// file endings supporting auto detection: .json
$.dataset.types.JSON

// tabular
// file ending supporting auto detection:
// .csv - "," delimiter unless otherwise specified
// .tsv - "\t" delimiter unless otherwise specified
// any other common formats?
$.dataset.types.TABULAR
$.dataset.types.DOMTABLE
?

// dataset delimiter regex
// TODO: add what delimiters we are going to check for by default:
// columns: , \t : | 
// rows: \n 
// anything else?

// event types:
// change - value change
// 1. On Entire Dataset
// TODO: what should the callback take here? I feel like it needs old value and new value but it also
// needs to know what the position of that was and I don't feel like it needs to also take a column and row
// what do you think alex?
dataset.change(function(??) {
  // do things here on dataset change
});

// 2. On Column
data.column(3).change(function(??) {
  
});

// 3. On Row
data.row(12).change(function(??) {
  
});

// 4. On Field
dataset.column(3).row(4).change(function(oldValue, newValue) {
  // do things here on field value change
});

// columns
// --------

// data types
$.dataset.datatypes.NUMBER
$.dataset.datatypes.STRING
$.dataset.datatypes.BOOLEAN
$.dataset.datatypes.ARRAY
$.dataset.datatypes.OBJECT
$.dataset.datatypes.DATASET
//explicit support for nested datasets could allow for a variety of smarter operations
//with a touch more magic than simple OBJECT or ARRAY would allow. Nested arrays
//can be recursively be built as datasets when passed in.

// getting all columns:
dataset.columns();

=> [{ name : "Column1", type: "number|string|boolean|array|object", position: 0 ... }, ... ] //any other types?

// getting specific column:
dataset.column(3);
dataset.column("name");

=> { name: "Column5", type: "number", position: 3 ... };

// typechecking
dataset.column(3).isNumber();
dataset.column(3).isString();
dataset.column(3).isBoolean();
dataset.column(3).isArray();
dataset.column(3).isObject();

// type setting
dataset.column(3).setType($.dataset.datatypes.NUMBER);
// This will result in the column being iterated over and converted appropriately.

// math:
// -----------------------
// Note: when calling a math function on a non numeric column, the result will be NaN.

dataset.column(3).min();

=> 3.21 

dataset.column(3).max();

=> 4.2

dataset.column(3).mean();

=> 4

dataset.column(3).mode();

// frequency table
dataset.column(3).freq();

=>[{ 
    value : 12,
    count : 100
  }, ...];

// standard deviation  
dataset.column(3).std();

=> 0.4

// what other math operations might be worth while here?

// column mapping:
// -----------------

// allows one to map a column based on the value it has.
// Note2: Transformation function needs to be stored on column so that inserted data can be transformed appropriatly - OR SHOULD IT?! < TODO
dataset.column(3).map(function(value) {
  
  // this modifier accesses column. Should it access field?
  this.setType($.dataset.datatypes.BOOLEAN);

  // return value becomes the new row value
  if (value > 1) {
    return true;
  } else {
    return false;
  }
});


//Sorting - allow the dataset to be sorted by a function
dataset.sortBy(function(a,b) {
  return (a['total_cost'] > b['total_cost']);
});

//Filtering / Querying
{'fraction_non_alphanumeric': 0.14938505203405866, 'fraction_numerical': 0.009271523178807948, 'mean_word_length': 3.095699341340566, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1827696', 'n_tokens_mistral': 3178, 'n_tokens_neox': 3075, 'n_words': 1199}