<commit_before>this.recline = this.recline || {};
this.recline.Backend = this.recline.Backend || {};

(function($, my) {
  // ## createDataset
  //
  // Convenience function to create a simple 'in-memory' dataset in one step.
  //
  // @param data: list of hashes for each document/row in the data ({key:
  // value, key: value})
  // @param fields: (optional) list of field hashes (each hash defining a hash
  // as per recline.Model.Field). If fields not specified they will be taken
  // from the data.
  // @param metadata: (optional) dataset metadata - see recline.Model.Dataset.
  // If not defined (or id not provided) id will be autogenerated.
  my.createDataset = function(data, fields, metadata) {
    if (!metadata) {
      metadata = {};
    }
    if (!metadata.id) {
      metadata.id = String(Math.floor(Math.random() * 100000000) + 1);
    }
    var backend = new recline.Backend.Memory();
    var datasetInfo = {
      documents: data,
      metadata: metadata
    };
    if (fields) {
      datasetInfo.fields = fields;
    } else {
      if (data) {
        datasetInfo.fields = _.map(data[0], function(value, key) {
          return {id: key};
        });
      }
    }
    backend.addDataset(datasetInfo);
    var dataset = new recline.Model.Dataset({id: metadata.id}, backend);
    dataset.fetch();
    return dataset;
  };


  // ## Memory Backend - uses in-memory data
  //
  // To use it you should provide in your constructor data:
  // 
  //   * metadata (including fields array)
  //   * documents: list of hashes, each hash being one doc. A doc *must* have an id attribute which is unique.
  //<commit_msg>[backend/memory][xs]: createDataset calls query on dataset object so that the Dataset object already has data in it.<commit_after>this.recline = this.recline || {};
this.recline.Backend = this.recline.Backend || {};

(function($, my) {
  // ## createDataset
  //
  // Convenience function to create a simple 'in-memory' dataset in one step.
  //
  // @param data: list of hashes for each document/row in the data ({key:
  // value, key: value})
  // @param fields: (optional) list of field hashes (each hash defining a hash
  // as per recline.Model.Field). If fields not specified they will be taken
  // from the data.
  // @param metadata: (optional) dataset metadata - see recline.Model.Dataset.
  // If not defined (or id not provided) id will be autogenerated.
  my.createDataset = function(data, fields, metadata) {
    if (!metadata) {
      metadata = {};
    }
    if (!metadata.id) {
      metadata.id = String(Math.floor(Math.random() * 100000000) + 1);
    }
    var backend = new recline.Backend.Memory();
    var datasetInfo = {
      documents: data,
      metadata: metadata
    };
    if (fields) {
      datasetInfo.fields = fields;
    } else {
      if (data) {
        datasetInfo.fields = _.map(data[0], function(value, key) {
          return {id: key};
        });
      }
    }
    backend.addDataset(datasetInfo);
    var dataset = new recline.Model.Dataset({id: metadata.id}, backend);
    dataset.fetch();
    dataset.query();
    return dataset;
  };


  // ## Memory Backend - uses in-memory data
  //
  // To use it you should provide in your constructor data:
  // 
  //   * metadata (including fields array)
  //   * documents: list of hashes, each hash being one doc. A doc *must* have an id attribute which is unique.
  //
{'fraction_non_alphanumeric': 0.13060141509433962, 'fraction_numerical': 0.006485849056603774, 'mean_word_length': 2.560335781741868, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1033221', 'n_tokens_mistral': 1043, 'n_tokens_neox': 1016, 'n_words': 341}