<commit_before>
import * as utils from "base/utils";
import Class from "base/class";

function _getQueryId(query, path, lastModified) {
  return utils.hashCode([
    query.select.key,
    query.where,
    query.join,
    query.dataset,
    query.version,
    query.language,
    path,
    lastModified
  ]);
}

export class Storage {
  constructor() {
    this.queryIds = {};
    this.queries = {};
    this._collection = {};
    this._collectionPromises = {}; // stores promises, making sure we don't do one calulation twice
  }

  /**
   * Loads resource from reader
   * @param {Array} query Array with queries to be loaded
   * @param {Object} parsers An object with concepts as key and parsers as value
   * @param {Object} readerObject for this query
   */
  loadFromReader(query, parsers, readerObject) {
    const _this = this;
    const queryMergeId = _getQueryId(query, readerObject._basepath, readerObject._lastModified);
<commit_msg>Add from to datastorage hash for availability queries<commit_after>
import * as utils from "base/utils";
import Class from "base/class";

function _getQueryId(query, path, lastModified) {
  return utils.hashCode([
    query.select.key,
    query.where,
    query.from,
    query.join,
    query.dataset,
    query.version,
    query.language,
    path,
    lastModified
  ]);
}

export class Storage {
  constructor() {
    this.queryIds = {};
    this.queries = {};
    this._collection = {};
    this._collectionPromises = {}; // stores promises, making sure we don't do one calulation twice
  }

  /**
   * Loads resource from reader
   * @param {Array} query Array with queries to be loaded
   * @param {Object} parsers An object with concepts as key and parsers as value
   * @param {Object} readerObject for this query
   */
  loadFromReader(query, parsers, readerObject) {
    const _this = this;
    const queryMergeId = _getQueryId(query, readerObject._basepath, readerObject._lastModified);

{'fraction_non_alphanumeric': 0.10380267214799589, 'fraction_numerical': 0.0, 'mean_word_length': 3.3654708520179373, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1842864', 'n_tokens_mistral': 578, 'n_tokens_neox': 571, 'n_words': 178}