<commit_before>/*
 * typeahead.js
 * https://github.com/twitter/typeahead
 * Copyright 2013 Twitter, Inc. and other contributors; Licensed MIT
 */

var Dataset = (function() {

  function Dataset(o) {
    utils.bindAll(this);

    this.storage = new PersistentStorage(o.name);
    this.adjacencyList = {};
    this.itemHash = {};

    this.name = o.name;
    this.resetDataOnProtocolSwitch = o.resetDataOnProtocolSwitch || false;
    this.queryUrl = o.remote;
    this.transport = o.transport;
    this.limit = o.limit || 10;
    this._customMatcher = o.matcher || null;
    this._customRanker = o.ranker || null;
    this._ttl_ms = o.ttl_ms || 3 * 24 * 60 * 60 * 1000; // 3 days;

    this.keys = {
      version: 'version',
      protocol: 'protocol',
      itemHash: 'itemHash',
      adjacencyList: 'adjacencyList'
    };

    o.local && this._processLocalData(o.local);
    o.prefetch && this._loadPrefetchData(o.prefetch);
  }

  utils.mixin(Dataset.prototype, {

    // private methods
    // ---------------

    _processLocalData: function(data) {
      data && this._mergeProcessedData(this._processData(data));
    },

    _loadPrefetchData: function(url) {
      var that = this,
          itemHash = this.storage.get(this.keys.itemHash),
          adjacencyList = this.storage.get(this.keys.adjacencyList),
          protocol = this.storage.get(this.keys.protocol),
          version = this.storage.get(this.keys.version),
          isExpired = version !== VERSION || protocol !== utils.getProtocol();

      // data was available in local storage, use it
      if (itemHash && adjacencyList && !isExpired) {
        this._mergeProcessedData({
          itemHash: itemHash,
          adjacencyList: adjacencyList
        });
      }

      else {
        $.getJSON(url).done(processPrefetchData);
      }

      function processPrefetchData(data) {
        var processedData = that._processData(data),
            itemHash = processedData.itemHash,
            adjacencyList = processedData.adjacencyList;

        // store process data in local storage
        // this saves us from processing the data on every page load
        that.storage.set(that.keys.itemHash, itemHash, that._ttl_ms);
        that.storage.set(that.keys.adjacencyList, adjacencyList, that._ttl_ms);
        that.storage.set(that.keys.version, VERSION, that._ttl_ms);
        that.storage.set(that.keys.protocol, utils.getProtocol(), that._ttl_ms);

        that._mergeProcessedData(processedData);
      }
    },

    _processData: function(data) {
      var itemHash = {}, adjacencyList = {};

      utils.each(data, function(i, item) {
        var id;

        // convert string datums to datum objects
        if (utils.isString(item)) {
          item = { value: item, tokens: utils.tokenizeText(item) };
        }

        // normalize tokens
        item.tokens = utils.map(item.tokens || [], function(token) {
          return token.toLowerCase();
        });

        itemHash[id = utils.getUniqueId(item.value)] = item;

        utils.each(item.tokens, function(i, token) {
          var char = token.charAt(0),
              adjacency = adjacencyList[char] || (adjacencyList[char] = [id]);

          !~utils.indexOf(adjacency, id) && adjacency.push(id);
        });
      });

      return { itemHash: itemHash, adjacencyList: adjacencyList };
    },

    _mergeProcessedData: function(processedData) {
      var that = this;

      // merge item hash
      utils.mixin(this.itemHash, processedData.itemHash);

      // merge adjacency list
      utils.each(processedData.adjacencyList, function(char, adjacency) {
        var masterAdjacency = that.adjacencyList[char];

        that.adjacencyList[char] = masterAdjacency ?
          masterAdjacency.concat(adjacency) : adjacency;
      });
    },

    _getPotentiallyMatchingIds: function(terms) {
      var potentiallyMatchingIds = [];
      var lists = [];
      utils.map(terms, utils.bind(function(term) {
        var list = this.adjacencyList[term.charAt(0)];
        if (!list) { return; }
        lists.push(list);
      },this));
      if (lists.length === 1) {
        return lists[0];
      }
      var listLengths = [];
      $.each(lists, function(i, list) {
        listLengths.push(list.length);
      });<commit_msg>Google Closure Compiler doesn't like 'char'<commit_after>/*
 * typeahead.js
 * https://github.com/twitter/typeahead
 * Copyright 2013 Twitter, Inc. and other contributors; Licensed MIT
 */

var Dataset = (function() {

  function Dataset(o) {
    utils.bindAll(this);

    this.storage = new PersistentStorage(o.name);
    this.adjacencyList = {};
    this.itemHash = {};

    this.name = o.name;
    this.resetDataOnProtocolSwitch = o.resetDataOnProtocolSwitch || false;
    this.queryUrl = o.remote;
    this.transport = o.transport;
    this.limit = o.limit || 10;
    this._customMatcher = o.matcher || null;
    this._customRanker = o.ranker || null;
    this._ttl_ms = o.ttl_ms || 3 * 24 * 60 * 60 * 1000; // 3 days;

    this.keys = {
      version: 'version',
      protocol: 'protocol',
      itemHash: 'itemHash',
      adjacencyList: 'adjacencyList'
    };

    o.local && this._processLocalData(o.local);
    o.prefetch && this._loadPrefetchData(o.prefetch);
  }

  utils.mixin(Dataset.prototype, {

    // private methods
    // ---------------

    _processLocalData: function(data) {
      data && this._mergeProcessedData(this._processData(data));
    },

    _loadPrefetchData: function(url) {
      var that = this,
          itemHash = this.storage.get(this.keys.itemHash),
          adjacencyList = this.storage.get(this.keys.adjacencyList),
          protocol = this.storage.get(this.keys.protocol),
          version = this.storage.get(this.keys.version),
          isExpired = version !== VERSION || protocol !== utils.getProtocol();

      // data was available in local storage, use it
      if (itemHash && adjacencyList && !isExpired) {
        this._mergeProcessedData({
          itemHash: itemHash,
          adjacencyList: adjacencyList
        });
      }

      else {
        $.getJSON(url).done(processPrefetchData);
      }

      function processPrefetchData(data) {
        var processedData = that._processData(data),
            itemHash = processedData.itemHash,
            adjacencyList = processedData.adjacencyList;

        // store process data in local storage
        // this saves us from processing the data on every page load
        that.storage.set(that.keys.itemHash, itemHash, that._ttl_ms);
        that.storage.set(that.keys.adjacencyList, adjacencyList, that._ttl_ms);
        that.storage.set(that.keys.version, VERSION, that._ttl_ms);
        that.storage.set(that.keys.protocol, utils.getProtocol(), that._ttl_ms);

        that._mergeProcessedData(processedData);
      }
    },

    _processData: function(data) {
      var itemHash = {}, adjacencyList = {};

      utils.each(data, function(i, item) {
        var id;

        // convert string datums to datum objects
        if (utils.isString(item)) {
          item = { value: item, tokens: utils.tokenizeText(item) };
        }

        // normalize tokens
        item.tokens = utils.map(item.tokens || [], function(token) {
          return token.toLowerCase();
        });

        itemHash[id = utils.getUniqueId(item.value)] = item;

        utils.each(item.tokens, function(i, token) {
          var character = token.charAt(0),
              adjacency = adjacencyList[character] || (adjacencyList[character] = [id]);

          !~utils.indexOf(adjacency, id) && adjacency.push(id);
        });
      });

      return { itemHash: itemHash, adjacencyList: adjacencyList };
    },

    _mergeProcessedData: function(processedData) {
      var that = this;

      // merge item hash
      utils.mixin(this.itemHash, processedData.itemHash);

      // merge adjacency list
      utils.each(processedData.adjacencyList, function(character, adjacency) {
        var masterAdjacency = that.adjacencyList[character];

        that.adjacencyList[character] = masterAdjacency ?
          masterAdjacency.concat(adjacency) : adjacency;
      });
    },

    _getPotentiallyMatchingIds: function(terms) {
      var potentiallyMatchingIds = [];
      var lists = [];
      utils.map(terms, utils.bind(function(term) {
        var list = this.adjacencyList[term.charAt(0)];
        if (!list) { return; }
        lists.push(list);
      },this));
      if (lists.length === 1) {
        return lists[0];
      }
      var listLengths = [];
      $.each(lists, function(i, list) {
        listLengths.push(list.length);
      });
{'fraction_non_alphanumeric': 0.13117606123869172, 'fraction_numerical': 0.005103224309904894, 'mean_word_length': 2.4547275641025643, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2049827', 'n_tokens_mistral': 2707, 'n_tokens_neox': 2552, 'n_words': 499}