<commit_before>module Indexing {
	declare var DEBUG;
  declare var api;

  type Id = string;
  type ArrayFact = any[];
  interface MapFact {[field:string]: any};
  type PayloadChange = [Id, Id[], ArrayFact[], ArrayFact[]];
  interface Payload { changes: PayloadChange[] };

  export function arraysIdentical(a:any[], b:any[]):boolean {
    var i = a.length;
    if (!b || i != b.length) return false;
    while (i--) {
      if(a[i] && a[i].constructor === Array) {
        if(!arraysIdentical(a[i], b[i])) return false;
        continue;
      }
      if(a[i] && a[i].relation && b[i] && b[i].relation) {
        if(!arraysIdentical(a[i].relation, b[i].relation)) return false;
        continue;
      }
      if(a[i] && a[i].eid && b[i] && b[i].eid) { continue; }
      if (a[i] !== b[i]) return false;
    }
    return true;
  }
  
  export function objectsIdentical(a:any, b:any):boolean {
    if(typeof a !== typeof b) { return false; }
    if(typeof a !== "object") { return a === b; }
    var aKeys = Object.keys(a);
    if(!arraysIdentical(aKeys, Object.keys(b))) { return false; }
    
    for(var key of aKeys) {
      if(typeof a[key] !== typeof b[key]) { return false; }
      if(typeof a[key] !== "object" && a[key] !== b[key]) { return false; }
      else if(!objectsIdentical(a[key], b[key])) { return false; }
    }
    
    return true;
  }
  
  export function clone<T>(item:T): T;
  export function clone(item:Object): Object;
  export function clone(item:any[]): any[];
  export function clone(item:any): any {
    if (!item) { return item; }
    var result;

    if(item instanceof Array) {
      result = [];
      item.forEach(function(child, index, array) {
        result[index] = clone( child );
      });
    } else if(typeof item == "object") {
      result = {};
      for (var i in item) {
        result[i] = clone( item[i] );
      }
    } else {
      //it's a primitive
      result = item;
    }
    return result;
  }
  
  export function zip(rows, keys) {
    var keysLength = keys.length;
    var zipped = [];
    for (var row of rows) {
      var zippedRow = {};
      for(var keyIx = 0; keyIx < keysLength; keyIx++) {
        zippedRow[keys[keyIx]] = row[keyIx];
      }
      zipped.push(zippedRow);
    }
    return zipped;
  }
  
  export function unzip(zipped, keys?) {
    if(!zipped || !zipped.length) { return {keys: [], rows: []}; }
    if(!keys) {
      keys = Object.keys(zipped[0]);
    }
    var rows = [];
    for(var zippedRow of zipped) {
      var row = [];
      for(var key of keys) {
        row.push(zippedRow[key]);
      }
      rows.push(row);
    }
    return {keys: keys, rows: rows};
  }
  
  type Extractor = (fact:ArrayFact) => MapFact;
  function generateExtractorFn(view:Id, keys:Id[]):Extractor {
    return <Extractor> new Function("fact", `return { ${keys.map(function(key, ix) {
      return `"${key}": fact["${ix}"]`;
    }).join(", ")} };`);
  }
  
  type Packer = {(fact:MapFact): ArrayFact; fields: Id[]};
  function generatePackerFn(view:Id, keys:Id[]):Packer {
    var packer = <Packer> new Function("fact", `return [${keys.map(function(key) {
      return `fact["${key}"]`;
    }).join(", ")}];`);
    packer.fields = keys;
    return packer;
  }
  
  type EqualityChecker = (a:MapFact, b:MapFact) => Boolean;
  function generateEqualityFn(view:Id, keys:Id[]):EqualityChecker {
    return <EqualityChecker> new Function("a", "b",  `return ${keys.map(function(key, ix) {
      return `a["${key}"] === b["${key}"]`;
    }).join(" && ")};`);
  }

  function indexOfFact(equals:EqualityChecker, haystack:MapFact[], needle:MapFact):number {
    var result = -1;
    if(!equals) { return result; }
    for(var haystackIx = 0, haystackLen = haystack.length; haystackIx < haystackLen; haystackIx++) {
      var cur = haystack[haystackIx];
      if(equals(cur, needle)) {
        result = haystackIx;
        break;
      }
    }
    return result;
  }

  function applyTableDiff(equals:EqualityChecker, table:MapFact[], adds:MapFact[], removes:MapFact[]) {
    var dedupedAdds:MapFact[] = [];
    var dedupedRemoves:MapFact[] = [];
    for(var remIx = 0, remLen = removes.length; remIx < remLen; remIx++) {
      var rem = removes[remIx];
      var foundIx = indexOfFact(equals, table, rem);
      if(foundIx !== -1) {
        table.splice(foundIx, 1);
        dedupedRemoves.push(rem);
      }
    }
    for(var addIx = 0, addLen = adds.length; addIx < addLen; addIx++) {
      var add = adds[addIx];
      var foundIx = indexOfFact(equals, table, add);
      if(foundIx !== -1) continue;
      table.push(add);
      dedupedAdds.push(add);
    }
    return {adds: dedupedAdds, removes: dedupedRemoves};
  }

/*  var factExtractorFns:{[key:string]: Extractor} = {};
  var factPackerFns:{[key:string]: Packer} = {};
  var factEqualityFns:{[key:string]: EqualityChecker} = {};
*/
  export class Indexer {
    tables:{[key:string]: MapFact[]};
    indexes: {};
    tableToIndex: {};
    needsRebuild: {[table:string]: boolean};
    constructor() {
      this.tables = {};
      this.indexes = {};
      this.tableToIndex = {};
      this.needsRebuild = {};
    }
    totalFacts():Number {
      var total = 0;
      for(var table in this.tables) {
        total += this.tables[table].length;
      }
      return total;
    }
    clear():Payload {
      var final = [];
      for(var table in this.tables) {
        if(this.tables[table]) {
          this.handleDiff(table, [], this.tables[table].slice());
        }
      }
      for(var index in this.indexes) {
        this.indexes[index].index = {};
      }
      return {changes: final};
    }
    clearTable(table: Id) {
      if(this.tables[table]) {
        this.handleDiff(table, [], this.tables[table].slice());
      }
    }
    markForRebuild(table: Id) {
      this.needsRebuild[table] = true;<commit_msg>Fix clearTable method<commit_after>module Indexing {
	declare var DEBUG;
  declare var api;

  type Id = string;
  type ArrayFact = any[];
  interface MapFact {[field:string]: any};
  type PayloadChange = [Id, Id[], ArrayFact[], ArrayFact[]];
  interface Payload { changes: PayloadChange[] };

  export function arraysIdentical(a:any[], b:any[]):boolean {
    var i = a.length;
    if (!b || i != b.length) return false;
    while (i--) {
      if(a[i] && a[i].constructor === Array) {
        if(!arraysIdentical(a[i], b[i])) return false;
        continue;
      }
      if(a[i] && a[i].relation && b[i] && b[i].relation) {
        if(!arraysIdentical(a[i].relation, b[i].relation)) return false;
        continue;
      }
      if(a[i] && a[i].eid && b[i] && b[i].eid) { continue; }
      if (a[i] !== b[i]) return false;
    }
    return true;
  }
  
  export function objectsIdentical(a:any, b:any):boolean {
    if(typeof a !== typeof b) { return false; }
    if(typeof a !== "object") { return a === b; }
    var aKeys = Object.keys(a);
    if(!arraysIdentical(aKeys, Object.keys(b))) { return false; }
    
    for(var key of aKeys) {
      if(typeof a[key] !== typeof b[key]) { return false; }
      if(typeof a[key] !== "object" && a[key] !== b[key]) { return false; }
      else if(!objectsIdentical(a[key], b[key])) { return false; }
    }
    
    return true;
  }
  
  export function clone<T>(item:T): T;
  export function clone(item:Object): Object;
  export function clone(item:any[]): any[];
  export function clone(item:any): any {
    if (!item) { return item; }
    var result;

    if(item instanceof Array) {
      result = [];
      item.forEach(function(child, index, array) {
        result[index] = clone( child );
      });
    } else if(typeof item == "object") {
      result = {};
      for (var i in item) {
        result[i] = clone( item[i] );
      }
    } else {
      //it's a primitive
      result = item;
    }
    return result;
  }
  
  export function zip(rows, keys) {
    var keysLength = keys.length;
    var zipped = [];
    for (var row of rows) {
      var zippedRow = {};
      for(var keyIx = 0; keyIx < keysLength; keyIx++) {
        zippedRow[keys[keyIx]] = row[keyIx];
      }
      zipped.push(zippedRow);
    }
    return zipped;
  }
  
  export function unzip(zipped, keys?) {
    if(!zipped || !zipped.length) { return {keys: [], rows: []}; }
    if(!keys) {
      keys = Object.keys(zipped[0]);
    }
    var rows = [];
    for(var zippedRow of zipped) {
      var row = [];
      for(var key of keys) {
        row.push(zippedRow[key]);
      }
      rows.push(row);
    }
    return {keys: keys, rows: rows};
  }
  
  type Extractor = (fact:ArrayFact) => MapFact;
  function generateExtractorFn(view:Id, keys:Id[]):Extractor {
    return <Extractor> new Function("fact", `return { ${keys.map(function(key, ix) {
      return `"${key}": fact["${ix}"]`;
    }).join(", ")} };`);
  }
  
  type Packer = {(fact:MapFact): ArrayFact; fields: Id[]};
  function generatePackerFn(view:Id, keys:Id[]):Packer {
    var packer = <Packer> new Function("fact", `return [${keys.map(function(key) {
      return `fact["${key}"] || ""`;
    }).join(", ")}];`);
    packer.fields = keys;
    return packer;
  }
  
  type EqualityChecker = (a:MapFact, b:MapFact) => Boolean;
  function generateEqualityFn(view:Id, keys:Id[]):EqualityChecker {
    return <EqualityChecker> new Function("a", "b",  `return ${keys.map(function(key, ix) {
      return `a["${key}"] === b["${key}"]`;
    }).join(" && ")};`);
  }

  function indexOfFact(equals:EqualityChecker, haystack:MapFact[], needle:MapFact):number {
    var result = -1;
    if(!equals) { return result; }
    for(var haystackIx = 0, haystackLen = haystack.length; haystackIx < haystackLen; haystackIx++) {
      var cur = haystack[haystackIx];
      if(equals(cur, needle)) {
        result = haystackIx;
        break;
      }
    }
    return result;
  }

  function applyTableDiff(equals:EqualityChecker, table:MapFact[], adds:MapFact[], removes:MapFact[]) {
    var dedupedAdds:MapFact[] = [];
    var dedupedRemoves:MapFact[] = [];
    for(var remIx = 0, remLen = removes.length; remIx < remLen; remIx++) {
      var rem = removes[remIx];
      var foundIx = indexOfFact(equals, table, rem);
      if(foundIx !== -1) {
        table.splice(foundIx, 1);
        dedupedRemoves.push(rem);
      }
    }
    for(var addIx = 0, addLen = adds.length; addIx < addLen; addIx++) {
      var add = adds[addIx];
      var foundIx = indexOfFact(equals, table, add);
      if(foundIx !== -1) continue;
      table.push(add);
      dedupedAdds.push(add);
    }
    return {adds: dedupedAdds, removes: dedupedRemoves};
  }

/*  var factExtractorFns:{[key:string]: Extractor} = {};
  var factPackerFns:{[key:string]: Packer} = {};
  var factEqualityFns:{[key:string]: EqualityChecker} = {};
*/
  export class Indexer {
    tables:{[key:string]: MapFact[]};
    indexes: {};
    tableToIndex: {};
    needsRebuild: {[table:string]: boolean};
    constructor() {
      this.tables = {};
      this.indexes = {};
      this.tableToIndex = {};
      this.needsRebuild = {};
    }
    totalFacts():Number {
      var total = 0;
      for(var table in this.tables) {
        total += this.tables[table].length;
      }
      return total;
    }
    clear():Payload {
      var final = [];
      for(var table in this.tables) {
        if(this.tables[table]) {
          this.handleDiff(table, [], this.tables[table].slice());
        }
      }
      for(var index in this.indexes) {
        this.indexes[index].index = {};
      }
      return {changes: final};
    }
    clearTable(table: Id) {
      if(this.tables[table]) {
        this.handleDiff(table, this.getFields(table), [], this.tables[table].slice());
      }
    }
    markForRebuild(table: Id) {
      this.needsRebuild[table] = true;
{'fraction_non_alphanumeric': 0.1744771660264618, 'fraction_numerical': 0.0017072129748186087, 'mean_word_length': 2.713470681458003, 'pattern_counts': {'":': 2, '<': 19, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5855563', 'n_tokens_mistral': 4071, 'n_tokens_neox': 4055, 'n_words': 878}