<commit_before>import { Document } from './document'

import { SortData } from './types/sortdata'

function nestedProperty(doc:Document, key:string):any {
  key = key.replace(/\[(\w+)\]/g, '.$1')
  key = key.replace(/^\./, '')
  let parts:string[] = key.split('.')
  let val:any = doc
  parts.forEach(k => {
    // Is k within val?
    if (!val.hasOwnProperty(k)) throw new Error(`Nested property '${key}' could not be resolved at '${k}' property.`)
    val = val[k]
  })
  return val
}

function sortDocuments (documents:Document[], sortData:SortData[], algorithm?:'native'|'quick'|'insertion'):void {
  switch(algorithm) {
    case 'insertion': insertionSort(documents, sortData); return
    case 'native': sort(documents, sortData); return
    case 'quick': quickSort(documents, sortData); return
    default: sort(documents, sortData); return
  }
}

function sort (documents:Document[], sortData:SortData[]):void {
  documents.sort(evalCompare(sortData))
}

// This function is quite slow!
function comparisonFn (sortData:SortData[]):((a:Document, b:Document) => number) {
  return function (a:Document, b:Document):number {
    const l = sortData.length
    for (let i = 0; i < l; ++i) {
      const sort = sortData[i]
      let _a
      let _b
      if (sort.nested) {
        _a = nestedProperty(a, sort.key)
        _b = nestedProperty(b, sort.key)
      } else {
        _a = a[sort.key]
        _b = b[sort.key]
      }
      if (_a !== _b) {
        return (_a > _b ? 1 : -1) * sort.order
      }
    }
    return 0
  }
}

function evalCompare (sortData:SortData[]):((a:Document, b:Document) => number) {
  let str = ''
  sortData.forEach(sort => {
    str += `if(a.${sort.key}${sort.order === 1 ? '>' : '<'}b.${sort.key})return 1;`
    str += `if(a.${sort.key}${sort.order === 1 ? '<' : '>'}b.${sort.key})return -1;`
  })
  str += `return 0;`
  return <((a:Document, b:Document) => number)>Function('a', 'b', str)
}

function insertionSort (documents:Document[], sortData:SortData[]):void {
  const compare = evalCompare(sortData)
  for (let i = 1; i < documents.length; ++i) {
    let j = i
    while (j > 0 && compare(documents[j-1], documents[j]) > 0) {
      arraySwap(documents, j, j - 1)
      j--
    }
  }
}

function quickSort (documents:Document[], sortData:SortData[]):void {
  quickSortFn(documents, 0, documents.length - 1, evalCompare(sortData))
}
<commit_msg>Added heapsort to utils (unused as of now)<commit_after>import { Document } from './document'

import { SortData } from './types/sortdata'

function nestedProperty(doc:Document, key:string):any {
  key = key.replace(/\[(\w+)\]/g, '.$1')
  key = key.replace(/^\./, '')
  let parts:string[] = key.split('.')
  let val:any = doc
  parts.forEach(k => {
    // Is k within val?
    if (!val.hasOwnProperty(k)) throw new Error(`Nested property '${key}' could not be resolved at '${k}' property.`)
    val = val[k]
  })
  return val
}

function sortDocuments (documents:Document[], sortData:SortData[], algorithm?:'native'|'quick'|'insertion'|'heap'):void {
  switch(algorithm) {
    case 'insertion': insertionSort(documents, sortData); return
    case 'heap': heapSort(documents, sortData); return
    case 'native': sort(documents, sortData); return
    case 'quick': quickSort(documents, sortData); return
    default: sort(documents, sortData); return
  }
}

function sort (documents:Document[], sortData:SortData[]):void {
  documents.sort(evalCompare(sortData))
}

// This function is quite slow!
function comparisonFn (sortData:SortData[]):((a:Document, b:Document) => number) {
  return function (a:Document, b:Document):number {
    const l = sortData.length
    for (let i = 0; i < l; ++i) {
      const sort = sortData[i]
      let _a
      let _b
      if (sort.nested) {
        _a = nestedProperty(a, sort.key)
        _b = nestedProperty(b, sort.key)
      } else {
        _a = a[sort.key]
        _b = b[sort.key]
      }
      if (_a !== _b) {
        return (_a > _b ? 1 : -1) * sort.order
      }
    }
    return 0
  }
}

function evalCompare (sortData:SortData[]):((a:Document, b:Document) => number) {
  let str = ''
  sortData.forEach(sort => {
    str += `if(a.${sort.key}${sort.order === 1 ? '>' : '<'}b.${sort.key})return 1;`
    str += `if(a.${sort.key}${sort.order === 1 ? '<' : '>'}b.${sort.key})return -1;`
  })
  str += `return 0;`
  return <((a:Document, b:Document) => number)>Function('a', 'b', str)
}

function insertionSort (documents:Document[], sortData:SortData[]):void {
  const compare = evalCompare(sortData)
  for (let i = 1; i < documents.length; ++i) {
    let j = i
    while (j > 0 && compare(documents[j-1], documents[j]) > 0) {
      arraySwap(documents, j, j - 1)
      j--
    }
  }
}

function heapSort (documents:Document[], sortData:SortData[]):void {
  const compare = evalCompare(sortData)
  let len = documents.length
  for (let i = Math.floor(len * .5); i > -1; --i) heapify(documents, i, len, compare)
  for (let i = documents.length - 1; i > 0; --i) {
    arraySwap(documents, 0, i)
    len--
    heapify(documents, 0, len, compare)
  }
}

function heapify (documents:Document[], i:number, len:number, compare:((a:Document, b:Document) => number)):void {
  const l = 2 * i + 1
  const r = 2 * i + 2
  let largest = i

  if (l < len && compare(documents[l], documents[largest]) > 0) largest = l
  if (r < len && compare(documents[r], documents[largest]) > 0) largest = r
  if (largest !== i) {
    arraySwap(documents, i, largest)
    heapify(documents, largest, len, compare)
  }
}

function quickSort (documents:Document[], sortData:SortData[]):void {
  quickSortFn(documents, 0, documents.length - 1, evalCompare(sortData))
}

{'fraction_non_alphanumeric': 0.18116069853589697, 'fraction_numerical': 0.008114305874051861, 'mean_word_length': 3.395348837209302, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1297850', 'n_tokens_mistral': 2008, 'n_tokens_neox': 1944, 'n_words': 485}