<commit_before>const moment = require('moment')

module.exports = class Results {
  // params = {
  //   sort: params.sort,
  //   columns: params.columns,
  //   filter: params.filter || [], # [{ key, value }]
  //
  //   offset: params.offset.try(:to_i) || 0,
  //   limit: params.limit.try(:to_i) || 25,
  //
  //   export: params.export,
  //   exportTitle: params.exportTitle,
  //   svg: params.svg,
  //
  //   percentages: !!params.percentages,
  //
  //   graphTimeFilter: params.graphTimeFilter || 'last-60',
  //   graphCumulative: !!params.graphCumulative,
  //
  //   facetsColumn: params.facetsColumn,
  //   facetsCount: params.facetsCount.present? ? params.facetsCount.to_i : 6
  //
  //   graphOnly: false
  // }

  constructor ({ params, adapter, structure }) {
    this.params = params
    this.adapter = adapter
    this.structure = structure
  }

  async getResponse () {
    this.resetCache()

    // common
    this.setJoinsAndColumnMetadata()
    this.setFilter()

    // table
    this.setGroup()
    this.setSort()
    this.setLimit()

    if (!this.params.graphOnly) {
      await this.getCount()
      await this.getResults()
    }

    // graph
    await this.getGraph()

    this.setResponse()

    return this.response
  }

  resetCache () {
    this.baseModelName = null
    this.columnMetadata = []
    this.resultsTableColumns = []

    this.commonSqlConditions = {
      having: [],
      where: []
    }

    this.commonSqlParts = {
      fromAndJoins: '',
      where: '',
      having: ''
    }

    this.resultsTableCount = 1

    this.resultsTableSqlParts = {
      select: '',
      group: '',
      sort: '',
      limit: '',
      offset: ''
    }

    this.finalResults = []

    this.graphResponse = null

    this.response = null
  }

  setJoinsAndColumnMetadata () {
    // input params
    const { columns, filter } = this.params

    // are we requesting any data?
    if (!columns) {
      throw new Error('No columns selected')
    }

    // the model we're requesting must exist
    this.baseModelName = columns[0].split('.')[0]
    if (!this.structure[this.baseModelName]) {
      throw new Error(`Base model "${this.baseModelName}" not found`)
    }

    // get all the columns we must be able to access (results table + filter)
    const filterColumns = uniq(filter.map(v => v.key))
    const columnsToJoin = uniq(columns.concat(filterColumns))

    // each requested column must be for this model
    if (columnsToJoin.filter(s => s.split('.')[0] !== this.baseModelName).count > 0) {
      throw new Error('Each column must be from the same base model!')
    }

    // counters for value and table aliases
    let tableCounter = 0
    let valueCounter = 0

    // add the base model's path to the "join list"
    const baseTableName = this.structure[this.baseModelName]['table_name']
    const baseTableAlias = `$T${tableCounter}` // "$T0"

    let joins = {
      [this.baseModelName]: {
        alias: baseTableAlias,
        sql: this.adapter.fromTableAs(baseTableName, baseTableAlias) // "FROM baseTableName AS baseTableAlias"
      }
    }

    // make sure we have access to each of the columns we need
    columnsToJoin.forEach(column => {
      // each incoming column is in the format "path!transform!aggregate"
      // e.g. "Order.order_lines.created_at!month!max"
      const [ path, transform, aggregate ] = column.replace('$', '!').split('!', 3)

      // we will split the path by "." and follow the links going forward, adding joins when needed

      // always starting the traverse from the base model
      let node = this.structure[this.baseModelName]
      let tableAlias = baseTableAlias

      // merge the links if using the old deprecated links: { incoming: {}, outgoing: {} } this.structure
      if (Object.keys(node.links).sort().join(' ') === 'incoming outgoing') {
        node.links = Object.assign({}, node.links.incoming, node.links.outgoing)
      }

      // log of how far we have gotten. empty at first
      let traversedPath = [this.baseModelName]

      // remove the modal from the path, so "Order.order_lines.created_at" -> "order_lines.created_at"
      let localPath = path.split('.')
      localPath.shift()

      localPath.forEach(key => {
        // if the next part of the key is a column, deal with it
        // (alternatively, if it's a link to a model, and we'll deal with it below in the "else")
        if (node.columns[key] || node.custom[key]) {
          let meta = null // from insights.yml
          let sql = null // how to access this field
          let sqlBeforeTransform = null

          // get meta and sql
          if (node.columns[key]) {
            meta = node.columns[key]
            sql = this.adapter.tableAliasWithColumn(tableAlias, key)
          } else if (node.custom[key]) {
            meta = node.custom[key]
            sql = this.adapter.customSqlInTableReplace(meta.sql, tableAlias)
          }

          // perform date conversion if needed
          if (meta.type === 'time') {
            sql = this.adapter.convertSqlTimezone(sql)
          }

          if (transform && (meta.type === 'time' || meta.type === 'date')) {
            sqlBeforeTransform = sql<commit_msg>use keys like ${key} in custom sql<commit_after>const moment = require('moment')

module.exports = class Results {
  // params = {
  //   sort: params.sort,
  //   columns: params.columns,
  //   filter: params.filter || [], # [{ key, value }]
  //
  //   offset: params.offset.try(:to_i) || 0,
  //   limit: params.limit.try(:to_i) || 25,
  //
  //   export: params.export,
  //   exportTitle: params.exportTitle,
  //   svg: params.svg,
  //
  //   percentages: !!params.percentages,
  //
  //   graphTimeFilter: params.graphTimeFilter || 'last-60',
  //   graphCumulative: !!params.graphCumulative,
  //
  //   facetsColumn: params.facetsColumn,
  //   facetsCount: params.facetsCount.present? ? params.facetsCount.to_i : 6
  //
  //   graphOnly: false
  // }

  constructor ({ params, adapter, structure }) {
    this.params = params
    this.adapter = adapter
    this.structure = structure
  }

  async getResponse () {
    this.resetCache()

    // common
    this.setJoinsAndColumnMetadata()
    this.setFilter()

    // table
    this.setGroup()
    this.setSort()
    this.setLimit()

    if (!this.params.graphOnly) {
      await this.getCount()
      await this.getResults()
    }

    // graph
    await this.getGraph()

    this.setResponse()

    return this.response
  }

  resetCache () {
    this.baseModelName = null
    this.columnMetadata = []
    this.resultsTableColumns = []

    this.commonSqlConditions = {
      having: [],
      where: []
    }

    this.commonSqlParts = {
      fromAndJoins: '',
      where: '',
      having: ''
    }

    this.resultsTableCount = 1

    this.resultsTableSqlParts = {
      select: '',
      group: '',
      sort: '',
      limit: '',
      offset: ''
    }

    this.finalResults = []

    this.graphResponse = null

    this.response = null
  }

  setJoinsAndColumnMetadata () {
    // input params
    const { columns, filter } = this.params

    // are we requesting any data?
    if (!columns) {
      throw new Error('No columns selected')
    }

    // the model we're requesting must exist
    this.baseModelName = columns[0].split('.')[0]
    if (!this.structure[this.baseModelName]) {
      throw new Error(`Base model "${this.baseModelName}" not found`)
    }

    // get all the columns we must be able to access (results table + filter)
    const filterColumns = uniq(filter.map(v => v.key))
    const columnsToJoin = uniq(columns.concat(filterColumns))

    // each requested column must be for this model
    if (columnsToJoin.filter(s => s.split('.')[0] !== this.baseModelName).count > 0) {
      throw new Error('Each column must be from the same base model!')
    }

    // counters for value and table aliases
    let tableCounter = 0
    let valueCounter = 0

    // add the base model's path to the "join list"
    const baseTableName = this.structure[this.baseModelName]['table_name']
    const baseTableAlias = `$T${tableCounter}` // "$T0"

    let joins = {
      [this.baseModelName]: {
        alias: baseTableAlias,
        sql: this.adapter.fromTableAs(baseTableName, baseTableAlias) // "FROM baseTableName AS baseTableAlias"
      }
    }

    // make sure we have access to each of the columns we need
    columnsToJoin.forEach(column => {
      // each incoming column is in the format "path!transform!aggregate"
      // e.g. "Order.order_lines.created_at!month!max"
      const [ path, transform, aggregate ] = column.replace('$', '!').split('!', 3)

      // we will split the path by "." and follow the links going forward, adding joins when needed

      // always starting the traverse from the base model
      let node = this.structure[this.baseModelName]
      let tableAlias = baseTableAlias

      // merge the links if using the old deprecated links: { incoming: {}, outgoing: {} } this.structure
      if (Object.keys(node.links).sort().join(' ') === 'incoming outgoing') {
        node.links = Object.assign({}, node.links.incoming, node.links.outgoing)
      }

      // log of how far we have gotten. empty at first
      let traversedPath = [this.baseModelName]

      // remove the modal from the path, so "Order.order_lines.created_at" -> "order_lines.created_at"
      let localPath = path.split('.')
      localPath.shift()

      localPath.forEach(key => {
        // if the next part of the key is a column, deal with it
        // (alternatively, if it's a link to a model, and we'll deal with it below in the "else")
        if (node.columns[key] || node.custom[key]) {
          let meta = null // from insights.yml
          let sql = null // how to access this field
          let sqlBeforeTransform = null

          // get meta and sql
          if (node.columns[key]) {
            meta = node.columns[key]
            sql = this.adapter.tableAliasWithColumn(tableAlias, key)
          } else if (node.custom[key]) {
            meta = node.custom[key]

            sql = meta.sql
            let limit = 0

            // replace keys like "${other_column}"
            while (true) {
              limit += 1
              if (limit > 100) {
                throw new Error(`Recursive loop when evalutating custom key ${traversedPath.join('.')}.${key}`)
              }

              const match = sql.match(/\${([^}]+)}/)
              if (!match) {
                break
              }

              const toReplace = match[0]
              const keyword = match[1]

              if (node.columns[keyword]) {
                sql = sql.split(toReplace).join(this.adapter.tableAliasWithColumn(tableAlias, keyword))
              }

              if (node.custom[keyword]) {
                sql = sql.split(toReplace).join(`(${node.custom[keyword].sql})`)
              }
            }

            // replace "$$." with the table name
            sql = this.adapter.customSqlInTableReplace(sql, tableAlias)
          }

          // perform date conversion if needed
          if (meta.type === 'time') {
            sql = this.adapter.convertSqlTimezone(sql)
          }

          if (transform && (meta.type === 'time' || meta.type === 'date')) {
            sqlBeforeTransform = sql
{'fraction_non_alphanumeric': 0.1294107268877911, 'fraction_numerical': 0.00326393789696542, 'mean_word_length': 2.1890295358649787, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6518195', 'n_tokens_mistral': 3399, 'n_tokens_neox': 3270, 'n_words': 880}