<commit_before>'use strict'

var pagesdb = require('../../../db/pagesdb')
var processSearchTerms = require('./processSearchTerms')
var appLogger = require('../../../utils/appLogger')
var printSearchSQL = require('./printSearchSQL')

var devMode = process.env.NODE_ENV === 'development'

function search(req, res) {

  var processedSearchTerms = processSearchTerms(req.params.searchTerms)
  var domainToSearchFor = processedSearchTerms.domainToSearchFor
  var searchTerms = processedSearchTerms.processedSearchTerms
  var dateFilter = {
      dateFilterStartDate: req.body.dateFilterStartDate,
      dateFilterEndDate: req.body.dateFilterEndDate
  }
  var usingDateFilter = !!(dateFilter.dateFilterStartDate && dateFilter.dateFilterEndDate)
  var knexSQL = null

  /****
   * note: knex will automatically convert additional WHERE's to AND
   */
  if(!searchTerms.length){
    /****
     * If user just wants to list all saved pages by a domain (with no text search)
     *
     */
    if(domainToSearchFor){
      /****
       * Omiting the pageText as that will be fairly large and we don't need that.
       * Also don't need checkedForExpiry.
       */
      knexSQL = pagesdb.db.select(
                  'pageUrl',
                  'pageDomain',
                  'pageTitle',
                  'pageDescription',
                  'dateCreated',
                  'archiveLink',
                  'safeBrowsing'
                )
                .from('pages')
                .where({pageDomain: domainToSearchFor})
      if(usingDateFilter){
        knexSQL = knexSQL
          .where('dateCreated', '>=', dateFilter.dateFilterStartDate)
          .where('dateCreated', '<=', dateFilter.dateFilterEndDate)
      }
      knexSQL = knexSQL.orderBy('dateCreated', 'desc')
    }
    else{
      /****
       * If there are no search terms and they are not searching for domain
       * results, then send back zero results as an empty array and also return so
       * we dont continue with the rest of the function under the 'else{}'.
       * Could also do a nothing sql statment like knexSQL = pagesdb.db('pages').where({1: 0})
       * i guess.
       */
      res.json([])
      return
    }
  }
  else{
    /****
     * Omiting the pageText as that will be fairly large and we don't need that, the
     * fts match ? order by bm25... below seems to still work and search pageText even
     * if it is not selected and returned.
     * Also don't need checkedForExpiry.
     */
    knexSQL = pagesdb
      .db
      .select(
        `rank`,
        `pageUrl`,
        `dateCreated`,
        `pageDomain`,
        `pageTitle`,
        // `pageText`,
        `pageDescription`,
        `archiveLink`,
        `safeBrowsing`,
        pagesdb.db.raw(
          `snippet(fts, -1, '<span class="searchHighlight">',
          '</span>', '...', 64) as snippet`
        )
      )
      .from('fts')

    if(domainToSearchFor){
      knexSQL = knexSQL.where({pageDomain: domainToSearchFor})
    }
    if(usingDateFilter){
      knexSQL = knexSQL
        .where('dateCreated', '>=', dateFilter.dateFilterStartDate)<commit_msg>Removed comemnted out code in search.js<commit_after>'use strict'

var pagesdb = require('../../../db/pagesdb')
var processSearchTerms = require('./processSearchTerms')
var appLogger = require('../../../utils/appLogger')
var printSearchSQL = require('./printSearchSQL')

var devMode = process.env.NODE_ENV === 'development'

function search(req, res) {

  var processedSearchTerms = processSearchTerms(req.params.searchTerms)
  var domainToSearchFor = processedSearchTerms.domainToSearchFor
  var searchTerms = processedSearchTerms.processedSearchTerms
  var dateFilter = {
      dateFilterStartDate: req.body.dateFilterStartDate,
      dateFilterEndDate: req.body.dateFilterEndDate
  }
  var usingDateFilter = !!(dateFilter.dateFilterStartDate && dateFilter.dateFilterEndDate)
  var knexSQL = null

  /****
   * note: knex will automatically convert additional WHERE's to AND
   */
  if(!searchTerms.length){
    /****
     * If user just wants to list all saved pages by a domain (with no text search)
     *
     */
    if(domainToSearchFor){
      /****
       * Omiting the pageText as that will be fairly large and we don't need that.
       * Also don't need checkedForExpiry.
       */
      knexSQL = pagesdb.db.select(
                  'pageUrl',
                  'pageDomain',
                  'pageTitle',
                  'pageDescription',
                  'dateCreated',
                  'archiveLink',
                  'safeBrowsing'
                )
                .from('pages')
                .where({pageDomain: domainToSearchFor})
      if(usingDateFilter){
        knexSQL = knexSQL
          .where('dateCreated', '>=', dateFilter.dateFilterStartDate)
          .where('dateCreated', '<=', dateFilter.dateFilterEndDate)
      }
      knexSQL = knexSQL.orderBy('dateCreated', 'desc')
    }
    else{
      /****
       * If there are no search terms and they are not searching for domain
       * results, then send back zero results as an empty array and also return so
       * we dont continue with the rest of the function under the 'else{}'.
       * Could also do a nothing sql statment like knexSQL = pagesdb.db('pages').where({1: 0})
       * i guess.
       */
      res.json([])
      return
    }
  }
  else{
    /****
     * Omiting the pageText as that will be fairly large and we don't need that, the
     * fts match ? order by bm25... below seems to still work and search pageText even
     * if it is not selected and returned.
     * Also don't need checkedForExpiry.
     */
    knexSQL = pagesdb
      .db
      .select(
        `rank`,
        `pageUrl`,
        `dateCreated`,
        `pageDomain`,
        `pageTitle`,
        `pageDescription`,
        `archiveLink`,
        `safeBrowsing`,
        pagesdb.db.raw(
          `snippet(fts, -1, '<span class="searchHighlight">',
          '</span>', '...', 64) as snippet`
        )
      )
      .from('fts')

    if(domainToSearchFor){
      knexSQL = knexSQL.where({pageDomain: domainToSearchFor})
    }
    if(usingDateFilter){
      knexSQL = knexSQL
        .where('dateCreated', '>=', dateFilter.dateFilterStartDate)
{'fraction_non_alphanumeric': 0.11578442347673117, 'fraction_numerical': 0.0022389253158483927, 'mean_word_length': 2.225373904074265, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3296910', 'n_tokens_mistral': 1831, 'n_tokens_neox': 1777, 'n_words': 479}