<commit_before>---
---

"use strict"

$ ->
  $form = $('#search-form')
  $input = $('#search-input')
  $similar = $('#similar')
  $similarList = $('#similar-list')
  $matchesString = $('#matches')
  $results = $('#results')
  $progress = $('#progress')
  $template = $('#template')

  searchIndex = null

  ajax = new $.Deferred()

  wordsplit = /[\s,;!?=\+"'\\\/]+/
  typeTimer = null

  generateWordList = ->
    window.wordlistAll = {}

    searchIndex.pages.forEach (obj, pageId) ->
      # Uppercase is a work-around to avoid reserved words
      # (Thanks, JavaScript!)
      obj.wordlist.toUpperCase().split(/\s+/).forEach (word) ->
        return if word == ''

        wordlistAll[word] ||= []

        try
          wordlistAll[word].push pageId
        catch e
          console.log "Problem:", word, pageId
        return

    searchIndex.words = wordlistAll

  loadsearchIndex = ->
    return if ajax.state() == "resolved" && ajax.state() == "pending"

    $.get '{{ site.baseurl }}/search.json', (data)->
      searchIndex = data
      do generateWordList unless data.words
      ajax.resolve data


  displayResults = (words, matches, similar) ->
    $pending = $('<div class="pending"/>')
    pages = searchIndex.pages
    text = $input.val().toLowerCase().replace(/ ?-\w+/g, '').replace(/\+/g, '')

    $('#results-none').hide()

    if matches.length > 0
      fakedate = (new Date).toISOString()

      matches.sort (a, b) ->
        # Sort by date
        dateA = pages[a].date || fakedate
        dateB = pages[b].date || fakedate

        # Title match score (weighted heavily)
        titleA = pages[a].title.toLowerCase().score(text)
        titleB = pages[b].title.toLowerCase().score(text)

        # Summary snippet score (weighted a good bit)
        summA = pages[a].summary.toLowerCase().score(text)
        summB = pages[b].summary.toLowerCase().score(text)

        # Date score (very small weight)
        dateScoreA = Date.parse(dateA.split(' ')[0]) / Date.now() * 0.05
        dateScoreB = Date.parse(dateB.split(' ')[0]) / Date.now() * 0.05

        # Generate page scores
        # (and add a 'score' value to the page info)
        scoreA = pages[a].score = (titleA * 3 + summA + dateScoreA) / 5 / 0.71
        scoreB = pages[b].score = (titleB * 3 + summB + dateScoreB) / 5 / 0.71

        # Sort based on score
        return scoreA < scoreB

      for match in matches
        page = pages[match]

        # Clone search result snippet
        $snippet = $template.children().clone()

        # Add data to the cloned template
        $snippet.removeClass('hidden')
          .find('.title').html(page.title).attr('href', page.url).end()
          .find('.url').html(page.url).attr('href', page.url).end()
          .find('.summary').html(page.summary).end()

        $snippet.find('.date').text(page.datestring).end() if page.date

        # Add snippet to pending (off-page) search results
        $pending.append $snippet

    else
      if $input.val() != ''
        $('#results-none').removeClass('hidden').show()

    # Add search results to the page
    $results.html($pending.children())

    if similar.length > 0
      $similar.removeClass('hidden').hide()
      $similarList.html ->
        result = ''

        $.each similar, (k, v)->
          rating = "Score: " + Math.round(v.score * 50) / 10
          result += '<a href="#%" title="$">%</a> '.replace(/%/g, v.word).replace(/\$/, rating)

        return result<commit_msg>search: remove date field if unused<commit_after>---
---

"use strict"

$ ->
  $form = $('#search-form')
  $input = $('#search-input')
  $similar = $('#similar')
  $similarList = $('#similar-list')
  $matchesString = $('#matches')
  $results = $('#results')
  $progress = $('#progress')
  $template = $('#template')

  searchIndex = null

  ajax = new $.Deferred()

  wordsplit = /[\s,;!?=\+"'\\\/]+/
  typeTimer = null

  generateWordList = ->
    window.wordlistAll = {}

    searchIndex.pages.forEach (obj, pageId) ->
      # Uppercase is a work-around to avoid reserved words
      # (Thanks, JavaScript!)
      obj.wordlist.toUpperCase().split(/\s+/).forEach (word) ->
        return if word == ''

        wordlistAll[word] ||= []

        try
          wordlistAll[word].push pageId
        catch e
          console.log "Problem:", word, pageId
        return

    searchIndex.words = wordlistAll

  loadsearchIndex = ->
    return if ajax.state() == "resolved" && ajax.state() == "pending"

    $.get '{{ site.baseurl }}/search.json', (data)->
      searchIndex = data
      do generateWordList unless data.words
      ajax.resolve data


  displayResults = (words, matches, similar) ->
    $pending = $('<div class="pending"/>')
    pages = searchIndex.pages
    text = $input.val().toLowerCase().replace(/ ?-\w+/g, '').replace(/\+/g, '')

    $('#results-none').hide()

    if matches.length > 0
      fakedate = (new Date).toISOString()

      matches.sort (a, b) ->
        # Sort by date
        dateA = pages[a].date || fakedate
        dateB = pages[b].date || fakedate

        # Title match score (weighted heavily)
        titleA = pages[a].title.toLowerCase().score(text)
        titleB = pages[b].title.toLowerCase().score(text)

        # Summary snippet score (weighted a good bit)
        summA = pages[a].summary.toLowerCase().score(text)
        summB = pages[b].summary.toLowerCase().score(text)

        # Date score (very small weight)
        dateScoreA = Date.parse(dateA.split(' ')[0]) / Date.now() * 0.05
        dateScoreB = Date.parse(dateB.split(' ')[0]) / Date.now() * 0.05

        # Generate page scores
        # (and add a 'score' value to the page info)
        scoreA = pages[a].score = (titleA * 3 + summA + dateScoreA) / 5 / 0.71
        scoreB = pages[b].score = (titleB * 3 + summB + dateScoreB) / 5 / 0.71

        # Sort based on score
        return scoreA < scoreB

      for match in matches
        page = pages[match]

        # Clone search result snippet
        $snippet = $template.children().clone()

        # Add data to the cloned template
        $snippet.removeClass('hidden')
          .find('.title').html(page.title).attr('href', page.url).end()
          .find('.url').html(page.url).attr('href', page.url).end()
          .find('.summary').html(page.summary).end()

        if page.date
          $snippet.find('.date').text(page.datestring).end()
        else
          $snippet.find('.date').remove().end()

        # Add snippet to pending (off-page) search results
        $pending.append $snippet

    else
      if $input.val() != ''
        $('#results-none').removeClass('hidden').show()

    # Add search results to the page
    $results.html($pending.children())

    if similar.length > 0
      $similar.removeClass('hidden').hide()
      $similarList.html ->
        result = ''

        $.each similar, (k, v)->
          rating = "Score: " + Math.round(v.score * 50) / 10
          result += '<a href="#%" title="$">%</a> '.replace(/%/g, v.word).replace(/\$/, rating)

        return result
{'fraction_non_alphanumeric': 0.17496100949950374, 'fraction_numerical': 0.006805614632071459, 'mean_word_length': 2.3179680150517403, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5736303', 'n_tokens_mistral': 2250, 'n_tokens_neox': 2139, 'n_words': 512}