<commit_before>class Suggestion
  showRelevancy: true # Set this to true to render relevancy when debugging the ranking scores.

  # - type: one of [bookmark, history, tab].
  constructor: (@queryTerms, @type, @url, @title, @computeRelevancyFunction, @extraRelevancyData) ->
    @title ||= ""

  generateHtml: ->
    return @html if @html
    relevancyHtml = if @showRelevancy then "<span class='relevancy'>#{@computeRelevancy() + ''}</span>" else ""
    @html =
      "<div class='topHalf'>
         <span class='source'>#{@type}</span>
         <span class='title'>#{@highlightTerms(utils.escapeHtml(@title))}</span>
       </div>
       <div class='bottomHalf'>
        <span class='url'>#{@shortenUrl(@highlightTerms(@url))}</span>
        #{relevancyHtml}
      </div>"

  shortenUrl: (url) ->
    @stripTrailingSlash(url).replace(/^http:\/\//, "")

  stripTrailingSlash: (url) ->
    url = url.substring(url, url.length - 1) if url[url.length - 1] == "/"
    url

  # Merges the given list of ranges such that any overlapping regions are combined. E.g.
  #   mergeRanges([0, 4], [3, 6]) => [0, 6].  A range is [startIndex, endIndex].
  mergeRanges: (ranges) ->
    previous = ranges.shift()
    mergedRanges = [previous]
    ranges.forEach (range) ->
      if previous[1] >= range[0]
        previous[1] = Math.max(range[1], previous[1])
      else
        mergedRanges.push(range)
        previous = range
    mergedRanges

  # Wraps each occurence of the query terms in the given string in a <span>.
  highlightTerms: (string) ->
    ranges = []
    for term in @queryTerms
      regexp = @escapeRegexp(term)
      i = string.search(regexp)
      ranges.push([i, i + term.length]) if i >= 0

    return string if ranges.length == 0

    ranges = @mergeRanges(ranges.sort (a, b) -> a[0] - b[0])
    # Replace portions of the string from right to left.
    ranges = ranges.sort (a, b) -> b[0] - a[0]
    for [start, end] in ranges
      string =
        string.substring(0, start) +
        "<span class='match'>" + string.substring(start, end) + "</span>" +
        string.substring(end)
    string

  # Creates a Regexp from the given string, with all special Regexp characters escaped.
  escapeRegexp: (string) ->
    # Taken from http://stackoverflow.com/questions/3446170/escape-string-for-use-in-javascript-regex
    Suggestion.escapeRegExp ||= /[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g
    new RegExp(string.replace(Suggestion.escapeRegExp, "\\$&"), "i")

  computeRelevancy: -> @relevancy = @computeRelevancyFunction(@queryTerms, this)

class BookmarkCompleter
  currentSearch: null
  # These bookmarks are loaded asynchronously when refresh() is called.
  bookmarks: null

  filter: (@queryTerms, @onComplete) ->
    @currentSearch = { queryTerms: @queryTerms, onComplete: @onComplete }
    @performSearch() if @bookmarks

  onBookmarksLoaded: -> @performSearch() if @currentSearch

  performSearch: ->
    results = @bookmarks.filter (bookmark) =>
        RankingUtils.matches(@currentSearch.queryTerms, bookmark.url, bookmark.title)
    suggestions = results.map (bookmark) =>
      new Suggestion(@currentSearch.queryTerms, "bookmark", bookmark.url, bookmark.title, @computeRelevancy)
    onComplete = @currentSearch.onComplete
    @currentSearch = null
    onComplete(suggestions)

  refresh: ->
    @bookmarks = null
    chrome.bookmarks.getTree (bookmarks) =>
      @bookmarks = @traverseBookmarks(bookmarks).filter((bookmark) -> bookmark.url?)
      @onBookmarksLoaded()

  # Traverses the bookmark hierarchy, and retuns a flattened list of all bookmarks in the tree.
  traverseBookmarks: (bookmarks) ->
    results = []
    toVisit = bookmarks
    while toVisit.length > 0
      bookmark = toVisit.shift()
      results.push(bookmark)
      toVisit.push.apply(toVisit, bookmark.children) if (bookmark.children)
    results

  computeRelevancy: (queryTerms, suggestion) ->
    RankingUtils.wordRelevancy(queryTerms, suggestion.url, suggestion.title)

class HistoryCompleter
  filter: (queryTerms, onComplete) ->
    @currentSearch = { queryTerms: @queryTerms, onComplete: @onComplete }
    results = []
    HistoryCache.use (history) ->
      results = history.filter (entry) -> RankingUtils.matches(queryTerms, entry.url, entry.title)
    suggestions = results.map (entry) =>
      new Suggestion(queryTerms, "history", entry.url, entry.title, @computeRelevancy, entry)
    onComplete(suggestions)

  computeRelevancy: (queryTerms, suggestion) ->
    historyEntry = suggestion.extraRelevancyData
    recencyScore = RankingUtils.recencyScore(historyEntry.lastVisitTime)
    wordRelevancy = RankingUtils.wordRelevancy(queryTerms, suggestion.url, suggestion.title)
    # Average out the word score and the recency. Recency has the ability to pull the score up, but not down.
    score = (wordRelevancy + Math.max(recencyScore, wordRelevancy)) / 2

  refresh: ->

# The domain completer is designed to match a single-word query which looks like it is a domain. This supports
# the user experience where they quickly type a partial domain, hit tab -> enter, and expect to arrive there.
class DomainCompleter
  domains: null # A map of domain -> history

  filter: (queryTerms, onComplete) ->
    return onComplete([]) if queryTerms.length > 1
    if @domains
      @performSearch(queryTerms, onComplete)
    else
      @populateDomains => @performSearch(queryTerms, onComplete)

  performSearch: (queryTerms, onComplete) ->
    query = queryTerms[0]
    domainCandidates = (domain for domain of @domains when domain.indexOf(query) >= 0)
    domains = @sortDomainsByRelevancy(queryTerms, domainCandidates)
    return onComplete([]) if domains.length == 0
    topDomain = domains[0][0]
    onComplete([new Suggestion(queryTerms, "domain", topDomain, null, @computeRelevancy)])

  # Returns a list of domains of the form: [ [domain, relevancy], ... ]
  sortDomainsByRelevancy: (queryTerms, domainCandidates) ->
    results = []
    for domain in domainCandidates
      recencyScore = RankingUtils.recencyScore(@domains[domain].lastVisitTime || 0)
      wordRelevancy = RankingUtils.wordRelevancy(queryTerms, domain, null)
      score = wordRelevancy + Math.max(recencyScore, wordRelevancy) / 2
      results.push([domain, score])
    results.sort (a, b) -> b[1] - a[1]
    results

  populateDomains: (onComplete) ->
    HistoryCache.use (history) =>
      @domains = {}
      history.forEach (entry) =>
        # We want each key in our domains hash to point to the most recent History entry for that domain.
        domain = @parseDomain(entry.url)
        if domain
          previousEntry = @domains[domain]
          @domains[domain] = entry if !previousEntry || (previousEntry.lastVisitTime < entry.lastVisitTime)
      chrome.history.onVisited.addListener(@onPageVisited.proxy(this))
      onComplete()

  onPageVisited: (newPage) ->
    domain = @parseDomain(newPage.url)
    @domains[domain] = newPage if domain

  parseDomain: (url) -> url.split("/")[2] || ""

  # Suggestions from the Domain completer have the maximum relevancy. They should be shown first in the list.
  computeRelevancy: -> 1

class MultiCompleter
  constructor: (@completers) ->
    @maxResults = 10 # TODO(philc): Should this be configurable?

  refresh: -> completer.refresh() for completer in @completers when completer.refresh

  filter: (queryTerms, onComplete) ->
    suggestions = []
    completersFinished = 0
    for completer in @completers
      # Call filter() on every source completer and wait for them all to finish before returning results.
      completer.filter queryTerms, (newSuggestions) =>
        suggestions = suggestions.concat(newSuggestions)
        completersFinished += 1
        if completersFinished >= @completers.length
          results = @sortSuggestions(suggestions)[0...@maxResults]
          result.generateHtml() for result in results
          onComplete(results)

  sortSuggestions: (suggestions) ->
    for suggestion in suggestions
      suggestion.computeRelevancy(@queryTerms)
    suggestions.sort (a, b) -> b.relevancy - a.relevancy
    suggestions<commit_msg>Only allow one query to run at a time<commit_after>class Suggestion
  showRelevancy: true # Set this to true to render relevancy when debugging the ranking scores.

  # - type: one of [bookmark, history, tab].
  constructor: (@queryTerms, @type, @url, @title, @computeRelevancyFunction, @extraRelevancyData) ->
    @title ||= ""

  generateHtml: ->
    return @html if @html
    relevancyHtml = if @showRelevancy then "<span class='relevancy'>#{@computeRelevancy() + ''}</span>" else ""
    @html =
      "<div class='topHalf'>
         <span class='source'>#{@type}</span>
         <span class='title'>#{@highlightTerms(utils.escapeHtml(@title))}</span>
       </div>
       <div class='bottomHalf'>
        <span class='url'>#{@shortenUrl(@highlightTerms(@url))}</span>
        #{relevancyHtml}
      </div>"

  shortenUrl: (url) ->
    @stripTrailingSlash(url).replace(/^http:\/\//, "")

  stripTrailingSlash: (url) ->
    url = url.substring(url, url.length - 1) if url[url.length - 1] == "/"
    url

  # Merges the given list of ranges such that any overlapping regions are combined. E.g.
  #   mergeRanges([0, 4], [3, 6]) => [0, 6].  A range is [startIndex, endIndex].
  mergeRanges: (ranges) ->
    previous = ranges.shift()
    mergedRanges = [previous]
    ranges.forEach (range) ->
      if previous[1] >= range[0]
        previous[1] = Math.max(range[1], previous[1])
      else
        mergedRanges.push(range)
        previous = range
    mergedRanges

  # Wraps each occurence of the query terms in the given string in a <span>.
  highlightTerms: (string) ->
    ranges = []
    for term in @queryTerms
      regexp = @escapeRegexp(term)
      i = string.search(regexp)
      ranges.push([i, i + term.length]) if i >= 0

    return string if ranges.length == 0

    ranges = @mergeRanges(ranges.sort (a, b) -> a[0] - b[0])
    # Replace portions of the string from right to left.
    ranges = ranges.sort (a, b) -> b[0] - a[0]
    for [start, end] in ranges
      string =
        string.substring(0, start) +
        "<span class='match'>" + string.substring(start, end) + "</span>" +
        string.substring(end)
    string

  # Creates a Regexp from the given string, with all special Regexp characters escaped.
  escapeRegexp: (string) ->
    # Taken from http://stackoverflow.com/questions/3446170/escape-string-for-use-in-javascript-regex
    Suggestion.escapeRegExp ||= /[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g
    new RegExp(string.replace(Suggestion.escapeRegExp, "\\$&"), "i")

  computeRelevancy: -> @relevancy = @computeRelevancyFunction(@queryTerms, this)

class BookmarkCompleter
  currentSearch: null
  # These bookmarks are loaded asynchronously when refresh() is called.
  bookmarks: null

  filter: (@queryTerms, @onComplete) ->
    @currentSearch = { queryTerms: @queryTerms, onComplete: @onComplete }
    @performSearch() if @bookmarks

  onBookmarksLoaded: -> @performSearch() if @currentSearch

  performSearch: ->
    results = @bookmarks.filter (bookmark) =>
        RankingUtils.matches(@currentSearch.queryTerms, bookmark.url, bookmark.title)
    suggestions = results.map (bookmark) =>
      new Suggestion(@currentSearch.queryTerms, "bookmark", bookmark.url, bookmark.title, @computeRelevancy)
    onComplete = @currentSearch.onComplete
    @currentSearch = null
    onComplete(suggestions)

  refresh: ->
    @bookmarks = null
    chrome.bookmarks.getTree (bookmarks) =>
      @bookmarks = @traverseBookmarks(bookmarks).filter((bookmark) -> bookmark.url?)
      @onBookmarksLoaded()

  # Traverses the bookmark hierarchy, and retuns a flattened list of all bookmarks in the tree.
  traverseBookmarks: (bookmarks) ->
    results = []
    toVisit = bookmarks
    while toVisit.length > 0
      bookmark = toVisit.shift()
      results.push(bookmark)
      toVisit.push.apply(toVisit, bookmark.children) if (bookmark.children)
    results

  computeRelevancy: (queryTerms, suggestion) ->
    RankingUtils.wordRelevancy(queryTerms, suggestion.url, suggestion.title)

class HistoryCompleter
  filter: (queryTerms, onComplete) ->
    @currentSearch = { queryTerms: @queryTerms, onComplete: @onComplete }
    results = []
    HistoryCache.use (history) ->
      results = history.filter (entry) -> RankingUtils.matches(queryTerms, entry.url, entry.title)
    suggestions = results.map (entry) =>
      new Suggestion(queryTerms, "history", entry.url, entry.title, @computeRelevancy, entry)
    onComplete(suggestions)

  computeRelevancy: (queryTerms, suggestion) ->
    historyEntry = suggestion.extraRelevancyData
    recencyScore = RankingUtils.recencyScore(historyEntry.lastVisitTime)
    wordRelevancy = RankingUtils.wordRelevancy(queryTerms, suggestion.url, suggestion.title)
    # Average out the word score and the recency. Recency has the ability to pull the score up, but not down.
    score = (wordRelevancy + Math.max(recencyScore, wordRelevancy)) / 2

  refresh: ->

# The domain completer is designed to match a single-word query which looks like it is a domain. This supports
# the user experience where they quickly type a partial domain, hit tab -> enter, and expect to arrive there.
class DomainCompleter
  domains: null # A map of domain -> history

  filter: (queryTerms, onComplete) ->
    return onComplete([]) if queryTerms.length > 1
    if @domains
      @performSearch(queryTerms, onComplete)
    else
      @populateDomains => @performSearch(queryTerms, onComplete)

  performSearch: (queryTerms, onComplete) ->
    query = queryTerms[0]
    domainCandidates = (domain for domain of @domains when domain.indexOf(query) >= 0)
    domains = @sortDomainsByRelevancy(queryTerms, domainCandidates)
    return onComplete([]) if domains.length == 0
    topDomain = domains[0][0]
    onComplete([new Suggestion(queryTerms, "domain", topDomain, null, @computeRelevancy)])

  # Returns a list of domains of the form: [ [domain, relevancy], ... ]
  sortDomainsByRelevancy: (queryTerms, domainCandidates) ->
    results = []
    for domain in domainCandidates
      recencyScore = RankingUtils.recencyScore(@domains[domain].lastVisitTime || 0)
      wordRelevancy = RankingUtils.wordRelevancy(queryTerms, domain, null)
      score = wordRelevancy + Math.max(recencyScore, wordRelevancy) / 2
      results.push([domain, score])
    results.sort (a, b) -> b[1] - a[1]
    results

  populateDomains: (onComplete) ->
    HistoryCache.use (history) =>
      @domains = {}
      history.forEach (entry) =>
        # We want each key in our domains hash to point to the most recent History entry for that domain.
        domain = @parseDomain(entry.url)
        if domain
          previousEntry = @domains[domain]
          @domains[domain] = entry if !previousEntry || (previousEntry.lastVisitTime < entry.lastVisitTime)
      chrome.history.onVisited.addListener(@onPageVisited.proxy(this))
      onComplete()

  onPageVisited: (newPage) ->
    domain = @parseDomain(newPage.url)
    @domains[domain] = newPage if domain

  parseDomain: (url) -> url.split("/")[2] || ""

  # Suggestions from the Domain completer have the maximum relevancy. They should be shown first in the list.
  computeRelevancy: -> 1

class MultiCompleter
  constructor: (@completers) ->
    @maxResults = 10 # TODO(philc): Should this be configurable?

  refresh: -> completer.refresh() for completer in @completers when completer.refresh

  filter: (queryTerms, onComplete) ->
    # Only allow one query to run at a time.
    if @filterInProgress
      @mostRecentQuery = { queryTerms: queryTerms, onComplete: onComplete }
      return
    @mostRecentQuery = null
    @filterInProgress = true
    suggestions = []
    completersFinished = 0
    for completer in @completers
      # Call filter() on every source completer and wait for them all to finish before returning results.
      completer.filter queryTerms, (newSuggestions) =>
        suggestions = suggestions.concat(newSuggestions)
        completersFinished += 1
        if completersFinished >= @completers.length
          results = @sortSuggestions(suggestions)[0...@maxResults]
          result.generateHtml() for result in results
          onComplete(results)
          @filterInProgress = false
          @filter(@mostRecentQuery.queryTerms, @mostRecentQuery.onComplete) if @mostRecentQuery

  sortSuggestions: (suggestions) ->
    for suggestion in suggestions
      suggestion.computeRelevancy(@queryTerms)
    suggestions.sort (a, b) -> b.relevancy - a.relevancy
    suggestions
{'fraction_non_alphanumeric': 0.12041284403669725, 'fraction_numerical': 0.005552873008208595, 'mean_word_length': 3.4732721382289418, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 139, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2347374', 'n_tokens_mistral': 5171, 'n_tokens_neox': 5003, 'n_words': 1340}