<commit_before># This file contains the definition of the completers used for the Vomnibox's suggestion UI. A completer will
# take a query (whatever the user typed into the Vomnibox) and return a list of Suggestions, e.g. bookmarks,
# domains, URLs from history.
#
# The Vomnibox frontend script makes a "filterCompleter" request to the background page, which in turn calls
# filter() on each these completers.
#
# A completer is a class which has three functions:
#  - filter(query, onComplete): "query" will be whatever the user typed into the Vomnibox.
#  - refresh(): (optional) refreshes the completer's data source (e.g. refetches the list of bookmarks).
#  - cancel(): (optional) cancels any pending, cancelable action.
class Suggestion
  showRelevancy: true # Set this to true to render relevancy when debugging the ranking scores.

  constructor: (@options) ->
    # Required options.
    @queryTerms = null
    @type = null
    @url = null
    @relevancyFunction = null
    # Other options.
    @title = ""
    # Extra data which will be available to the relevancy function.
    @relevancyData = null
    # If @autoSelect is truthy, then this suggestion is automatically pre-selected in the vomnibar.  There may
    # be at most one such suggestion.
    @autoSelect = false
    # If truthy (and @autoSelect is truthy too), then this suggestion is always pre-selected when the query
    # changes.  There may be at most one such suggestion.
    @forceAutoSelect = false
    # If @highlightTerms is true, then we highlight matched terms in the title and URL.
    @highlightTerms = true
    # If @insertText is a string, then the indicated text is inserted into the vomnibar input when the
    # suggestion is selected.
    @insertText = null

    extend this, @options

  computeRelevancy: ->
    # We assume that, once the relevancy has been set, it won't change.  Completers must set either @relevancy
    # or @relevancyFunction.
    @relevancy ?= @relevancyFunction this

  generateHtml: ->
    return @html if @html
    relevancyHtml = if @showRelevancy then "<span class='relevancy'>#{@computeRelevancy()}</span>" else ""
    # NOTE(philc): We're using these vimium-specific class names so we don't collide with the page's CSS.
    @html =
      """
      <div class="vimiumReset vomnibarTopHalf">
         <span class="vimiumReset vomnibarSource">#{@type}</span>
         <span class="vimiumReset vomnibarTitle">#{@highlightQueryTerms Utils.escapeHtml @title}</span>
       </div>
       <div class="vimiumReset vomnibarBottomHalf">
        <span class="vimiumReset vomnibarUrl">#{@highlightQueryTerms Utils.escapeHtml @shortenUrl()}</span>
        #{relevancyHtml}
      </div>
      """

  # Use neat trick to snatch a domain (http://stackoverflow.com/a/8498668).
  getUrlRoot: (url) ->
    a = document.createElement 'a'
    a.href = url
    a.protocol + "//" + a.hostname

  getHostname: (url) ->
    a = document.createElement 'a'
    a.href = url
    a.hostname

  stripTrailingSlash: (url) ->
    url = url.substring(url, url.length - 1) if url[url.length - 1] == "/"
    url

  # Push the ranges within `string` which match `term` onto `ranges`.
  pushMatchingRanges: (string,term,ranges) ->
    textPosition = 0
    # Split `string` into a (flat) list of pairs:
    #   - for i=0,2,4,6,...
    #     - splits[i] is unmatched text
    #     - splits[i+1] is the following matched text (matching `term`)
    #       (except for the final element, for which there is no following matched text).
    # Example:
    #   - string = "Abacab"
    #   - term = "a"
    #   - splits = [ "", "A",    "b", "a",    "c", "a",    b" ]
    #                UM   M       UM   M       UM   M      UM      (M=Matched, UM=Unmatched)
    splits = string.split(RegexpCache.get(term, "(", ")"))
    for index in [0..splits.length-2] by 2
      unmatchedText = splits[index]
      matchedText = splits[index+1]
      # Add the indices spanning `matchedText` to `ranges`.
      textPosition += unmatchedText.length
      ranges.push([textPosition, textPosition + matchedText.length])
      textPosition += matchedText.length

  # Wraps each occurence of the query terms in the given string in a <span>.
  highlightQueryTerms: (string) ->
    return string unless @highlightTerms
    ranges = []
    escapedTerms = @queryTerms.map (term) -> Utils.escapeHtml(term)
    for term in escapedTerms
      @pushMatchingRanges string, term, ranges

    return string if ranges.length == 0

    ranges = @mergeRanges(ranges.sort (a, b) -> a[0] - b[0])
    # Replace portions of the string from right to left.
    ranges = ranges.sort (a, b) -> b[0] - a[0]
    for [start, end] in ranges
      string =
        string.substring(0, start) +
        "<span class='vomnibarMatch'>#{string.substring(start, end)}</span>" +
        string.substring(end)
    string

  # Merges the given list of ranges such that any overlapping regions are combined. E.g.
  #   mergeRanges([0, 4], [3, 6]) => [0, 6].  A range is [startIndex, endIndex].
  mergeRanges: (ranges) ->
    previous = ranges.shift()
    mergedRanges = [previous]
    ranges.forEach (range) ->
      if previous[1] >= range[0]
        previous[1] = Math.max(range[1], previous[1])
      else
        mergedRanges.push(range)
        previous = range
    mergedRanges

  # Simplify a suggestion's URL (by removing those parts which aren't useful for display or comparison).
  shortenUrl: () ->
    return @shortUrl if @shortUrl?
    url = @url
    for [ filter, replacements ] in @stripPatterns
      if new RegExp(filter).test url
        for replace in replacements
          url = url.replace replace, ""
    @shortUrl = url

  # Patterns to strip from URLs; of the form [ [ filter, replacements ], [ filter, replacements ], ... ]
  #   - filter is a regexp string; a URL must match this regexp first.
  #   - replacements (itself a list) is a list of regexp objects, each of which is removed from URLs matching
  #     the filter.
  #
  # Note. This includes site-specific patterns for very-popular sites with URLs which don't work well in the
  # vomnibar.
  #
  stripPatterns: [
    # Google search specific replacements; this replaces query parameters which are known to not be helpful.
    # There's some additional information here: http://www.teknoids.net/content/google-search-parameters-2012
    [ "^https?://www\.google\.(com|ca|com\.au|co\.uk|ie)/.*[&?]q="
      "ei gws_rd url ved usg sa usg sig2 bih biw cd aqs ie".split(/\s+/).map (param) -> new RegExp "\&#{param}=[^&]+" ]

    # General replacements; replaces leading and trailing fluff.
    [ '.', [ "^https?://", "\\W+$" ].map (re) -> new RegExp re ]
  ]

class BookmarkCompleter
  folderSeparator: "/"
  currentSearch: null
  # These bookmarks are loaded asynchronously when refresh() is called.
  bookmarks: null

  filter: ({ @queryTerms }, @onComplete) ->
    @currentSearch = { queryTerms: @queryTerms, onComplete: @onComplete }
    @performSearch() if @bookmarks

  onBookmarksLoaded: -> @performSearch() if @currentSearch

  performSearch: ->
    # If the folder separator character the first character in any query term, then we'll use the bookmark's full path as its title.<commit_msg>Add a couple more useless Google search parameters.<commit_after># This file contains the definition of the completers used for the Vomnibox's suggestion UI. A completer will
# take a query (whatever the user typed into the Vomnibox) and return a list of Suggestions, e.g. bookmarks,
# domains, URLs from history.
#
# The Vomnibox frontend script makes a "filterCompleter" request to the background page, which in turn calls
# filter() on each these completers.
#
# A completer is a class which has three functions:
#  - filter(query, onComplete): "query" will be whatever the user typed into the Vomnibox.
#  - refresh(): (optional) refreshes the completer's data source (e.g. refetches the list of bookmarks).
#  - cancel(): (optional) cancels any pending, cancelable action.
class Suggestion
  showRelevancy: true # Set this to true to render relevancy when debugging the ranking scores.

  constructor: (@options) ->
    # Required options.
    @queryTerms = null
    @type = null
    @url = null
    @relevancyFunction = null
    # Other options.
    @title = ""
    # Extra data which will be available to the relevancy function.
    @relevancyData = null
    # If @autoSelect is truthy, then this suggestion is automatically pre-selected in the vomnibar.  There may
    # be at most one such suggestion.
    @autoSelect = false
    # If truthy (and @autoSelect is truthy too), then this suggestion is always pre-selected when the query
    # changes.  There may be at most one such suggestion.
    @forceAutoSelect = false
    # If @highlightTerms is true, then we highlight matched terms in the title and URL.
    @highlightTerms = true
    # If @insertText is a string, then the indicated text is inserted into the vomnibar input when the
    # suggestion is selected.
    @insertText = null

    extend this, @options

  computeRelevancy: ->
    # We assume that, once the relevancy has been set, it won't change.  Completers must set either @relevancy
    # or @relevancyFunction.
    @relevancy ?= @relevancyFunction this

  generateHtml: ->
    return @html if @html
    relevancyHtml = if @showRelevancy then "<span class='relevancy'>#{@computeRelevancy()}</span>" else ""
    # NOTE(philc): We're using these vimium-specific class names so we don't collide with the page's CSS.
    @html =
      """
      <div class="vimiumReset vomnibarTopHalf">
         <span class="vimiumReset vomnibarSource">#{@type}</span>
         <span class="vimiumReset vomnibarTitle">#{@highlightQueryTerms Utils.escapeHtml @title}</span>
       </div>
       <div class="vimiumReset vomnibarBottomHalf">
        <span class="vimiumReset vomnibarUrl">#{@highlightQueryTerms Utils.escapeHtml @shortenUrl()}</span>
        #{relevancyHtml}
      </div>
      """

  # Use neat trick to snatch a domain (http://stackoverflow.com/a/8498668).
  getUrlRoot: (url) ->
    a = document.createElement 'a'
    a.href = url
    a.protocol + "//" + a.hostname

  getHostname: (url) ->
    a = document.createElement 'a'
    a.href = url
    a.hostname

  stripTrailingSlash: (url) ->
    url = url.substring(url, url.length - 1) if url[url.length - 1] == "/"
    url

  # Push the ranges within `string` which match `term` onto `ranges`.
  pushMatchingRanges: (string,term,ranges) ->
    textPosition = 0
    # Split `string` into a (flat) list of pairs:
    #   - for i=0,2,4,6,...
    #     - splits[i] is unmatched text
    #     - splits[i+1] is the following matched text (matching `term`)
    #       (except for the final element, for which there is no following matched text).
    # Example:
    #   - string = "Abacab"
    #   - term = "a"
    #   - splits = [ "", "A",    "b", "a",    "c", "a",    b" ]
    #                UM   M       UM   M       UM   M      UM      (M=Matched, UM=Unmatched)
    splits = string.split(RegexpCache.get(term, "(", ")"))
    for index in [0..splits.length-2] by 2
      unmatchedText = splits[index]
      matchedText = splits[index+1]
      # Add the indices spanning `matchedText` to `ranges`.
      textPosition += unmatchedText.length
      ranges.push([textPosition, textPosition + matchedText.length])
      textPosition += matchedText.length

  # Wraps each occurence of the query terms in the given string in a <span>.
  highlightQueryTerms: (string) ->
    return string unless @highlightTerms
    ranges = []
    escapedTerms = @queryTerms.map (term) -> Utils.escapeHtml(term)
    for term in escapedTerms
      @pushMatchingRanges string, term, ranges

    return string if ranges.length == 0

    ranges = @mergeRanges(ranges.sort (a, b) -> a[0] - b[0])
    # Replace portions of the string from right to left.
    ranges = ranges.sort (a, b) -> b[0] - a[0]
    for [start, end] in ranges
      string =
        string.substring(0, start) +
        "<span class='vomnibarMatch'>#{string.substring(start, end)}</span>" +
        string.substring(end)
    string

  # Merges the given list of ranges such that any overlapping regions are combined. E.g.
  #   mergeRanges([0, 4], [3, 6]) => [0, 6].  A range is [startIndex, endIndex].
  mergeRanges: (ranges) ->
    previous = ranges.shift()
    mergedRanges = [previous]
    ranges.forEach (range) ->
      if previous[1] >= range[0]
        previous[1] = Math.max(range[1], previous[1])
      else
        mergedRanges.push(range)
        previous = range
    mergedRanges

  # Simplify a suggestion's URL (by removing those parts which aren't useful for display or comparison).
  shortenUrl: () ->
    return @shortUrl if @shortUrl?
    url = @url
    for [ filter, replacements ] in @stripPatterns
      if new RegExp(filter).test url
        for replace in replacements
          url = url.replace replace, ""
    @shortUrl = url

  # Patterns to strip from URLs; of the form [ [ filter, replacements ], [ filter, replacements ], ... ]
  #   - filter is a regexp string; a URL must match this regexp first.
  #   - replacements (itself a list) is a list of regexp objects, each of which is removed from URLs matching
  #     the filter.
  #
  # Note. This includes site-specific patterns for very-popular sites with URLs which don't work well in the
  # vomnibar.
  #
  stripPatterns: [
    # Google search specific replacements; this replaces query parameters which are known to not be helpful.
    # There's some additional information here: http://www.teknoids.net/content/google-search-parameters-2012
    [ "^https?://www\.google\.(com|ca|com\.au|co\.uk|ie)/.*[&?]q="
      "ei gws_rd url ved usg sa usg sig2 bih biw cd aqs ie sourceid es_sm"
        .split(/\s+/).map (param) -> new RegExp "\&#{param}=[^&]+" ]

    # General replacements; replaces leading and trailing fluff.
    [ '.', [ "^https?://", "\\W+$" ].map (re) -> new RegExp re ]
  ]

class BookmarkCompleter
  folderSeparator: "/"
  currentSearch: null
  # These bookmarks are loaded asynchronously when refresh() is called.
  bookmarks: null

  filter: ({ @queryTerms }, @onComplete) ->
    @currentSearch = { queryTerms: @queryTerms, onComplete: @onComplete }
    @performSearch() if @bookmarks

  onBookmarksLoaded: -> @performSearch() if @currentSearch

  performSearch: ->
    # If the folder separator character the first character in any query term, then we'll use the bookmark's full path as its title.
{'fraction_non_alphanumeric': 0.11253108593534125, 'fraction_numerical': 0.0056645482177397075, 'mean_word_length': 3.001381978993919, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 75, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3206333', 'n_tokens_mistral': 4567, 'n_tokens_neox': 4396, 'n_words': 1539}