<commit_before>import heapq
import os
import re

# Now take the n best matches from amongst the filenames which contain
# search, ranked as follows:
# 1) Full matches.
# 2) Matches anchored at the beginning.
# 3) Matches which begin with a capital letter (e.g. matching "bar" in
#    "FooBar.cpp") or for which the preceeding char is non-alpha.
# 4) Matches anchored at the end.
# 5) Matches anchored right before ".".
# 6) Matches which are immediately followed by a non-lower-case-alpha char.
# 7) Matches with shorter filenames.
#
# If the search string does not contain "/", we consider only filenames;
# otherwise we consider the full path.
def ItemRank(f, search_lower):
  """
  >>> ItemRank('dir/Tool.cpp', 'tool') > ItemRank('dir/Tools.cpp', 'tool')
  True
  >>> ItemRank('dir/ToolFoo', 'tool') > ItemRank('dir/Toolfoo', 'tool')
  True
  >>> ItemRank('dir/Tool.cpp', 'tool') > ItemRank('dir/ToolFoo.cpp', 'tool')
  True
  >>> ItemRank('dir/Tool.cpp', 'tool') > ItemRank('dir/Toool.cpp', 'tool')
  True
  """
  filename_len = len(os.path.basename(f))
  if '/' not in search_lower:
    f = os.path.basename(f)

  start_idx = f.lower().find(search_lower)
  end_idx = start_idx + len(search_lower)  # exclusive
  if start_idx == -1:
    return (False,)
  beg_match = start_idx == 0
  end_match = end_idx == len(f)
  full_match = beg_match and end_match
  beg_token = f[start_idx].isupper() or start_idx == 0 or (
      start_idx > 0 and not f[start_idx - 1].isalpha())
  end_dot = end_idx < len(f) and f[end_idx] == '.'
  end_token = end_idx < len(f) and not f[end_idx].islower()
  return (True, full_match, beg_match, beg_token,
          end_match, end_dot, end_token, -filename_len)

def CtrlPPyMatch():
  import vim

  items = vim.eval('a:items')
  astr = vim.eval('a:str')
  limit = int(vim.eval('a:limit'))
  mmode = vim.eval('a:mmode')
  aregex = int(vim.eval('a:regex'))
  rez = vim.eval('s:rez')

  if aregex == 1:
    raise ValueError('Regexp matches are not supported.')

  # TODO: Do something with mmode.  Possible values are 'filename-only',
  # 'first-non-tab', 'until-last-tab', and one other (unsure what).

  key_pairs = ((ItemRank(f, astr.lower()), f) for f in items)
  rez = (f for (k, f)
         in heapq.nlargest(limit, key_pairs, lambda kp: kp[0])
         if k[0])  # filter out non-matches

  # Use double quoted vim strings, and escape \
  vimrez = ('"%s"' % f.replace('\\', '\\\\').replace('"', '\\"') for f in rez)

  vim.command('let s:rez = [%s]' % ','.join(vimrez))


if __name__ == "__main__":
    import doctest
    doctest.testmod()<commit_msg>Dramatically improve performance by not emitting sort keys for non-matches.<commit_after>import heapq
import os
import re

# If the search string does not contain "/", we consider only filenames;
# otherwise we consider the full path.
def ItemRanks(search, items):
  """
  Get (sort_key, item) tuples for entries in items.

  For each element of items which matches search, generate a tuple (sort_key,
  item) such that elements with larget sort_keys are better matches for search.

  Matches are ranked as follows:
  1) Full matches.
  2) Matches anchored at the beginning.
  3) Matches which begin with a capital letter (e.g. matching "bar" in
     "FooBar.cpp") or for which the preceeding char is non-alpha.
  4) Matches anchored at the end.
  5) Matches anchored right before ".".
  6) Matches which are immediately followed by a non-lower-case-alpha char.
  7) Matches with shorter filenames.

  >>> list(ItemRanks('tool', ['dir/Tool.cpp']))[0][1] > list(ItemRanks('tool', ['dir/Tools.cpp']))[0][1]
  True
  >>> list(ItemRanks('tool', ['dir/ToolFoo']))[0][1] > list(ItemRanks('tool', ['dir/Toolfoo']))[0][1]
  True
  >>> list(ItemRanks('tool', ['dir/Tool.cpp']))[0][1] > list(ItemRanks('tool', ['dir/ToolFoo.cpp']))[0][1]
  True
  >>> len(list(ItemRanks('tool', ['dir/Toool.cpp'])))
  0
  """
  search_lower = search.lower()

  for i in items:
    f = i.lower()
    if search_lower not in f:
      # Bail early if we can, to avoid calling os.path.basename.
      continue

    if '/' not in search_lower:
      f = os.path.basename(f)

    start_idx = f.lower().find(search_lower)
    end_idx = start_idx + len(search_lower)  # exclusive
    if start_idx == -1:
      continue
    beg_match = start_idx == 0
    end_match = end_idx == len(f)
    full_match = beg_match and end_match
    beg_token = f[start_idx].isupper() or start_idx == 0 or (
        start_idx > 0 and not f[start_idx - 1].isalpha())
    end_dot = end_idx < len(f) and f[end_idx] == '.'
    end_token = end_idx < len(f) and not f[end_idx].islower()
    filename_len = len(os.path.basename(f))
    yield ((True, full_match, beg_match, beg_token,
            end_match, end_dot, end_token, -filename_len), i)

def CtrlPPyMatch():
  import vim

  items = vim.eval('a:items')
  astr = vim.eval('a:str')
  limit = int(vim.eval('a:limit'))
  mmode = vim.eval('a:mmode')
  aregex = int(vim.eval('a:regex'))
  rez = vim.eval('s:rez')

  if aregex == 1:
    raise ValueError('Regexp matches are not supported.')

  # TODO: Do something with mmode.  Possible values are 'filename-only',
  # 'first-non-tab', 'until-last-tab', and one other (unsure what).

  rez = (f for (k, f)
         in heapq.nlargest(limit, ItemRanks(astr, items), lambda kp: kp[0]))

  # Use double quoted vim strings, and escape \
  vimrez = ('"%s"' % f.replace('\\', '\\\\').replace('"', '\\"') for f in rez)

  vim.command('let s:rez = [%s]' % ','.join(vimrez))


if __name__ == "__main__":
    import doctest
    doctest.testmod()
{'fraction_non_alphanumeric': 0.1564638098667627, 'fraction_numerical': 0.007562117392870004, 'mean_word_length': 3.5421095666394113, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '619722', 'n_tokens_mistral': 2035, 'n_tokens_neox': 1963, 'n_words': 576}