<commit_before># encoding: utf-8
# this code must work under pythons 2.2 through 3.0

import sys
import os


version = (
  (sys.hexversion & (0xff << 24)) >> 24,
  (sys.hexversion & (0xff << 16)) >> 16
)

def sortedNoCase(p_array):
  "Sort an array case insensitevely, returns a sorted copy"
  p_array = list(p_array)    
  if version[0] < 3:
    def c(x, y):
      x = x.upper()
      y = y.upper()
      if x > y:
        return 1
      elif x < y:
        return -1
      else:
        return 0
    p_array.sort(c)
  else:
    p_array.sort(key=lambda x: x.upper())
    
  return p_array

def is_binary(path, f):
    suffixes = ('.so', '.pyd')
    for suf in suffixes:
      if f.endswith(suf):
        return True
    if f.endswith('.pyc') or f.endswith('.pyo'):
      fullname = os.path.join(path, f[:-1])
      return not os.path.exists(fullname)
    return False

def find_binaries(paths):
  """
  Finds binaries in the given list of paths. 
  Understands nested paths, as sys.paths have it (both "a/b" and "a/b/c").
  Tries to be case-insensitive, but case-preserving.
  @param paths a list of paths.
  @return a list like [(module_name: full_path),.. ]
  """
  SEP = os.path.sep
  res = {} # {name.upper(): (name, full_path)}
  if not paths:
    return {}
  if hasattr(os, "java"): # jython can't have binary modules
    return {} 
  paths = sortedNoCase(paths)
  for path in paths:
    for root, dirs, files in os.walk(path):
      cutpoint = path.rfind(SEP)
      if cutpoint > 0:
        preprefix = path[(cutpoint + len(SEP)):] + '.'
      else:
        preprefix = ''
      prefix = root[(len(path) + len(SEP)):].replace(SEP, '.')
      if prefix:
        prefix += '.'
      #print root, path, prefix, preprefix # XXX
      for f in files:
        if is_binary(root, f):
          name = f[:f.rindex('.')]
          #print "+++ ", name
          if preprefix:
            #print("prefixes: ", prefix, preprefix) # XXX
            pre_name = (preprefix + prefix + name).upper()
            if pre_name in res:
              res.pop(pre_name) # there might be a dupe, if paths got both a/b and a/b/c
            #print "+ ", name # XXX
          the_name = prefix + name
          res[the_name.upper()] = (the_name, root + SEP + f)
  return list(res.values())


# command-line interface<commit_msg>don't generate skeletons for stdlib tests on Mac<commit_after># encoding: utf-8
# this code must work under pythons 2.2 through 3.0

import sys
import os
import re

version = (
  (sys.hexversion & (0xff << 24)) >> 24,
  (sys.hexversion & (0xff << 16)) >> 16
)

def sortedNoCase(p_array):
  "Sort an array case insensitevely, returns a sorted copy"
  p_array = list(p_array)    
  if version[0] < 3:
    def c(x, y):
      x = x.upper()
      y = y.upper()
      if x > y:
        return 1
      elif x < y:
        return -1
      else:
        return 0
    p_array.sort(c)
  else:
    p_array.sort(key=lambda x: x.upper())
    
  return p_array

def is_binary(path, f):
    suffixes = ('.so', '.pyd')
    for suf in suffixes:
      if f.endswith(suf):
        return True
    if f.endswith('.pyc') or f.endswith('.pyo'):
      fullname = os.path.join(path, f[:-1])
      return not os.path.exists(fullname)
    return False

mac_stdlib_pattern = re.compile("/System/Library/Frameworks/Python\\.framework/Versions/(.+)/lib/python\\1/(.+)")
test_modules = ["test", "ctypes/test", "distutils/tests", "email/test",
                "importlib/test", "json/tests", "lib2to3/tests",
                "sqlite3/test", "tkinter/test"]

def is_stdlib_test(path, f):
    fullname = os.path.join(path, f)
    m = mac_stdlib_pattern.match(fullname)
    if not m: return 0
    relpath = m.group(2)
    for module in test_modules:
        if relpath.startswith(module): return 1
    return 0

def find_binaries(paths):
  """
  Finds binaries in the given list of paths. 
  Understands nested paths, as sys.paths have it (both "a/b" and "a/b/c").
  Tries to be case-insensitive, but case-preserving.
  @param paths a list of paths.
  @return a list like [(module_name: full_path),.. ]
  """
  SEP = os.path.sep
  res = {} # {name.upper(): (name, full_path)}
  if not paths:
    return {}
  if hasattr(os, "java"): # jython can't have binary modules
    return {} 
  paths = sortedNoCase(paths)
  for path in paths:
    for root, dirs, files in os.walk(path):
      cutpoint = path.rfind(SEP)
      if cutpoint > 0:
        preprefix = path[(cutpoint + len(SEP)):] + '.'
      else:
        preprefix = ''
      prefix = root[(len(path) + len(SEP)):].replace(SEP, '.')
      if prefix:
        prefix += '.'
      #print root, path, prefix, preprefix # XXX
      for f in files:
        if is_binary(root, f) and not is_stdlib_test(root, f):
          name = f[:f.rindex('.')]
          #print "+++ ", name
          if preprefix:
            #print("prefixes: ", prefix, preprefix) # XXX
            pre_name = (preprefix + prefix + name).upper()
            if pre_name in res:
              res.pop(pre_name) # there might be a dupe, if paths got both a/b and a/b/c
            #print "+ ", name # XXX
          the_name = prefix + name
          res[the_name.upper()] = (the_name, root + SEP + f)
  return list(res.values())


# command-line interface
{'fraction_non_alphanumeric': 0.14160472651038689, 'fraction_numerical': 0.009910425004764627, 'mean_word_length': 2.29855436832181, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '498340', 'n_tokens_mistral': 1875, 'n_tokens_neox': 1822, 'n_words': 527}