<commit_before>#!/usr/bin/env python

import os, sys, subprocess, pygments, itertools, string
from pygments import lexers

debug = ("--debug" in sys.argv[1:])
dry_run = ("--dry-run" in sys.argv[1:])
provided_files = [f for f in sys.argv[1:] if not f.startswith('-')]

def is_declaration(token) :
  """
  Checks if this token declares an identifier, so that identifier doesn't need
  to be imported
  """
  return token in [
      "class", "trait", "def", "val",
      "type", "var", "package", "object"]

def get_used_tokens(filename) :
  """
  Get all the tokens that are referenced in this file
  """
  lines = (line for line in open(filename, 'r'))
  lexer = lexers.get_lexer_for_filename(filename)
  seen = set([])

  used_tokens = []
  declared_tokens = set([])

  token_iter = lexer.get_tokens('\n'.join(lines))
  filtered_iter = (tok for tok in token_iter if tok[1].strip())

  prev_iter, curr_iter, next_iter = itertools.tee(filtered_iter, 3)

  # we miss the first token here... whatever
  next(curr_iter)
  next(next_iter)
  next(next_iter)

  for (prev, curr, next_tok) in itertools.izip(prev_iter, curr_iter, next_iter) :
    if str(prev[0]) in ['Token.Operator', 'Token.Keyword.Type'] and prev[1] == '.' :
      continue
    if (next_tok[1]) == u'=>' :
      continue
    if str(curr[0]) in ['Token.Name.Class', 'Token.Keyword.Type'] and curr[1][0].isupper() :
      if is_declaration(prev[1]) :
        declared_tokens.add(curr[1])
      elif curr not in seen :
        seen.add(curr)
        used_tokens.append(curr[1].split(".")[0])

  return [t for t in used_tokens if t not in declared_tokens]

def get_unimported_tokens(filename, imported_tokens) :
  used_tokens = get_used_tokens(filename)
  if debug :
    print '%d imported tokens: %s' % (len(imported_tokens), imported_tokens)
    print '%d used tokens: %s' % (len(used_tokens), used_tokens)
  as_set = set(imported_tokens)
  return [t for t in used_tokens if t not in as_set and needs_import(t)]

# Copied from the "scala" package, and also from Predef.scala
# And we add Value because it's frequently seen in enums
known_tokens = set([
  'AbstractMethodError',
  'Any',
  'AnyVal',
  'Array',
  'ArrayIndexOutOfBoundsException',
  'BigDecimal',
  'BigInt',
  'Boolean',
  'BufferedIterator',
  'Byte',
  'Char',
  'Class',
  'ClassCastException',
  'ClassManfiest',
  'Cloneable',
  'Console',
  'DelayedInit',
  'Double',
  'Dynamic',
  'Either',
  'Enumeration',
  'Equals',
  'Error',
  'Exception',
  'Float',
  'Function',
  'IllegalArgumentException',
  'Immutable',
  'IndexOutOfBoundsException',
  'IndexedSeq',
  'Int',
  'Integer',
  'Integral',
  'InterruptedException',
  'Iterable',
  'Iterator',
  'Left',
  'List',
  'Long',
  'Manifest',
  'Map',
  'MatchError',
  'Nil',
  'NoManifest',
  'NoSuchElementException',
  'None',
  'Numeric',
  'NullPointerException',
  'OptManifest',
  'Option',
  'Ordered',
  'Ordering',
  'Pair',<commit_msg>Add new ignored class<commit_after>#!/usr/bin/env python

import os, sys, subprocess, pygments, itertools, string
from pygments import lexers

debug = ("--debug" in sys.argv[1:])
dry_run = ("--dry-run" in sys.argv[1:])
provided_files = [f for f in sys.argv[1:] if not f.startswith('-')]

def is_declaration(token) :
  """
  Checks if this token declares an identifier, so that identifier doesn't need
  to be imported
  """
  return token in [
      "class", "trait", "def", "val",
      "type", "var", "package", "object"]

def get_used_tokens(filename) :
  """
  Get all the tokens that are referenced in this file
  """
  lines = (line for line in open(filename, 'r'))
  lexer = lexers.get_lexer_for_filename(filename)
  seen = set([])

  used_tokens = []
  declared_tokens = set([])

  token_iter = lexer.get_tokens('\n'.join(lines))
  filtered_iter = (tok for tok in token_iter if tok[1].strip())

  prev_iter, curr_iter, next_iter = itertools.tee(filtered_iter, 3)

  # we miss the first token here... whatever
  next(curr_iter)
  next(next_iter)
  next(next_iter)

  for (prev, curr, next_tok) in itertools.izip(prev_iter, curr_iter, next_iter) :
    if str(prev[0]) in ['Token.Operator', 'Token.Keyword.Type'] and prev[1] == '.' :
      continue
    if (next_tok[1]) == u'=>' :
      continue
    if str(curr[0]) in ['Token.Name.Class', 'Token.Keyword.Type'] and curr[1][0].isupper() :
      if is_declaration(prev[1]) :
        declared_tokens.add(curr[1])
      elif curr not in seen :
        seen.add(curr)
        used_tokens.append(curr[1].split(".")[0])

  return [t for t in used_tokens if t not in declared_tokens]

def get_unimported_tokens(filename, imported_tokens) :
  used_tokens = get_used_tokens(filename)
  if debug :
    print '%d imported tokens: %s' % (len(imported_tokens), imported_tokens)
    print '%d used tokens: %s' % (len(used_tokens), used_tokens)
  as_set = set(imported_tokens)
  return [t for t in used_tokens if t not in as_set and needs_import(t)]

# Copied from the "scala" package, and also from Predef.scala
# And we add Value because it's frequently seen in enums
known_tokens = set([
  'AbstractMethodError',
  'Any',
  'AnyVal',
  'Array',
  'ArrayIndexOutOfBoundsException',
  'BigDecimal',
  'BigInt',
  'Boolean',
  'BufferedIterator',
  'Byte',
  'Char',
  'Class',
  'ClassCastException',
  'ClassManfiest',
  'Cloneable',
  'Console',
  'DelayedInit',
  'Double',
  'Dynamic',
  'Either',
  'Enumeration',
  'Equals',
  'Error',
  'Exception',
  'Float',
  'Function',
  'IllegalArgumentException',
  'Immutable',
  'IndexOutOfBoundsException',
  'IndexedSeq',
  'Int',
  'Integer',
  'Integral',
  'InterruptedException',
  'Iterable',
  'Iterator',
  'Left',
  'List',
  'Long',
  'Manifest',
  'Map',
  'MatchError',
  'Nil',
  'NoManifest',
  'NoSuchElementException',
  'None',
  'Numeric',
  'NullPointerException',
  'Object',
  'OptManifest',
  'Option',
  'Ordered',
  'Ordering',
  'Pair',
{'fraction_non_alphanumeric': 0.15511382942575602, 'fraction_numerical': 0.0050968399592252805, 'mean_word_length': 3.9429051217464317, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2922222', 'n_tokens_mistral': 2125, 'n_tokens_neox': 2110, 'n_words': 537}