<commit_before>from collections import defaultdict

# TODO: handle negations (neg, advmod + neg word) specially!

# See: http://nlp.stanford.edu/software/dependencies_manual.pdf

MAX_PATH_LEN = 100

class DepPathDAG:
  def __init__(self, dep_parents, dep_paths, words, max_path_len=None, no_count_tags=('conj',), no_count_words=('_','*',)):
    self.max_path_len = max_path_len
    self.no_count_tags = tuple(no_count_tags)
    self.no_count_words = no_count_words
    self.words = words
    self.edges = defaultdict(list)
    self.edge_labels = {}
    for i,dp in enumerate(dep_parents):
      if dp >= 0:
        self.edges[i].append(dp-1)
        self.edges[dp-1].append(i)
        self.edge_labels[(i,dp-1)] = dep_paths[i]
        self.edge_labels[(dp-1,i)] = dep_paths[i]

  def _path_len(self, path):
    """Get the length of a list of nodes, skipping counting of certain dep path types"""
    l = 1
    for i in range(len(path)-1):
      if self.edge_labels[(path[i], path[i+1])].startswith(self.no_count_tags):
        continue
      if self.words[i] in self.no_count_words:
        continue
      l += 1
    return l

  def min_path(self, i, j, path=[]):
    if self.max_path_len and len(path) > self.max_path_len:
      return None
    min_path_len = MAX_PATH_LEN
    min_path = None
    for node in self.edges[i]:
      if node in path:<commit_msg>Fixed #145 --- off-by-one error gone<commit_after>from collections import defaultdict

# TODO: handle negations (neg, advmod + neg word) specially!

# See: http://nlp.stanford.edu/software/dependencies_manual.pdf

MAX_PATH_LEN = 100

class DepPathDAG:
  def __init__(self, dep_parents, dep_paths, words, max_path_len=None, no_count_tags=('conj',), no_count_words=('_','*',)):
    self.max_path_len = max_path_len
    self.no_count_tags = tuple(no_count_tags)
    self.no_count_words = no_count_words
    self.words = words
    self.edges = defaultdict(list)
    self.edge_labels = {}
    for i,dp in enumerate(dep_parents):
      if dp >= 0:
        self.edges[i].append(dp)
        self.edges[dp].append(i)
        self.edge_labels[(i,dp)] = dep_paths[i]
        self.edge_labels[(dp,i)] = dep_paths[i]

  def _path_len(self, path):
    """Get the length of a list of nodes, skipping counting of certain dep path types"""
    l = 1
    for i in range(len(path)-1):
      if self.edge_labels[(path[i], path[i+1])].startswith(self.no_count_tags):
        continue
      if self.words[i] in self.no_count_words:
        continue
      l += 1
    return l

  def min_path(self, i, j, path=[]):
    if self.max_path_len and len(path) > self.max_path_len:
      return None
    min_path_len = MAX_PATH_LEN
    min_path = None
    for node in self.edges[i]:
      if node in path:
{'fraction_non_alphanumeric': 0.1251372118551043, 'fraction_numerical': 0.008415660446395901, 'mean_word_length': 3.265210608424337, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3343658', 'n_tokens_mistral': 999, 'n_tokens_neox': 968, 'n_words': 225}