<commit_before>import sys

class Tag: # based on http://nlp.stanford.edu:8080/ner/process with classifier english.muc.7class.distsim.crf.ser.gz
  nill = 0
  location = 1
  time = 2
  person = 3
  organization = 4
  money = 5
  percent = 6
  date = 7
  

class Node:
  """ 
    One node of the parse tree.
    It is a group of words of same NamedEntityTag (e.g. George Washington).    
  """
  def __init__(self, word_list, namedentitytag, subnodes=[]):
    self.words = word_list
    self.child = subnodes
    if(namedentitytag=="O"):
      self.tag = Tag.nill
    if(namedentitytag=="LOCATION"):
      self.tag = Tag.location
    elif(namedentitytag=="TIME"):
      self.tag = Tag.time
    elif(namedentitytag=="PERSON"):
      self.tag = Tag.person
    elif(namedentitytag=="ORGANIZATION"):
      self.tag = Tag.organization
    elif(namedentitytag=="MONEY"):
      self.tag = Tag.money
    elif(namedentitytag=="PERCENT"):
      self.tag = Tag.percent
    elif(namedentitytag=="DATE"):
      self.tag = Tag.date
    else:
      self.tag = Tag.nill
      print("ERROR: unknown NamedEntityTag, set it to nill.",file=sys.stderr)<commit_msg>Add compute_tree.<commit_after>import sys
  

class Node:
  """ 
    One node of the parse tree.
    It is a group of words of same NamedEntityTag (e.g. George Washington).    
  """
  def __init__(self, word_list, namedentitytag='undef', dependency='undef', subnodes=[]):
    self.words = word_list      # List of words for this node (generally of size 1, at most 2 or 3)
    self.tag = namedentitytag   # String for the NamedEntityTag (e.g. 'PERSON' or 'DATE')
    self.dep = dependency       # Relation with the parent node (e.g. 'nn' or 'det' or 'root')
    self.child = subnodes       # List of children nodes
                                # parent attribute will be available after computaiton of the tree

      

def compute_tree(r):
  """
    Compute the dependence tree.
    Take in input a piece of the result produced by StanfordNLP.
    If foo is this result, then r = foo['sentences'][0]
  """
  name_to_nodes = {} # find the nodes with their original string
  for edge in r['indexeddependencies']:
    try:
      n1 = name_to_nodes[edge[1]]
    except KeyError:
      n1 = Node([edge[1]])
      name_to_nodes[edge[1]] = n1
    try:
      n2 = name_to_nodes[edge[2]]
    except KeyError:
      n2 = Node([edge[2]])
      name_to_nodes[edge[2]] = n2
    # n1 is the parent of n2
    n1.child = n1.child+[n2]
    n2.parent = n1
    n2.dependency = edge[0]
  index=1
  for word in r['words']:
    if word[0].isalnum() or word[0] == '$' or  word[0] == '%':
      w=word[0]+'-'+str(index) # key in the name_to_nodes map
      index+=1
      n = name_to_nodes[w]
      n.tag = word[1]['NamedEntityTag']
  return name_to_nodes['ROOT-0']
{'fraction_non_alphanumeric': 0.12495511669658887, 'fraction_numerical': 0.01651705565529623, 'mean_word_length': 2.7295850066934406, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4083245', 'n_tokens_mistral': 967, 'n_tokens_neox': 948, 'n_words': 264}