<commit_before>#!/usr/bin/python
import os
import sys
from elementtree import ElementTree

shutup = ' 1> /dev/null 2> /dev/null'
shortcuts = {}
actions = []
sources = {}
targets = {}
implementations = {}
testset = []
graph_big = []
graph_small = []
log = None
tools = {}
failednode = []
failedarc  = []
failedaction = []

def postfix2prefix(post):
 #  input: 'x.c.b.a'
 # output: 'a b c x'
 pre = post.split('.')
 pre.reverse()
 return ' '.join(pre)

def prefix2postfix(pre):
 #  input: 'a b c x'
 # output: 'x.c.b.a'
 post = pre.split()
 post.reverse()
 return '.'.join(post)

def logwrite(s):
 log.write(s+'\n')
 log.flush()

def sysexit(n):
 log.close()
 sys.exit(n)

def readxmlconfig (cfg):
 config = ElementTree.parse(cfg)
 # shortcuts
 for outline in config.findall('//shortcut'):
  shortcuts[outline.findtext('name')]=expandxml(outline.findall('expansion')[0],{})
 # actions
 for outline in config.findall('//target/branch/perform'):
  if outline.text not in actions:
   actions.append(outline.text)
 # sources
 for outline in config.findall('//source'):
  args = [expandxml(outline.findall('extraction/name')[0],{})]
  for arg in outline.findall('extraction/argument'):
   args.append(expandxml(arg,{}))
  sources[outline.findtext('name')]=args
  pcmd = ecmd = ''
  if outline.findall('parsing/command'):
   pcmd = expandxml(outline.findall('parsing/command')[0],{})
  if outline.findall('evaluation/command'):
   ecmd = expandxml(outline.findall('evaluation/command')[0],{})
  if pcmd and ecmd:
   implementations[outline.findtext('name')]=[pcmd,ecmd]
 # targets
 for outline in config.findall('//target'):
  name = outline.findtext('name')
  targets[name]= [[],'']
  for br in outline.findall('branch'):
   branch = [br.findtext('input')]
   for p in br.findall('perform'):
    branch.append(p.text)
   targets[name][0].append(branch)
 # tools
 for outline in config.findall('//tool'):
  cmd = outline.findall('command')[0]
  line = expandxml(cmd,{})
  if cmd.attrib.has_key('out'):
   line += ' 1> '+cmd.attrib['out']
  if cmd.attrib.has_key('err'):
   line += ' 2> '+cmd.attrib['err']
  tools[outline.findtext('name')]=line

 print 'Read',len(shortcuts),'shortcuts,',len(tools),'tools,',len(actions),'actions,',
 print len(sources),'sources','('+`len(implementations)`,'implemented),',len(targets),'targets.'

def expandone(tag,text,rep):
 if text:
  wte = text
 else:
  wte = tag.replace('expand-','')
 if shortcuts.has_key(wte):
  return shortcuts[wte]
 elif rep.has_key(wte):
  return rep[wte]
 else:
  # postpone expanding
  return '%'+wte+'%'

def expandxml (mixed,rep):
 s = mixed.text
 for tag in mixed.getchildren():
  s += expandone(tag.tag,tag.text,rep)
  s += tag.tail
 return s.strip()

def expanduni(where,rep):
 cut = where.split('%')
 for i in range(0,len(cut)):
  if i%2:
   if shortcuts.has_key(cut[i]):
    cut[i]=shortcuts[cut[i]]
   elif rep.has_key(cut[i]):
    cut[i]=rep[cut[i]]
   else:
    print 'Misused expand, referencing undefined "'+cut[i]+'":'
    print '?????',where
    sysexit(11)
 return ''.join(cut)

def quote(a):
 return '"'+a+'"'

def addarc(fromnode,tonode,q,labelnode):
 if [fromnode,tonode,q,labelnode] not in graph_big:
  graph_big.append([fromnode,tonode,q,labelnode])

def makegraph():
 # first we generate a complete picture
 for x in targets.keys():
  for src in targets[x][0]:
   if len(src)==1:
    addarc(src[0],x,'','')
   else:
    name  = src[0]
    qname = src[0]
    for i in range(1,len(src)-1):
     addarc(name,name+"'",qname,src[i])
     qname += '.'+src[i]
     name += "'"
    addarc(name,x,qname,src[-1])
 # make a simplified one
 for x in targets.keys():
  for src in targets[x][0]:
   graph_small.append([src[0],x])

def dumpgraph(df):
 dot = open(df+'_large.dot','w')
 dot.write('digraph generated{ {rank=same; node [shape=ellipse];')
 for x in sources.keys():
  dot.write(quote(x))
  if x in failednode:
   dot.write(' [color=red]')
  dot.write(';')
 dot.write('}\n')
 dot.write('node [shape=octagon];\n')
 for x in targets.keys():
  dot.write(quote(x))
  if x in failednode:
   dot.write(' [color=red]')
  dot.write(';')
 dot.write('node [shape=box];\n')
 nodezz=[]
 for arc in graph_big:
  dot.write(quote(arc[0])+'->'+quote(arc[1]))
  if arc[0] not in nodezz:
   nodezz.append(arc[0])
  if arc[1] not in nodezz:
   nodezz.append(arc[1])
  par = ''
  if arc[3]:
   par += 'label="'+arc[3]+'" '
  if [arc[2],arc[3]] in failedarc:
   par += 'color=red '
  if par:
   dot.write(' ['+par+']')
  dot.write(';\n')
 for node in nodezz:
  if node not in sources.keys():
   if node not in targets.keys():
    if node in failednode:
     dot.write(quote(node)+' [color=red];')
 dot.write('}')
 dot.close()
 run = 'dot -Tpdf '+dot.name+' -o '+df+'_large.pdf'
 logwrite(run)<commit_msg>LCI v1.8 - sources and targets made bolder<commit_after>#!/usr/bin/python
import os
import sys
from elementtree import ElementTree

shutup = ' 1> /dev/null 2> /dev/null'
shortcuts = {}
actions = []
sources = {}
targets = {}
implementations = {}
testset = []
graph_big = []
graph_small = []
log = None
tools = {}
failednode = []
failedarc  = []
failedaction = []

def postfix2prefix(post):
 #  input: 'x.c.b.a'
 # output: 'a b c x'
 pre = post.split('.')
 pre.reverse()
 return ' '.join(pre)

def prefix2postfix(pre):
 #  input: 'a b c x'
 # output: 'x.c.b.a'
 post = pre.split()
 post.reverse()
 return '.'.join(post)

def logwrite(s):
 log.write(s+'\n')
 log.flush()

def sysexit(n):
 log.close()
 sys.exit(n)

def readxmlconfig (cfg):
 config = ElementTree.parse(cfg)
 # shortcuts
 for outline in config.findall('//shortcut'):
  shortcuts[outline.findtext('name')]=expandxml(outline.findall('expansion')[0],{})
 # actions
 for outline in config.findall('//target/branch/perform'):
  if outline.text not in actions:
   actions.append(outline.text)
 # sources
 for outline in config.findall('//source'):
  args = [expandxml(outline.findall('extraction/name')[0],{})]
  for arg in outline.findall('extraction/argument'):
   args.append(expandxml(arg,{}))
  sources[outline.findtext('name')]=args
  pcmd = ecmd = ''
  if outline.findall('parsing/command'):
   pcmd = expandxml(outline.findall('parsing/command')[0],{})
  if outline.findall('evaluation/command'):
   ecmd = expandxml(outline.findall('evaluation/command')[0],{})
  if pcmd and ecmd:
   implementations[outline.findtext('name')]=[pcmd,ecmd]
 # targets
 for outline in config.findall('//target'):
  name = outline.findtext('name')
  targets[name]= [[],'']
  for br in outline.findall('branch'):
   branch = [br.findtext('input')]
   for p in br.findall('perform'):
    branch.append(p.text)
   targets[name][0].append(branch)
 # tools
 for outline in config.findall('//tool'):
  cmd = outline.findall('command')[0]
  line = expandxml(cmd,{})
  if cmd.attrib.has_key('out'):
   line += ' 1> '+cmd.attrib['out']
  if cmd.attrib.has_key('err'):
   line += ' 2> '+cmd.attrib['err']
  tools[outline.findtext('name')]=line

 print 'Read',len(shortcuts),'shortcuts,',len(tools),'tools,',len(actions),'actions,',
 print len(sources),'sources','('+`len(implementations)`,'implemented),',len(targets),'targets.'

def expandone(tag,text,rep):
 if text:
  wte = text
 else:
  wte = tag.replace('expand-','')
 if shortcuts.has_key(wte):
  return shortcuts[wte]
 elif rep.has_key(wte):
  return rep[wte]
 else:
  # postpone expanding
  return '%'+wte+'%'

def expandxml (mixed,rep):
 s = mixed.text
 for tag in mixed.getchildren():
  s += expandone(tag.tag,tag.text,rep)
  s += tag.tail
 return s.strip()

def expanduni(where,rep):
 cut = where.split('%')
 for i in range(0,len(cut)):
  if i%2:
   if shortcuts.has_key(cut[i]):
    cut[i]=shortcuts[cut[i]]
   elif rep.has_key(cut[i]):
    cut[i]=rep[cut[i]]
   else:
    print 'Misused expand, referencing undefined "'+cut[i]+'":'
    print '?????',where
    sysexit(11)
 return ''.join(cut)

def quote(a):
 return '"'+a+'"'

def addarc(fromnode,tonode,q,labelnode):
 if [fromnode,tonode,q,labelnode] not in graph_big:
  graph_big.append([fromnode,tonode,q,labelnode])

def makegraph():
 # first we generate a complete picture
 for x in targets.keys():
  for src in targets[x][0]:
   if len(src)==1:
    addarc(src[0],x,'','')
   else:
    name  = src[0]
    qname = src[0]
    for i in range(1,len(src)-1):
     addarc(name,name+"'",qname,src[i])
     qname += '.'+src[i]
     name += "'"
    addarc(name,x,qname,src[-1])
 # make a simplified one
 for x in targets.keys():
  for src in targets[x][0]:
   graph_small.append([src[0],x])

def dumpgraph(df):
 dot = open(df+'_large.dot','w')
 dot.write('digraph generated{ {rank=same; node [shape=ellipse, style=bold];')
 for x in sources.keys():
  dot.write(quote(x))
  if x in failednode:
   dot.write(' [color=red]')
  dot.write(';')
 dot.write('}\n')
 dot.write('node [shape=octagon, style=bold];\n')
 for x in targets.keys():
  dot.write(quote(x))
  if x in failednode:
   dot.write(' [color=red]')
  dot.write(';')
 dot.write('node [shape=box, style=solid];\n')
 nodezz=[]
 for arc in graph_big:
  dot.write(quote(arc[0])+'->'+quote(arc[1]))
  if arc[0] not in nodezz:
   nodezz.append(arc[0])
  if arc[1] not in nodezz:
   nodezz.append(arc[1])
  par = ''
  if arc[3]:
   par += 'label="'+arc[3]+'" '
  if [arc[2],arc[3]] in failedarc:
   par += 'color=red '
  if par:
   dot.write(' ['+par+']')
  dot.write(';\n')
 for node in nodezz:
  if node not in sources.keys():
   if node not in targets.keys():
    if node in failednode:
     dot.write(quote(node)+' [color=red];')
 dot.write('}')
 dot.close()
 run = 'dot -Tpdf '+dot.name+' -o '+df+'_large.pdf'
 logwrite(run)
{'fraction_non_alphanumeric': 0.20175438596491227, 'fraction_numerical': 0.007727652464494569, 'mean_word_length': 4.311702717692734, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 16}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5274943', 'n_tokens_mistral': 3681, 'n_tokens_neox': 3635, 'n_words': 772}