<commit_before>from . import ast

class Analyzer(object):
	
	def __init__(self):
		self.vars = None
		
	def visit(self, node):
		
		if node is None:
			return
		
		if hasattr(self, node.__class__.__name__):
			getattr(self, node.__class__.__name__)(node)
			return
		
		for k in node.fields:
			attr = getattr(node, k)
			if isinstance(attr, list):
				for v in attr:
					self.visit(v)
			else:
				self.visit(attr)
	
	def Name(self, node):
		self.vars[0].add(node.name)
	
	def Tuple(self, node):
		for n in node.values:
			self.visit(n)
	
	def Attrib(self, node):
		self.visit(node.obj)
	
	def SetAttr(self, node):
		self.visit(node.obj)
	
	def As(self, node):
		self.visit(node.left)
	
	def LoopSetup(self, node):
		self.visit(node.loop.source)
	
	def LoopHeader(self, node):
		self.vars[1].add(node.lvar.name)
	
	def Assign(self, node):
		if isinstance(node.left, ast.Tuple):
			for n in node.left.values:
				if isinstance(n, ast.Name):
					self.vars[1].add(n.name)
				else:
					self.visit(n)
		elif isinstance(node.left, ast.Name):
			self.vars[1].add(node.left.name)
		else:
			self.visit(node.left)
		self.visit(node.right)
	
	def Call(self, node):
		self.visit(node.name)
		for arg in node.args:
			self.visit(arg)
	
	def Phi(self, node):
		self.visit(node.left[1])
		self.visit(node.right[1])

def defined(name, bl, seen):
	
	if name in bl.assigns:
		return {bl.id}
	elif not bl.preds:
		return {None}
	
	all = set()
	for p in bl.preds:
		if p.id not in seen:
			all.update(defined(name, p, seen | {bl.id}))
	
	return all

def liveness(mod):
	
	analyzer = Analyzer()
	for fname, code in mod.code:
		
		refs = {}
		for id, bl in code.flow.blocks.iteritems():
			
			bl.uses = {}
			bl.assigns = {}
			
			for i, step in enumerate(bl.steps):
				
				analyzer.vars = set(), set()
				analyzer.visit(step)
				
				for name in analyzer.vars[0]:
					bl.uses.setdefault(name, set()).add(i)
					refs.setdefault(id, []).append((i, name))
				
				for name in analyzer.vars[1]:
					bl.assigns.setdefault(name, set()).add(i)
					refs.setdefault(id, []).append((i, name))
		
		for id, bl in sorted(code.flow.blocks.iteritems()):
			
			bl.origin = {}
			for sid, name in refs.get(id, []):
				
				origin = bl.origin[name, sid] = set()
				if name in bl.assigns and min(bl.assigns[name]) < sid:
					origin.add(id)
					continue
				elif not bl.preds:
					origin.add(None)
					continue
				<commit_msg>Extract and sort blocks once in liveness analysis<commit_after>from . import ast

class Analyzer(object):
	
	def __init__(self):
		self.vars = None
		
	def visit(self, node):
		
		if node is None:
			return
		
		if hasattr(self, node.__class__.__name__):
			getattr(self, node.__class__.__name__)(node)
			return
		
		for k in node.fields:
			attr = getattr(node, k)
			if isinstance(attr, list):
				for v in attr:
					self.visit(v)
			else:
				self.visit(attr)
	
	def Name(self, node):
		self.vars[0].add(node.name)
	
	def Tuple(self, node):
		for n in node.values:
			self.visit(n)
	
	def Attrib(self, node):
		self.visit(node.obj)
	
	def SetAttr(self, node):
		self.visit(node.obj)
	
	def As(self, node):
		self.visit(node.left)
	
	def LoopSetup(self, node):
		self.visit(node.loop.source)
	
	def LoopHeader(self, node):
		self.vars[1].add(node.lvar.name)
	
	def Assign(self, node):
		if isinstance(node.left, ast.Tuple):
			for n in node.left.values:
				if isinstance(n, ast.Name):
					self.vars[1].add(n.name)
				else:
					self.visit(n)
		elif isinstance(node.left, ast.Name):
			self.vars[1].add(node.left.name)
		else:
			self.visit(node.left)
		self.visit(node.right)
	
	def Call(self, node):
		self.visit(node.name)
		for arg in node.args:
			self.visit(arg)
	
	def Phi(self, node):
		self.visit(node.left[1])
		self.visit(node.right[1])

def defined(name, bl, seen):
	
	if name in bl.assigns:
		return {bl.id}
	elif not bl.preds:
		return {None}
	
	all = set()
	for p in bl.preds:
		if p.id not in seen:
			all.update(defined(name, p, seen | {bl.id}))
	
	return all

def liveness(mod):
	
	analyzer = Analyzer()
	for fname, code in mod.code:
		
		refs, blocks = {}, sorted(code.flow.blocks.iteritems())
		for id, bl in blocks:
			
			bl.uses = {}
			bl.assigns = {}
			
			for i, step in enumerate(bl.steps):
				
				analyzer.vars = set(), set()
				analyzer.visit(step)
				
				for name in analyzer.vars[0]:
					bl.uses.setdefault(name, set()).add(i)
					refs.setdefault(id, []).append((i, name))
				
				for name in analyzer.vars[1]:
					bl.assigns.setdefault(name, set()).add(i)
					refs.setdefault(id, []).append((i, name))
		
		for id, bl in blocks:
			
			bl.origin = {}
			for sid, name in refs.get(id, []):
				
				origin = bl.origin[name, sid] = set()
				if name in bl.assigns and min(bl.assigns[name]) < sid:
					origin.add(id)
					continue
				elif not bl.preds:
					origin.add(None)
					continue
				
{'fraction_non_alphanumeric': 0.1498351195383347, 'fraction_numerical': 0.0032976092333058533, 'mean_word_length': 2.818253343823761, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4898457', 'n_tokens_mistral': 2245, 'n_tokens_neox': 1802, 'n_words': 437}