<commit_before># -*- coding: utf-8 -*-

import re, sys, os, traceback, glob, signal
from multiprocessing import Event
from collections import defaultdict, OrderedDict
from tempfile import TemporaryFile

from snakemake.logging import logger
from snakemake.rules import Rule
from snakemake.exceptions import MissingOutputException, MissingInputException, \
	AmbiguousRuleException, CyclicGraphException, MissingRuleException, \
	RuleException, CreateRuleException, ProtectedOutputException, \
	UnknownRuleException, NoRulesException
from snakemake.shell import shell, format
from snakemake.jobs import Job, KnapsackJobScheduler, ClusterJobScheduler, print_job_dag
from snakemake.parser import compile_to_python
from snakemake.io import protected, temp, temporary, IOFile


__author__ = "<NAME>"

class Jobcounter:
	def __init__(self):
		self._count = 0
		self._done = 0
	
	def add(self):
		self._count += 1
	
	def done(self):
		self._done += 1
	
	def __str__(self):
		return "{} of {} steps ({}%) done".format(self._done, self._count, int(self._done / self._count * 100))

class Workflow:
	def __init__(self):
		"""
		Create the controller.
		"""
		self._rules = OrderedDict()
		self._first = None
		self._workdir = None
		self._runtimes = defaultdict(list)
		self.cores = 1
		self.rowmaps = dict()
		self.jobcounter = None
		self.rule_count = 0
		self.errors = False
	
	def report_runtime(self, rule, runtime):
		self._runtimes[rule].append(runtime)
		
	def get_runtimes(self):
		for rule, runtimes in self._runtimes.items():
			s = sum(runtimes)
			yield rule, min(runtimes), max(runtimes), s, s / len(runtimes)

	def get_rule_count(self):
		return len(self._rules)
	
	def add_rule(self, name = None, lineno = None, snakefile = None):
		"""
		Add a rule.
		"""
		if name == None:
			name = str(len(self._rules))
		if self.is_rule(name):
			raise CreateRuleException(
				"The name {} is already used by another rule".format(name))
		rule = Rule(name, self, lineno = lineno, snakefile = snakefile)
		self._rules[rule.name] = rule
		if not self._first:
			self._first = rule.name
		return name
			
	def is_rule(self, name):
		"""
		Return True if name is the name of a rule.
		
		Arguments
		name -- a name
		"""
		return name in self._rules
	
	def get_producers(self, files, exclude = None):
		for rule in self.get_rules():
			if rule != exclude:
				for f in files:
					if rule.is_producer(f):
						yield rule, f

	def get_rule(self, name):
		"""
		Get rule by name.
		
		Arguments
		name -- the name of the rule
		"""
		if not self._rules:
			raise NoRulesException()
		if not name in self._rules:
			raise UnknownRuleException(name)
		return self._rules[name]

	def run_first_rule(self, dryrun = False, touch = False, 
		forcethis = False, forceall = False, give_reason = False, 
		cluster = None, dag = False):
		"""
		Apply the rule defined first.
		"""
		first = self._first
		if not first:
			for key, value in self._rules.items():
				first = key
				break
		return self._run([(self.get_rule(first), None)], 
			dryrun = dryrun, touch = touch, forcethis = forcethis, 
			forceall = forceall, give_reason = give_reason, 
			cluster = cluster, dag = dag)
			
	def get_file_producers(self, files, dryrun = False, 
		forcethis = False, forceall = False):
		"""
		Return a dict of rules with requested files such that the requested files are produced.
		
		Arguments
		files -- the paths of the files to produce
		"""
		producers = dict()
		missing_input_ex = defaultdict(list)
		for rule, file in self.get_producers(files):
			try:
				rule.run(file, jobs=dict(), forceall = forceall, 
					dryrun = True, visited = set())
				if file in producers:
					raise AmbiguousRuleException(producers[file], rule)
				producers[file] = rule
			except MissingInputException as ex:
				missing_input_ex[file].append(ex)
		
		toraise = []
		for file in files:
			if not file in producers:
				if file in missing_input_ex:
					toraise += missing_input_ex[file]
				else:
					toraise.append(MissingRuleException(file))
		if toraise:
			raise RuleException(include = toraise)

		return [(rule, file) for file, rule in producers.items()]
	
	def run_rules(self, targets, dryrun = False, touch = False, 
		forcethis = False, forceall = False, give_reason = False, 
		cluster = None, dag = False):
		ruletargets, filetargets = [], []
		for target in targets:
			if workflow.is_rule(target):
				ruletargets.append(target)
			else:
				filetargets.append(target)
		
		torun = self.get_file_producers(filetargets, forcethis = forcethis, 
			forceall = forceall, dryrun = dryrun) + \
			[(self.get_rule(name), None) for name in ruletargets]
				
		return self._run(torun, dryrun = dryrun, touch = touch, 
			forcethis = forcethis, forceall = forceall, 
			give_reason = give_reason, cluster = cluster, dag = dag)
	
	def _run(self, torun, dryrun = False, touch = False, forcethis = False, <commit_msg>Normalize path names given as targets.<commit_after># -*- coding: utf-8 -*-

import re, sys, os, traceback, glob, signal
from multiprocessing import Event
from collections import defaultdict, OrderedDict
from tempfile import TemporaryFile

from snakemake.logging import logger
from snakemake.rules import Rule
from snakemake.exceptions import MissingOutputException, MissingInputException, \
	AmbiguousRuleException, CyclicGraphException, MissingRuleException, \
	RuleException, CreateRuleException, ProtectedOutputException, \
	UnknownRuleException, NoRulesException
from snakemake.shell import shell, format
from snakemake.jobs import Job, KnapsackJobScheduler, ClusterJobScheduler, print_job_dag
from snakemake.parser import compile_to_python
from snakemake.io import protected, temp, temporary, IOFile


__author__ = "<NAME>"

class Jobcounter:
	def __init__(self):
		self._count = 0
		self._done = 0
	
	def add(self):
		self._count += 1
	
	def done(self):
		self._done += 1
	
	def __str__(self):
		return "{} of {} steps ({}%) done".format(self._done, self._count, int(self._done / self._count * 100))

class Workflow:
	def __init__(self):
		"""
		Create the controller.
		"""
		self._rules = OrderedDict()
		self._first = None
		self._workdir = None
		self._runtimes = defaultdict(list)
		self.cores = 1
		self.rowmaps = dict()
		self.jobcounter = None
		self.rule_count = 0
		self.errors = False
	
	def report_runtime(self, rule, runtime):
		self._runtimes[rule].append(runtime)
		
	def get_runtimes(self):
		for rule, runtimes in self._runtimes.items():
			s = sum(runtimes)
			yield rule, min(runtimes), max(runtimes), s, s / len(runtimes)

	def get_rule_count(self):
		return len(self._rules)
	
	def add_rule(self, name = None, lineno = None, snakefile = None):
		"""
		Add a rule.
		"""
		if name == None:
			name = str(len(self._rules))
		if self.is_rule(name):
			raise CreateRuleException(
				"The name {} is already used by another rule".format(name))
		rule = Rule(name, self, lineno = lineno, snakefile = snakefile)
		self._rules[rule.name] = rule
		if not self._first:
			self._first = rule.name
		return name
			
	def is_rule(self, name):
		"""
		Return True if name is the name of a rule.
		
		Arguments
		name -- a name
		"""
		return name in self._rules
	
	def get_producers(self, files, exclude = None):
		for rule in self.get_rules():
			if rule != exclude:
				for f in files:
					if rule.is_producer(f):
						yield rule, f

	def get_rule(self, name):
		"""
		Get rule by name.
		
		Arguments
		name -- the name of the rule
		"""
		if not self._rules:
			raise NoRulesException()
		if not name in self._rules:
			raise UnknownRuleException(name)
		return self._rules[name]

	def run_first_rule(self, dryrun = False, touch = False, 
		forcethis = False, forceall = False, give_reason = False, 
		cluster = None, dag = False):
		"""
		Apply the rule defined first.
		"""
		first = self._first
		if not first:
			for key, value in self._rules.items():
				first = key
				break
		return self._run([(self.get_rule(first), None)], 
			dryrun = dryrun, touch = touch, forcethis = forcethis, 
			forceall = forceall, give_reason = give_reason, 
			cluster = cluster, dag = dag)
			
	def get_file_producers(self, files, dryrun = False, 
		forcethis = False, forceall = False):
		"""
		Return a dict of rules with requested files such that the requested files are produced.
		
		Arguments
		files -- the paths of the files to produce
		"""
		producers = dict()
		missing_input_ex = defaultdict(list)
		for rule, file in self.get_producers(files):
			try:
				rule.run(file, jobs=dict(), forceall = forceall, 
					dryrun = True, visited = set())
				if file in producers:
					raise AmbiguousRuleException(producers[file], rule)
				producers[file] = rule
			except MissingInputException as ex:
				missing_input_ex[file].append(ex)
		
		toraise = []
		for file in files:
			if not file in producers:
				if file in missing_input_ex:
					toraise += missing_input_ex[file]
				else:
					toraise.append(MissingRuleException(file))
		if toraise:
			raise RuleException(include = toraise)

		return [(rule, file) for file, rule in producers.items()]
	
	def run_rules(self, targets, dryrun = False, touch = False, 
		forcethis = False, forceall = False, give_reason = False, 
		cluster = None, dag = False):
		ruletargets, filetargets = [], []
		for target in targets:
			if workflow.is_rule(target):
				ruletargets.append(target)
			else:
				filetargets.append(os.path.relpath(target))
		
		torun = self.get_file_producers(filetargets, forcethis = forcethis, 
			forceall = forceall, dryrun = dryrun) + \
			[(self.get_rule(name), None) for name in ruletargets]
				
		return self._run(torun, dryrun = dryrun, touch = touch, 
			forcethis = forcethis, forceall = forceall, 
			give_reason = give_reason, cluster = cluster, dag = dag)
	
	def _run(self, torun, dryrun = False, touch = False, forcethis = False, 
{'fraction_non_alphanumeric': 0.10566537599347871, 'fraction_numerical': 0.002037905033625433, 'mean_word_length': 3.6984202967927238, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4380028', 'n_tokens_mistral': 3719, 'n_tokens_neox': 3184, 'n_words': 951}