<commit_before>import os
import sys
import yaml
import copy
import os.path
import logging

from itertools import chain

from sarah.order import sort_by_successors, dual_cover, restricted_cover

from mirte.core import ModuleDefinition, DepDefinition, VSettingDefinition

FILE_SUFFIX = '.mirte'
DEFAULT_FILE = 'default.mirte'


def depsOf_of_mirteFile_instance_definition(man, insts):
	""" Returns a function that returns the dependencies of
	    an instance definition by its name, where insts is a
	    dictionary of instance definitions from a mirteFile """
	return lambda x: map(lambda a: a[1],
			     filter(lambda b: b[0] in \
				[dn for dn, d in man.modules[insts[x][
					'module']].deps.iteritems()],
				insts[x].items()))

def depsOf_of_mirteFile_module_definition(defs):
	""" Returns a function that returns the dependencies of a module
	    definition by its name, where defs is a dictionary of module
	    definitions from a mirteFile """
	return lambda x: (filter(lambda z: not z is None and z in defs,
				 map(lambda y: y[1].get('type'),
			      	     defs[x]['settings'].items()
				     	if 'settings' in defs[x] else []))) + \
			 (defs[x]['inherits'] if 'inherits' in defs[x] else [])

def module_definition_from_mirteFile_dict(man, d):
	""" Creates a ModuleDefinition instance from the dictionary <d> from
	    a mirte-file for the Manager instance <man>. """
	m = ModuleDefinition()
	if not 'inherits' in d: d['inherits'] = list()
	if not 'settings' in d: d['settings'] = dict()
	if 'implementedBy' in d:
		m.implementedBy = d['implementedBy']
	if 'run' in d and d['run']:
		m.run = True
	m.inherits = set(d['inherits'])
	for p in d['inherits']:
		if not p in man.modules:
			raise ValueError, "No such module %s" % p
		m.deps.update(man.modules[p].deps)
		m.vsettings.update(man.modules[p].vsettings)
		m.inherits.update(man.modules[p].inherits)
		m.run = m.run or man.modules[p].run
	if len(m.inherits) == 0:
		m.inherits = set(['module'])
	for k, v in d['settings'].iteritems():
		if not 'type' in v:
			if not k in m.vsettings:
				raise ValueError, \
					"No such existing vsetting %s" % k
			if 'default' in v:
				m.vsettings[k] = copy.copy(m.vsettings[k])
				m.vsettings[k].default = v['default']
			continue
		if v['type'] in man.modules:
			m.deps[k] = DepDefinition(v['type'])
		elif v['type'] in man.valueTypes:
			m.vsettings[k] = VSettingDefinition(v['type'],
				(man.valueTypes[v['type']](v['default'])
					if 'default' in v else None))
		else:
			raise ValueError, \
				"No such module or valuetype %s" % v<commit_msg>mirteFile: allow inherited modules to override `run' attr<commit_after>import os
import sys
import yaml
import copy
import os.path
import logging

from itertools import chain

from sarah.order import sort_by_successors, dual_cover, restricted_cover

from mirte.core import ModuleDefinition, DepDefinition, VSettingDefinition

FILE_SUFFIX = '.mirte'
DEFAULT_FILE = 'default.mirte'


def depsOf_of_mirteFile_instance_definition(man, insts):
	""" Returns a function that returns the dependencies of
	    an instance definition by its name, where insts is a
	    dictionary of instance definitions from a mirteFile """
	return lambda x: map(lambda a: a[1],
			     filter(lambda b: b[0] in \
				[dn for dn, d in man.modules[insts[x][
					'module']].deps.iteritems()],
				insts[x].items()))

def depsOf_of_mirteFile_module_definition(defs):
	""" Returns a function that returns the dependencies of a module
	    definition by its name, where defs is a dictionary of module
	    definitions from a mirteFile """
	return lambda x: (filter(lambda z: not z is None and z in defs,
				 map(lambda y: y[1].get('type'),
			      	     defs[x]['settings'].items()
				     	if 'settings' in defs[x] else []))) + \
			 (defs[x]['inherits'] if 'inherits' in defs[x] else [])

def module_definition_from_mirteFile_dict(man, d):
	""" Creates a ModuleDefinition instance from the dictionary <d> from
	    a mirte-file for the Manager instance <man>. """
	m = ModuleDefinition()
	if not 'inherits' in d: d['inherits'] = list()
	if not 'settings' in d: d['settings'] = dict()
	if 'implementedBy' in d:
		m.implementedBy = d['implementedBy']
	m.inherits = set(d['inherits'])
	for p in d['inherits']:
		if not p in man.modules:
			raise ValueError, "No such module %s" % p
		m.deps.update(man.modules[p].deps)
		m.vsettings.update(man.modules[p].vsettings)
		m.inherits.update(man.modules[p].inherits)
		m.run = m.run or man.modules[p].run
	if 'run' in d:
		m.run = d['run']
	if len(m.inherits) == 0:
		m.inherits = set(['module'])
	for k, v in d['settings'].iteritems():
		if not 'type' in v:
			if not k in m.vsettings:
				raise ValueError, \
					"No such existing vsetting %s" % k
			if 'default' in v:
				m.vsettings[k] = copy.copy(m.vsettings[k])
				m.vsettings[k].default = v['default']
			continue
		if v['type'] in man.modules:
			m.deps[k] = DepDefinition(v['type'])
		elif v['type'] in man.valueTypes:
			m.vsettings[k] = VSettingDefinition(v['type'],
				(man.valueTypes[v['type']](v['default'])
					if 'default' in v else None))
		else:
			raise ValueError, \
				"No such module or valuetype %s" % v
{'fraction_non_alphanumeric': 0.13388878124394496, 'fraction_numerical': 0.0015500871924045726, 'mean_word_length': 3.8333333333333335, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2175758', 'n_tokens_mistral': 1919, 'n_tokens_neox': 1760, 'n_words': 550}