<commit_before>from .utils import getOrDefault
import datasuper as ds
from .snakemake_rule_builder import SnakemakeRuleBuilder
from .snakemake_utils import *
from .utils import *


class ResultSchema:

    def __init__(self, muRepo, pipeName, pipeVersion, schema, origin=False):
        self.muRepo = muRepo
        self.muConfig = self.muRepo.muConfig
        self.pipelineName = pipeName
        self.pipelineVersion = pipeVersion
        self.origin = origin

        # this is the name of the result type in datasuper as well
        self.name = schema['NAME']

        self.dependencies = getOrDefault(schema, 'DEPENDENCIES', [])
        self.module = getOrDefault(schema, 'MODULE', self.name)
        self.level = getOrDefault(schema, 'LEVEL', 'SAMPLE')

        self.snakeFilename = '{}.smk'.format(self.module)
        if not origin:
            self.snakeFilepath = self.muConfig.getSnakefile(self.pipelineName,
                                                            self.pipelineVersion,
                                                            self.snakeFilename)
        self.files = {}
        files = schema['FILES']
        if type(files) == []:
            self.files = {str(i): f for i, f in enumerate(files)}
        else:
            self.files = files

    def _makeSampleLevelRegisterRule(self):
        ruleBldr = SnakemakeRuleBuilder('register_{}'.format(self.module))

        dsRepo = ds.Repo.loadRepo()
        for fname, ftype in self.files.items():
            ext = dsRepo.getFileTypeExt(ftype)
            fpattern = self._makeFilePattern(fname, ext)
            ruleBldr.addInput(fname, fpattern)

        ruleBldr.setOutput(self.getOutputFilePattern())

        resName = joinResultNameType('{sample_name}', self.name)
        ruleBldr.addParam('dsResultName', resName)
        ruleBldr.addParam('sampleName', '{sample_name}')
        ruleBldr.addParam('dsResultType', self.name)
        for fname, ftype in self.files.items():
            ruleBldr.addParam(fname, ftype)

        runStr = '''
            with ds.Repo.loadRepo() as dsrepo:
                fileRecs = {}
                for fname, fpath in input.items():
                    name = os.path.basename(fpath)
                    abspath = os.path.abspath(fpath)

                    print('[DataSuper] FPATH: ' + fpath, file=sys.stderr)
                    print('[DataSuper] ABSPATH: ' + abspath, file=sys.stderr)
                    fileType = params[fname]
                    fileRec = ds.getOrMakeFile(dsrepo, name, fpath, fileType)
                    print('[DataSuper] FILEREC: ' + str(fileRec), file=sys.stderr)

                    fileRecs[fname] = name

                result = ds.ResultRecord(dsrepo,
                                         name=params.dsResultName,
                                         result_type=params.dsResultType,
                                         file_records=fileRecs)
                print('[DataSuper] Saving Result: {}'.format(result), file=sys.stderr)
                try:
                    result.save()
                except ds.database.RecordExistsError as ree:
                    print('[DataSuper] Result Record already exists: {}'.format(ree), file=sys.stderr)

                try:
                    sampleName = params.sampleName
                except KeyError:
                    sampleName = None<commit_msg>use abspath, not fpath, to build files<commit_after>from .utils import getOrDefault
import datasuper as ds
from .snakemake_rule_builder import SnakemakeRuleBuilder
from .snakemake_utils import *
from .utils import *


class ResultSchema:

    def __init__(self, muRepo, pipeName, pipeVersion, schema, origin=False):
        self.muRepo = muRepo
        self.muConfig = self.muRepo.muConfig
        self.pipelineName = pipeName
        self.pipelineVersion = pipeVersion
        self.origin = origin

        # this is the name of the result type in datasuper as well
        self.name = schema['NAME']

        self.dependencies = getOrDefault(schema, 'DEPENDENCIES', [])
        self.module = getOrDefault(schema, 'MODULE', self.name)
        self.level = getOrDefault(schema, 'LEVEL', 'SAMPLE')

        self.snakeFilename = '{}.smk'.format(self.module)
        if not origin:
            self.snakeFilepath = self.muConfig.getSnakefile(self.pipelineName,
                                                            self.pipelineVersion,
                                                            self.snakeFilename)
        self.files = {}
        files = schema['FILES']
        if type(files) == []:
            self.files = {str(i): f for i, f in enumerate(files)}
        else:
            self.files = files

    def _makeSampleLevelRegisterRule(self):
        ruleBldr = SnakemakeRuleBuilder('register_{}'.format(self.module))

        dsRepo = ds.Repo.loadRepo()
        for fname, ftype in self.files.items():
            ext = dsRepo.getFileTypeExt(ftype)
            fpattern = self._makeFilePattern(fname, ext)
            ruleBldr.addInput(fname, fpattern)

        ruleBldr.setOutput(self.getOutputFilePattern())

        resName = joinResultNameType('{sample_name}', self.name)
        ruleBldr.addParam('dsResultName', resName)
        ruleBldr.addParam('sampleName', '{sample_name}')
        ruleBldr.addParam('dsResultType', self.name)
        for fname, ftype in self.files.items():
            ruleBldr.addParam(fname, ftype)

        runStr = '''
            with ds.Repo.loadRepo() as dsrepo:
                fileRecs = {}
                for fname, fpath in input.items():
                    name = os.path.basename(fpath)
                    abspath = os.path.abspath(fpath)

                    print('[DataSuper] FPATH: ' + fpath, file=sys.stderr)
                    print('[DataSuper] ABSPATH: ' + abspath, file=sys.stderr)
                    fileType = params[fname]
                    fileRec = ds.getOrMakeFile(dsrepo, name, abspath, fileType)
                    print('[DataSuper] FILEREC: ' + str(fileRec), file=sys.stderr)

                    fileRecs[fname] = name

                result = ds.ResultRecord(dsrepo,
                                         name=params.dsResultName,
                                         result_type=params.dsResultType,
                                         file_records=fileRecs)
                print('[DataSuper] Saving Result: {}'.format(result), file=sys.stderr)
                try:
                    result.save()
                except ds.database.RecordExistsError as ree:
                    print('[DataSuper] Result Record already exists: {}'.format(ree), file=sys.stderr)

                try:
                    sampleName = params.sampleName
                except KeyError:
                    sampleName = None
{'fraction_non_alphanumeric': 0.09857289980873915, 'fraction_numerical': 0.0, 'mean_word_length': 1.7334137515078407, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1436077', 'n_tokens_mistral': 1879, 'n_tokens_neox': 1826, 'n_words': 381}