<commit_before>import argparse
import os
import sys
import textwrap
from collections import defaultdict
from xlrd import XLRDError
import pandas as pd
from jinja2 import Environment, FileSystemLoader

import consts

encoding = sys.getfilesystemencoding()
EXEC_DIR = os.path.dirname(unicode(__file__, encoding))


def save_or_print_json(json_str, outdir, json_name):
    if outdir:
        with open("%s/%s.json" % (outdir, json_name), 'w') as cout:
            cout.writelines(json_str)
    else:
        print "#%s.json" % json_name
        print json_str


def is_false(v):
    return str(v).lower() in ("", "no", "false", "f", "0")


class MetadataParser(object):
    def __init__(self, args):
        self.experiment_type = args.data_type
        self.nthreads = args.nthreads
        self.mem = args.mem
        self.file_path = args.meta_file
        self.records = self.load_file()
        self.separate_jsons = args.separate_jsons
        self.default_adapters = args.default_adapters
        self.genome_effective_size = args.genome_effective_size
        self.genome_ref_first_index = args.genome_ref_first_index
        self.genome_sizes_file = args.genome_sizes_file
        self.genome_fasta_files = args.genome_fasta_files
        self.annotation_file = args.annotation_file
        self.rsem_dir = args.rsem_dir
        self.encode_blacklist_bedfile = args.encode_blacklist_bedfile
        self.trimmomatic_jar = args.trimmomatic_jar
        self.picard_jar = args.picard_jar
        self.as_narrowPeak = args.as_narrowPeak
        self.as_broadPeak = args.as_broadPeak
        self.star_genome_dir = args.star_genome_dir
        self.bamtools_forward_filter = args.bamtools_forward_filter
        self.bamtools_reverse_filter = args.bamtools_reverse_filter
        self.preserve_arguments = args.preserve_arguments
        self.read_length = args.read_length

        # To allow certain configuration files to be overriden, this is needed
        self.purge_undef_args()

    def purge_undef_args(self):
        for empty_key in [k for k, v in self.__dict__.iteritems() if v is None]:
            del self.__dict__[empty_key]

    def render_json(self, wf_conf, samples_list, data_dir, template_name):
        pass

    def parse_metadata(self, data_dir):
        pass

    def load_file(self):
        try:
            rows = pd.read_excel(self.file_path,
                               true_values=['Yes', 'Y', 'yes', 'y', 1],
                               false_values=['No', 'N', 'no', 'n', 0])
        except XLRDError:
            rows = pd.read_csv(self.file_path,
                               true_values=['Yes', 'Y', 'yes', 'y', '1'],
                               false_values=['No', 'N', 'no', 'n', '0'], sep='\t',
                               encoding = 'utf-8')
        named_cols = [c for c in rows.columns if not c.startswith('unnamed: ')]
        rows = rows.loc[:, named_cols]
        rows.columns = [c.lower() for c in rows.columns]
        return rows

    def update_paths(self, ref_data_obj):
        options = ref_data_obj.__dict__.iteritems()
        if self.preserve_arguments:
            options = {(k, v) for k, v in ref_data_obj.__dict__.iteritems() if k not in self.__dict__}
        self.__dict__.update(options)


def generateMetadataParser(args):
    return MetadataParser(args)


class MetadataParserChipseq(object):
    def __init__(self, **kwargs):
        self.obj = generateMetadataParser(kwargs['args_obj'])

    def __getattr__(self, attr):
        return getattr(self.obj, attr)

    def render_json(self, wf_conf, samples_list, data_dir, template_name):
        env = Environment(extensions=["jinja2.ext.do"], loader=FileSystemLoader(os.path.join(EXEC_DIR, "templates")))
        template = env.get_template(template_name + '.j2')
        json_str = template.render({'wf_conf': wf_conf,
                                    'samples_list': samples_list,
                                    'data_dir': data_dir,
                                    'conf_args': self
                                    })
        json_str = '\n'.join([l for l in json_str.split('\n') if l.strip() != ''])  # Remove empty lines
        return json_str

    def parse_metadata(self, data_dir):
        samples_dict = defaultdict(list)
        wf_conf_dict = {}
        for r in self.records:
            read_type = r['paired-end or single-end'].lower()
            sample_info = {'treatment': r['name']}
            wf_key = '-'.join([read_type])
            if 'control' in r.keys() and r['control'] and r['control'].upper() != 'NA':
                sample_info['control'] = r['control']
                wf_key += '-with-control'
            wf_conf_dict[wf_key] = {'rt': read_type,
                                    'st': sample_info.keys()}
            genome = consts.GENOME  # Default genome
            if 'genome' in r.keys():
                genome = r['genome']

            samples_dict[wf_key].append([sample_info, genome])
        for wf_key, samples_genomes in samples_dict.iteritems():
            if self.obj.separate_jsons:
                for si, s in enumerate(sorted(samples_genomes)):
                    sample, genome = s[0], s[1]
                    ref_dataset = consts.ReferenceDataset(genome)
                    self.update_paths(ref_dataset)
                    yield self.render_json(wf_conf_dict[wf_key], [sample], data_dir, self.experiment_type), wf_key, si

            else:
                samples_list, genomes_list = zip(*samples_genomes)
                if len(set(genomes_list)) > 1:
                    raise Exception('More than one genome specified (%s). Please create a different metadata file'
                                    ' per genome or provide a sjdb and specify the --separate-jsons argument' %
                                    ', '.join(set(genomes_list)))
                ref_dataset = consts.ReferenceDataset(genomes_list[0])
                self.update_paths(ref_dataset)
                yield self.render_json(wf_conf_dict[wf_key], sorted(samples_list), data_dir, self.experiment_type), wf_key, None


class MetadataParserAtacseq(object):
    def __init__(self, **kwargs):
        self.obj = generateMetadataParser(kwargs['args_obj'])

    def __getattr__(self, attr):
        return getattr(self.obj, attr)

    def render_json(self, wf_conf, samples_list, data_dir, template_name):
        env = Environment(extensions=["jinja2.ext.do"], loader=FileSystemLoader(os.path.join(EXEC_DIR, "templates")))
        template = env.get_template(template_name + '.j2')
        json_str = template.render({'wf_conf': wf_conf,
                                    'samples_list': samples_list,
                                    'data_dir': data_dir,
                                    'nthreads': self.nthreads,
                                    'conf_args': self
                                    })
        json_str = '\n'.join([l for l in json_str.split('\n') if l.strip() != ''])  # Remove empty lines
        return json_str

    def parse_metadata(self, data_dir):
        samples_dict = defaultdict(list)
        wf_conf_dict = {}
        for r in self.records:<commit_msg>Fix bug un JSON generator<commit_after>import argparse
import os
import sys
import textwrap
from collections import defaultdict
from xlrd import XLRDError
import pandas as pd
from jinja2 import Environment, FileSystemLoader

import consts

encoding = sys.getfilesystemencoding()
EXEC_DIR = os.path.dirname(unicode(__file__, encoding))


def save_or_print_json(json_str, outdir, json_name):
    if outdir:
        with open("%s/%s.json" % (outdir, json_name), 'w') as cout:
            cout.writelines(json_str)
    else:
        print "#%s.json" % json_name
        print json_str


def is_false(v):
    return str(v).lower() in ("", "no", "false", "f", "0")


class MetadataParser(object):
    def __init__(self, args):
        self.experiment_type = args.data_type
        self.nthreads = args.nthreads
        self.mem = args.mem
        self.file_path = args.meta_file
        self.records = self.load_file()
        self.separate_jsons = args.separate_jsons
        self.default_adapters = args.default_adapters
        self.genome_effective_size = args.genome_effective_size
        self.genome_ref_first_index = args.genome_ref_first_index
        self.genome_sizes_file = args.genome_sizes_file
        self.genome_fasta_files = args.genome_fasta_files
        self.annotation_file = args.annotation_file
        self.rsem_dir = args.rsem_dir
        self.encode_blacklist_bedfile = args.encode_blacklist_bedfile
        self.trimmomatic_jar = args.trimmomatic_jar
        self.picard_jar = args.picard_jar
        self.as_narrowPeak = args.as_narrowPeak
        self.as_broadPeak = args.as_broadPeak
        self.star_genome_dir = args.star_genome_dir
        self.bamtools_forward_filter = args.bamtools_forward_filter
        self.bamtools_reverse_filter = args.bamtools_reverse_filter
        self.preserve_arguments = args.preserve_arguments
        self.read_length = args.read_length

        # To allow certain configuration files to be overriden, this is needed
        self.purge_undef_args()

    def purge_undef_args(self):
        for empty_key in [k for k, v in self.__dict__.iteritems() if v is None]:
            del self.__dict__[empty_key]

    def render_json(self, wf_conf, samples_list, data_dir, template_name):
        pass

    def parse_metadata(self, data_dir):
        pass

    def load_file(self):
        try:
            rows = pd.read_excel(self.file_path,
                               true_values=['Yes', 'Y', 'yes', 'y', 1],
                               false_values=['No', 'N', 'no', 'n', 0])
        except XLRDError:
            rows = pd.read_csv(self.file_path,
                               true_values=['Yes', 'Y', 'yes', 'y', '1'],
                               false_values=['No', 'N', 'no', 'n', '0'], sep='\t',
                               encoding = 'utf-8')
        named_cols = [c for c in rows.columns if not c.startswith('unnamed: ')]
        rows = rows.loc[:, named_cols]
        rows.columns = [c.lower() for c in rows.columns]
        return rows

    def update_paths(self, ref_data_obj):
        options = ref_data_obj.__dict__.iteritems()
        if self.preserve_arguments:
            options = {(k, v) for k, v in ref_data_obj.__dict__.iteritems() if k not in self.__dict__}
        self.__dict__.update(options)


def generateMetadataParser(args):
    return MetadataParser(args)


class MetadataParserChipseq(object):
    def __init__(self, **kwargs):
        self.obj = generateMetadataParser(kwargs['args_obj'])

    def __getattr__(self, attr):
        return getattr(self.obj, attr)

    def render_json(self, wf_conf, samples_list, data_dir, template_name):
        env = Environment(extensions=["jinja2.ext.do"], loader=FileSystemLoader(os.path.join(EXEC_DIR, "templates")))
        template = env.get_template(template_name + '.j2')
        json_str = template.render({'wf_conf': wf_conf,
                                    'samples_list': samples_list,
                                    'data_dir': data_dir,
                                    'conf_args': self
                                    })
        json_str = '\n'.join([l for l in json_str.split('\n') if l.strip() != ''])  # Remove empty lines
        return json_str

    def parse_metadata(self, data_dir):
        samples_dict = defaultdict(list)
        wf_conf_dict = {}
        for r_ix, r in self.records.iterrows():
            read_type = r['paired-end or single-end'].lower()
            sample_info = {'treatment': r['name']}
            wf_key = '-'.join([read_type])
            if 'control' in r.keys() and r['control'] and type(r['control']) != float:  # After reading this metadata info, this will contain a nan (float) if undetermined
                sample_info['control'] = r['control']
                wf_key += '-with-control'
            wf_conf_dict[wf_key] = {'rt': read_type,
                                    'st': sample_info.keys()}
            genome = consts.GENOME  # Default genome
            if 'genome' in r.keys():
                genome = r['genome']

            samples_dict[wf_key].append([sample_info, genome])
        for wf_key, samples_genomes in samples_dict.iteritems():
            if self.obj.separate_jsons:
                for si, s in enumerate(sorted(samples_genomes)):
                    sample, genome = s[0], s[1]
                    ref_dataset = consts.ReferenceDataset(genome)
                    self.update_paths(ref_dataset)
                    yield self.render_json(wf_conf_dict[wf_key], [sample], data_dir, self.experiment_type), wf_key, si

            else:
                samples_list, genomes_list = zip(*samples_genomes)
                if len(set(genomes_list)) > 1:
                    raise Exception('More than one genome specified (%s). Please create a different metadata file'
                                    ' per genome or provide a sjdb and specify the --separate-jsons argument' %
                                    ', '.join(set(genomes_list)))
                ref_dataset = consts.ReferenceDataset(genomes_list[0])
                self.update_paths(ref_dataset)
                yield self.render_json(wf_conf_dict[wf_key], sorted(samples_list), data_dir, self.experiment_type), wf_key, None


class MetadataParserAtacseq(object):
    def __init__(self, **kwargs):
        self.obj = generateMetadataParser(kwargs['args_obj'])

    def __getattr__(self, attr):
        return getattr(self.obj, attr)

    def render_json(self, wf_conf, samples_list, data_dir, template_name):
        env = Environment(extensions=["jinja2.ext.do"], loader=FileSystemLoader(os.path.join(EXEC_DIR, "templates")))
        template = env.get_template(template_name + '.j2')
        json_str = template.render({'wf_conf': wf_conf,
                                    'samples_list': samples_list,
                                    'data_dir': data_dir,
                                    'nthreads': self.nthreads,
                                    'conf_args': self
                                    })
        json_str = '\n'.join([l for l in json_str.split('\n') if l.strip() != ''])  # Remove empty lines
        return json_str

    def parse_metadata(self, data_dir):
        samples_dict = defaultdict(list)
        wf_conf_dict = {}
        for r_ix, r in self.records.iterrows():
{'fraction_non_alphanumeric': 0.1064284728928128, 'fraction_numerical': 0.002069250931162919, 'mean_word_length': 2.170566367811065, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4197619', 'n_tokens_mistral': 4427, 'n_tokens_neox': 4222, 'n_words': 926}