<commit_before>from collections import defaultdict

def merge(dts):
    """
    For dict with samples lines merge into one file
    """
    all_data = defaultdict(dict)
    all_lines = defaultdict()
    samples = set()
    for fn in dts:
        for m in dts[fn]:
            for s in dts[fn][m]:
                idu = dts[fn][m][0]
                samples.add(dts[fn][m][3])
                all_data[idu][dts[fn][m][3]] = dts[fn][m][1]
                all_lines[idu] = dts[fn][m][4]
    for idu in all_data:
        expression = _convert_to_string(all_data[idu], samples)
        if _start(all_lines[idu]) not in merged_lines[_chrom(all_lines[idu])]:
            merged_lines[_chrom(all_lines[idu])][_start(all_lines[idu])] = []
        merged_lines[_chrom(all_lines[idu])][_start(all_lines[idu])] = _fix(all_lines[idu], expression)

def _fix(line, expression):
    cols = line.split("\t")
    attr = cols[8].split(";")
    cols[8] = ";".join([a if a.find("Expression") < 0 else expression for a in attr])
    return "\t".join(cols)

def _convert_to_string(d, s):
    v = [d[ss] if ss in d else 0 for ss in s]
    return " Expression %s" % ",".join(map(str, v))

def _chrom(string):
    return string.split("\t")[0]

def _start(string):
    return string.split("\t")[3]

<commit_msg>complete merge files<commit_after>from collections import defaultdict

def merge(dts):
    """
    For dict with samples lines merge into one file
    """
    all_data = defaultdict(dict)
    all_lines = defaultdict()
    merged_lines = defaultdict(dict)
    samples = set()
    for fn in dts:
        for m in dts[fn]:
            for s in dts[fn][m]:
                for hit in dts[fn][m][s]:
                    idu = hit[0]
                    samples.add(hit[3])
                    all_data[idu][hit[3]] = hit[2] # get the expression of the sample from line
                    all_lines[idu] = hit[4] # get the line
    for idu in all_data:
        expression = _convert_to_string(all_data[idu], samples)
        if _start(all_lines[idu]) not in merged_lines[_chrom(all_lines[idu])]:
            merged_lines[_chrom(all_lines[idu])][_start(all_lines[idu])] = []
        merged_lines[_chrom(all_lines[idu])][_start(all_lines[idu])].append([idu, "", "", "", _fix(all_lines[idu], expression)])
    return merged_lines

def _fix(line, expression):
# Need to fix Read attribute since not usefull when multiple sample in a line.
    cols = line.split("\t")
    attr = cols[8].split(";")
    cols[8] = ";".join([a if a.find("Expression") < 0 else expression for a in attr])
    return "\t".join(cols)

def _convert_to_string(d, s):
    v = [d[ss] if ss in d else 0 for ss in s]
    return " Expression %s" % ",".join(map(str, v))

def _chrom(string):
    return string.split("\t")[0]

def _start(string):
    return string.split("\t")[3]


{'fraction_non_alphanumeric': 0.15161176053843428, 'fraction_numerical': 0.0077931278781438185, 'mean_word_length': 2.8369565217391304, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '875661', 'n_tokens_mistral': 1001, 'n_tokens_neox': 975, 'n_words': 243}