<commit_before>def aggregate_reads(o, reads):
    "aggregate reads by startpos, endpos and base"

    name_dict     = {} # name -> reads
    unique_pairs  = set() # (start, length, base, is_overlap)
    unique_single = set() # (start, length, base, is_reverse)

    nsum,  nerror    = 0, 0 # depth, errors (such as 3 reads share the same name)
    nlowq, ninconsis = 0, 0 # low quality bases, inconsistent pairs (count on reads)

    for read in reads:
        name, *info = read
        try_append(name_dict, name, info)
        nsum += 1

    for name, reads in name_dict.items():
        if len(reads) == 1: # non-overlap or single
            base, qual, r1start, r1len, r2start, tlen, isrev, paired = reads[0]

            if qual <= o.qual:
                nlowq += 1
                continue

            if paired:
                start = min(r1start, r2start)
                unique_pairs.add((start, tlen, base, False))
            else:
                unique_single.add((r1start, tlen, base, isrev))

        elif len(reads) == 2: # overlap
            r1, r2 = reads
            base1, qual1, r1start1, r1len, r2start1, tlen1, isrev1, paired1 = r1
            base2, qual2, r1start2, r1len, r2start2, tlen2, isrev2, paired2 = r2

            if qual1 <= o.qual or qual2 <= o.qual:
                nlowq += 2
                continue

            if base1 != base2:
                ninconsis += 2
                continue

            start = min(r1start1, r2start1)
            qual  = max(qual1, qual2)

            unique_pairs.add((start, tlen1, base1, True))

        else: # error
            if o.verbos:
                print("%s: more than 2 reads (%d total) share the same name; all droped." % (name, len(reads)))
            nerror += len(reads)
<commit_msg>Revert "slightly improve performance by no longer record quality infomations"<commit_after>def aggregate_reads(o, reads):
    "aggregate reads by startpos, endpos and base"

    name_dict     = {} # name -> reads
    unique_pairs  = {} # start, length, base, is_overlap -> qualities
    unique_single = {} # start, length, base, is_reverse -> qualities

    nsum,  nerror    = 0, 0 # depth, errors (such as 3 reads share the same name)
    nlowq, ninconsis = 0, 0 # low quality bases, inconsistent pairs (count on reads)

    for read in reads:
        name, *info = read
        try_append(name_dict, name, info)
        nsum += 1

    for name, reads in name_dict.items():
        if len(reads) == 1: # non-overlap or single
            base, qual, r1start, r1len, r2start, tlen, isrev, paired = reads[0]

            if qual <= o.qual:
                nlowq += 1
                continue

            if paired:
                start = min(r1start, r2start)
                try_append(unique_pairs, (start, tlen, base, False), qual)
            else:
                try_append(unique_single, (r1start, tlen, base, isrev), qual)

        elif len(reads) == 2: # overlap
            r1, r2 = reads
            base1, qual1, r1start1, r1len, r2start1, tlen1, isrev1, paired1 = r1
            base2, qual2, r1start2, r1len, r2start2, tlen2, isrev2, paired2 = r2

            if qual1 <= o.qual or qual2 <= o.qual:
                nlowq += 2
                continue

            if base1 != base2:
                ninconsis += 2
                continue

            start = min(r1start1, r2start1)
            qual  = max(qual1, qual2)

            try_append(unique_pairs, (start, tlen1, base1, True), qual)

        else: # error
            if o.verbos:
                print("%s: more than 2 reads (%d total) share the same name; all droped." % (name, len(reads)))
            nerror += len(reads)

{'fraction_non_alphanumeric': 0.10287730727470142, 'fraction_numerical': 0.02985884907709012, 'mean_word_length': 1.7937831690674753, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5586399', 'n_tokens_mistral': 1231, 'n_tokens_neox': 1143, 'n_words': 363}