<commit_before>#ifndef _OCOCO_IMPL_H_
#define _OCOCO_IMPL_H_

#pragma once

/****************************
 *** Consensus parameters ***
 ****************************/

ococo::consensus_params_t::consensus_params_t():
mode(BATCH),
strategy(MAJORITY),
min_mapq(1),
min_baseq(0),
init_ref_weight(2),
vcf_fo(nullptr),
fasta_cons_fo(nullptr)
{}


/***********************
 *** Main statistics ***
 ***********************/

template<typename T, int counter_size, int refbase_size>
ococo::stats_t<T,counter_size,refbase_size>::stats_t(ococo::consensus_params_t parameters,bam_hdr_t &h):
n_seqs(h.n_targets),
seq_active(new bool[n_seqs]()),
seq_len(new int64_t[n_seqs]()),
seq_name(new std::string[n_seqs]()),
seq_comment(new std::string[n_seqs]()),
seq_stats(new T*[n_seqs]()),
params(parameters)
{
    for (int seqid=0;seqid<n_seqs;seqid++){
        seq_len[seqid]=h.target_len[seqid];
        seq_active[seqid]=true;
        seq_name[seqid]=std::string(h.target_name[seqid]);
        seq_stats[seqid]=new T[seq_len[seqid]]();
    }
    
    /*while (true) {
        new int[100000000ul];
    }*/
}


template<typename T, int counter_size, int refbase_size>
ococo::stats_t<T,counter_size,refbase_size>::~stats_t(){
    for (int32_t seqid=0;seqid<n_seqs;seqid++){
        delete[] seq_stats[seqid];
    }
    delete[] seq_active;
    delete[] seq_len;
    delete[] seq_name;
    delete[] seq_comment;
    delete[] seq_stats;
}


template<typename T, int counter_size, int refbase_size>
int ococo::stats_t<T,counter_size,refbase_size>::
load_fasta(const std::string &fasta_fn) {
    gzFile fp;
    kseq_t *seq;
    int l;
    fp = gzopen(fasta_fn.c_str(), "r");
    seq = kseq_init(fp);
    
    for(int seqid=0;(l = kseq_read(seq)) >= 0;seqid++) {
        
        assert(seq_name[seqid].compare(seq->name.s)==0);
        assert(static_cast<int64_t>(seq->seq.l) == seq_len[seqid]);
        
        if (seq->comment.l && seq_comment[seqid].empty()){
            seq_comment[seqid]=std::string(seq->comment.s);
        }
        
        for(int64_t pos=0; pos < static_cast<int64_t>(seq->seq.l); pos++){
            assert(seq_stats[seqid][pos]==0);
            
            pos_stats_uncompr_t psu = {0,{0,0,0,0},0};
            psu.nt16=nt256_nt16[static_cast<int32_t>(seq->seq.s[pos])];
            
            if (psu.nt16!=nt256_nt16[static_cast<int32_t>('N')]){
                for(int32_t i=0;i<4;i++){
                    psu.counters[i]= ( (0x1<<i) & psu.nt16) ? params.init_ref_weight : 0;
                }
            }
            
            psu.sum=psu.counters[0]+psu.counters[1]+psu.counters[2]+psu.counters[3];
            seq_stats[seqid][pos]=compress_position_stats(psu);
        }
    }
    kseq_destroy(seq); // STEP 5: destroy seq
    gzclose(fp); // STEP 6: close the file handler
    return 0;
}<commit_msg>Fix bug with too large weight (exceeding counter size).<commit_after>#ifndef _OCOCO_IMPL_H_
#define _OCOCO_IMPL_H_

#pragma once

/****************************
 *** Consensus parameters ***
 ****************************/

ococo::consensus_params_t::consensus_params_t():
mode(BATCH),
strategy(MAJORITY),
min_mapq(1),
min_baseq(0),
init_ref_weight(2),
vcf_fo(nullptr),
fasta_cons_fo(nullptr)
{}


/***********************
 *** Main statistics ***
 ***********************/

template<typename T, int counter_size, int refbase_size>
ococo::stats_t<T,counter_size,refbase_size>::stats_t(ococo::consensus_params_t parameters,bam_hdr_t &h):
n_seqs(h.n_targets),
seq_active(new bool[n_seqs]()),
seq_len(new int64_t[n_seqs]()),
seq_name(new std::string[n_seqs]()),
seq_comment(new std::string[n_seqs]()),
seq_stats(new T*[n_seqs]()),
params(parameters)
{
    for (int seqid=0;seqid<n_seqs;seqid++){
        seq_len[seqid]=h.target_len[seqid];
        seq_active[seqid]=true;
        seq_name[seqid]=std::string(h.target_name[seqid]);
        seq_stats[seqid]=new T[seq_len[seqid]]();
    }
    
    /*while (true) {
        new int[100000000ul];
    }*/
}


template<typename T, int counter_size, int refbase_size>
ococo::stats_t<T,counter_size,refbase_size>::~stats_t(){
    for (int32_t seqid=0;seqid<n_seqs;seqid++){
        delete[] seq_stats[seqid];
    }
    delete[] seq_active;
    delete[] seq_len;
    delete[] seq_name;
    delete[] seq_comment;
    delete[] seq_stats;
}


template<typename T, int counter_size, int refbase_size>
int ococo::stats_t<T,counter_size,refbase_size>::
load_fasta(const std::string &fasta_fn) {
    gzFile fp;
    kseq_t *seq;
    int l;
    fp = gzopen(fasta_fn.c_str(), "r");
    seq = kseq_init(fp);
    
    constexpr int32_t max_counter_value = ococo::right_full_mask<T, counter_size>();
    
    for(int seqid=0;(l = kseq_read(seq)) >= 0;seqid++) {
        
        assert(seq_name[seqid].compare(seq->name.s)==0);
        assert(static_cast<int64_t>(seq->seq.l) == seq_len[seqid]);
        
        if (seq->comment.l && seq_comment[seqid].empty()){
            seq_comment[seqid]=std::string(seq->comment.s);
        }
        
        for(int64_t pos=0; pos < static_cast<int64_t>(seq->seq.l); pos++){
            assert(seq_stats[seqid][pos]==0);
            
            pos_stats_uncompr_t psu = {0,{0,0,0,0},0};
            psu.nt16=nt256_nt16[static_cast<int32_t>(seq->seq.s[pos])];
            
            if (psu.nt16!=nt256_nt16[static_cast<int32_t>('N')]){
                for(int32_t i=0;i<4;i++){
                    psu.counters[i]= ( (0x1<<i) & psu.nt16) ? std::min(params.init_ref_weight,max_counter_value) : 0;
                }
            }
            
            psu.sum=psu.counters[0]+psu.counters[1]+psu.counters[2]+psu.counters[3];
            seq_stats[seqid][pos]=compress_position_stats(psu);
        }
    }
    kseq_destroy(seq); // STEP 5: destroy seq
    gzclose(fp); // STEP 6: close the file handler
    return 0;
}
{'fraction_non_alphanumeric': 0.19405293915434857, 'fraction_numerical': 0.02406325197662427, 'mean_word_length': 3.5003866976024747, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '561503', 'n_tokens_mistral': 2308, 'n_tokens_neox': 2232, 'n_words': 295}