<commit_before>#include <SRC_linker_ram.hpp>


using namespace std;


/********************************************************************************/


// We define some constant strings for names of command line parameters
static const char* STR_URI_BANK_INPUT               = "-bank";
static const char* STR_URI_QUERY_INPUT              = "-query";
static const char* STR_FINGERPRINT                  = "-fingerprint_size";
static const char* STR_GAMMA                        = "-gamma";
static const char* STR_WINDOWS_SIZE                 = "-windows_size";
static const char* STR_THRESHOLD                    = "-kmer_threshold";
static const char* STR_OUT_FILE                     = "-out";
static const char* STR_CORE                         = "-core";
static const char* STR_COMMET_LIKE                  = "-no_sharing_detail";
static const char* STR_KEEP_LOW_COMPLEXITY          = "-keep_low_complexity";
static const char* STR_ZERO_DENSITY_WINDOWS_SIZE    = "-zero_density_windows_size";
static const char* STR_ZERO_DENSITY_THRESHOLD       = "-zero_density_threshold";


SRC_linker_ram::SRC_linker_ram ()  : Tool ("SRC_linker_ram"){
	// We add some custom arguments for command line interface
	getParser()->push_back (new OptionOneParam (STR_URI_GRAPH,                  "graph input",      true));
	getParser()->push_back (new OptionOneParam (STR_URI_BANK_INPUT,             "bank input",       true));
	getParser()->push_back (new OptionOneParam (STR_URI_QUERY_INPUT,            "query input",      true));
	getParser()->push_back (new OptionOneParam (STR_OUT_FILE,                   "output_file",      true));
	getParser()->push_back (new OptionOneParam (STR_THRESHOLD,                  "Minimal percentage of shared kmer span for considering 2 reads as similar.  The kmer span is the number of bases from the read query covered by a kmer shared with the target read. If a read of length 80 has a kmer-span of 60 with another read from the bank (of unkonwn size), then the percentage of shared kmer span is 75%. If a least a windows (of size \"windows_size\" contains at least kmer_threshold percent of positionf covered by shared kmers, the read couple is conserved).",    false, "75"));
	getParser()->push_back (new OptionOneParam (STR_WINDOWS_SIZE, "size of the window. If the windows size is zero (default value), then the full read is considered",    false, "0"));
    getParser()->push_back (new OptionOneParam (STR_GAMMA,                      "gamma value",      false,  "2"));
	getParser()->push_back (new OptionOneParam (STR_FINGERPRINT,                "fingerprint size", false,  "8"));
    getParser()->push_back (new OptionOneParam (STR_CORE,                       "Number of thread(s)", false,  "1"));
    getParser()->push_back (new OptionNoParam  (STR_COMMET_LIKE,                "Output ids of reads from query input that are shared with at least one read from reference bank input. With this option no information with whom a read is shared is provided, one only knows that a read is shared.", false));
    
    getParser()->push_back (new OptionNoParam  (STR_KEEP_LOW_COMPLEXITY,        "Conserve low complexity sequences during indexing and querying", false));
    getParser()->push_back (new OptionOneParam (STR_ZERO_DENSITY_WINDOWS_SIZE,  "If defined (>0): two reads are linked if they DO NOT contain a window of this size, with a percentage of zero higher than \"-zero_density_threshold\". Note: this test is performed over the full read length, not limited to \"-windows_size\"", false,  "0"));
    getParser()->push_back (new OptionOneParam (STR_ZERO_DENSITY_THRESHOLD,     "See \"-zero_density_windows_size\"", false, "80"));
}


void SRC_linker_ram::create_quasi_dictionary (int fingerprint_size, int nbCores){
	const int display = getInput()->getInt (STR_VERBOSE);
	// We get a handle on the HDF5 storage object.
	// Note that we use an auto pointer since the StorageFactory dynamically allocates an instance
	auto_ptr<Storage> storage (StorageFactory(STORAGE_HDF5).load (getInput()->getStr(STR_URI_GRAPH)));
	// We get the group for dsk
	Group& dskGroup = storage->getGroup("dsk");
	kmer_size = atoi(dskGroup.getProperty("kmer_size").c_str());
	// We get the solid kmers collection 1) from the 'dsk' group  2) from the 'solid' collection
	Partition<Kmer<>::Count>& solidKmers = dskGroup.getPartition<Kmer<>::Count> ("solid");
	nbSolidKmers = solidKmers.getNbItems();
	if(nbSolidKmers==0){
		cout<<"No solid kmers in bank -- exit"<<endl;
		exit(0);
	}
	IteratorKmerH5Wrapper iteratorOnKmers (solidKmers.iterator());
	quasiDico = quasidictionaryVectorKeyGeneric<IteratorKmerH5Wrapper, u_int32_t> (nbSolidKmers, iteratorOnKmers, fingerprint_size, nbCores, gamma_value);
}


struct FunctorIndexer{
    ISynchronizer*                                                      synchro;
	quasidictionaryVectorKeyGeneric <IteratorKmerH5Wrapper, u_int32_t > &quasiDico;
	int                                                                 kmer_size;
    bool                                                                keep_low_complexity;
    Kmer<KMER_SPAN(1)>::ModelCanonical                                  model;

    FunctorIndexer(quasidictionaryVectorKeyGeneric <IteratorKmerH5Wrapper, u_int32_t >& quasiDico, int kmer_size, bool keep_low_complexity, ISynchronizer* synchro)  :
        synchro(synchro),
        quasiDico(quasiDico),
        kmer_size(kmer_size),
        keep_low_complexity(keep_low_complexity) {
        model = Kmer<KMER_SPAN(1)>::ModelCanonical(kmer_size);
	}

	void operator() (Sequence& seq){
//        read_id++;          // we do not use the seq.getIndex() id as it is limited to a read file and not a read set.
		if(not keep_low_complexity and not is_high_complexity(seq,kmer_size)){return;}
        Kmer<KMER_SPAN(1)>::ModelCanonical::Iterator itKmer (model);
		itKmer.setData (seq.getData());
        
//        if(repeated_kmers(model, itKmer)){return;}
        u_int32_t read_id = static_cast<u_int32_t>(seq.getIndex());
//        if (cur_read_id>read_id) read_id = cur_read_id;
//        cout<<" indexing seq "<<read_id<<endl;
		for (itKmer.first(); !itKmer.isDone(); itKmer.next()){
			// Adding the read id to the list of ids associated to this kmer.note that the kmer may not exist in the dictionary if it was under the solidity threshold.in this case, nothing is done
			quasiDico.set_value((itKmer)->value().getVal(), read_id);
		}
	}
};


void SRC_linker_ram::fill_quasi_dictionary (const int nbCores){
	bool exists;
	IBank* bank = Bank::open (getInput()->getStr(STR_URI_BANK_INPUT));
	cout<<"Index "<<kmer_size<<"-mers from bank "<<getInput()->getStr(STR_URI_BANK_INPUT)<<endl;
	LOCAL (bank);
	ProgressIterator<Sequence> itSeq (*bank);
	Dispatcher dispatcher (nbCores, 10000);
    ISynchronizer* synchro = System::thread().newSynchronizer();
	dispatcher.iterate (itSeq, FunctorIndexer(quasiDico, kmer_size, keep_low_complexity, synchro));
}



class FunctorQuerySpanKmers // FunctorQuery used after claires discussion: number of positions covered by a shared kmer.
{
public:<commit_msg>fix lock isssue<commit_after>#include <SRC_linker_ram.hpp>


using namespace std;


/********************************************************************************/


// We define some constant strings for names of command line parameters
static const char* STR_URI_BANK_INPUT               = "-bank";
static const char* STR_URI_QUERY_INPUT              = "-query";
static const char* STR_FINGERPRINT                  = "-fingerprint_size";
static const char* STR_GAMMA                        = "-gamma";
static const char* STR_WINDOWS_SIZE                 = "-windows_size";
static const char* STR_THRESHOLD                    = "-kmer_threshold";
static const char* STR_OUT_FILE                     = "-out";
static const char* STR_CORE                         = "-core";
static const char* STR_COMMET_LIKE                  = "-no_sharing_detail";
static const char* STR_KEEP_LOW_COMPLEXITY          = "-keep_low_complexity";
static const char* STR_ZERO_DENSITY_WINDOWS_SIZE    = "-zero_density_windows_size";
static const char* STR_ZERO_DENSITY_THRESHOLD       = "-zero_density_threshold";


SRC_linker_ram::SRC_linker_ram ()  : Tool ("SRC_linker_ram"){
	// We add some custom arguments for command line interface
	getParser()->push_back (new OptionOneParam (STR_URI_GRAPH,                  "graph input",      true));
	getParser()->push_back (new OptionOneParam (STR_URI_BANK_INPUT,             "bank input",       true));
	getParser()->push_back (new OptionOneParam (STR_URI_QUERY_INPUT,            "query input",      true));
	getParser()->push_back (new OptionOneParam (STR_OUT_FILE,                   "output_file",      true));
	getParser()->push_back (new OptionOneParam (STR_THRESHOLD,                  "Minimal percentage of shared kmer span for considering 2 reads as similar.  The kmer span is the number of bases from the read query covered by a kmer shared with the target read. If a read of length 80 has a kmer-span of 60 with another read from the bank (of unkonwn size), then the percentage of shared kmer span is 75%. If a least a windows (of size \"windows_size\" contains at least kmer_threshold percent of positionf covered by shared kmers, the read couple is conserved).",    false, "75"));
	getParser()->push_back (new OptionOneParam (STR_WINDOWS_SIZE, "size of the window. If the windows size is zero (default value), then the full read is considered",    false, "0"));
    getParser()->push_back (new OptionOneParam (STR_GAMMA,                      "gamma value",      false,  "2"));
	getParser()->push_back (new OptionOneParam (STR_FINGERPRINT,                "fingerprint size", false,  "8"));
    getParser()->push_back (new OptionOneParam (STR_CORE,                       "Number of thread(s)", false,  "1"));
    getParser()->push_back (new OptionNoParam  (STR_COMMET_LIKE,                "Output ids of reads from query input that are shared with at least one read from reference bank input. With this option no information with whom a read is shared is provided, one only knows that a read is shared.", false));
    
    getParser()->push_back (new OptionNoParam  (STR_KEEP_LOW_COMPLEXITY,        "Conserve low complexity sequences during indexing and querying", false));
    getParser()->push_back (new OptionOneParam (STR_ZERO_DENSITY_WINDOWS_SIZE,  "If defined (>0): two reads are linked if they DO NOT contain a window of this size, with a percentage of zero higher than \"-zero_density_threshold\". Note: this test is performed over the full read length, not limited to \"-windows_size\"", false,  "0"));
    getParser()->push_back (new OptionOneParam (STR_ZERO_DENSITY_THRESHOLD,     "See \"-zero_density_windows_size\"", false, "80"));
}


void SRC_linker_ram::create_quasi_dictionary (int fingerprint_size, int nbCores){
	const int display = getInput()->getInt (STR_VERBOSE);
	// We get a handle on the HDF5 storage object.
	// Note that we use an auto pointer since the StorageFactory dynamically allocates an instance
	auto_ptr<Storage> storage (StorageFactory(STORAGE_HDF5).load (getInput()->getStr(STR_URI_GRAPH)));
	// We get the group for dsk
	Group& dskGroup = storage->getGroup("dsk");
	kmer_size = atoi(dskGroup.getProperty("kmer_size").c_str());
	// We get the solid kmers collection 1) from the 'dsk' group  2) from the 'solid' collection
	Partition<Kmer<>::Count>& solidKmers = dskGroup.getPartition<Kmer<>::Count> ("solid");
	nbSolidKmers = solidKmers.getNbItems();
	if(nbSolidKmers==0){
		cout<<"No solid kmers in bank -- exit"<<endl;
		exit(0);
	}
	IteratorKmerH5Wrapper iteratorOnKmers (solidKmers.iterator());
	quasiDico = quasidictionaryVectorKeyGeneric<IteratorKmerH5Wrapper, u_int32_t> (nbSolidKmers, iteratorOnKmers, fingerprint_size, nbCores, gamma_value);
}


struct FunctorIndexer{
    ISynchronizer*                                                      synchro;
	quasidictionaryVectorKeyGeneric <IteratorKmerH5Wrapper, u_int32_t > &quasiDico;
	int                                                                 kmer_size;
    bool                                                                keep_low_complexity;
    Kmer<KMER_SPAN(1)>::ModelCanonical                                  model;

    FunctorIndexer(quasidictionaryVectorKeyGeneric <IteratorKmerH5Wrapper, u_int32_t >& quasiDico, int kmer_size, bool keep_low_complexity, ISynchronizer* synchro)  :
        synchro(synchro),
        quasiDico(quasiDico),
        kmer_size(kmer_size),
        keep_low_complexity(keep_low_complexity) {
        model = Kmer<KMER_SPAN(1)>::ModelCanonical(kmer_size);
	}

	void operator() (Sequence& seq){
//        read_id++;          // we do not use the seq.getIndex() id as it is limited to a read file and not a read set.
		if(not keep_low_complexity and not is_high_complexity(seq,kmer_size)){return;}
        Kmer<KMER_SPAN(1)>::ModelCanonical::Iterator itKmer (model);
		itKmer.setData (seq.getData());
        
//        if(repeated_kmers(model, itKmer)){return;}
        u_int32_t read_id = static_cast<u_int32_t>(seq.getIndex());
//        if (cur_read_id>read_id) read_id = cur_read_id;
//        cout<<" indexing seq "<<read_id<<endl;
		for (itKmer.first(); !itKmer.isDone(); itKmer.next()){
			// Adding the read id to the list of ids associated to this kmer.note that the kmer may not exist in the dictionary if it was under the solidity threshold.in this case, nothing is done
            synchro->lock();
			quasiDico.set_value((itKmer)->value().getVal(), read_id);
            synchro->unlock();
		}
	}
};


void SRC_linker_ram::fill_quasi_dictionary (const int nbCores){
	bool exists;
	IBank* bank = Bank::open (getInput()->getStr(STR_URI_BANK_INPUT));
	cout<<"Index "<<kmer_size<<"-mers from bank "<<getInput()->getStr(STR_URI_BANK_INPUT)<<endl;
	LOCAL (bank);
	ProgressIterator<Sequence> itSeq (*bank);
	Dispatcher dispatcher (nbCores, 10000);
    ISynchronizer* synchro = System::thread().newSynchronizer();
	dispatcher.iterate (itSeq, FunctorIndexer(quasiDico, kmer_size, keep_low_complexity, synchro));
}



class FunctorQuerySpanKmers // FunctorQuery used after claires discussion: number of positions covered by a shared kmer.
{
public:
{'fraction_non_alphanumeric': 0.11231679819616686, 'fraction_numerical': 0.006341600901916573, 'mean_word_length': 3.3927576601671308, 'pattern_counts': {'":': 0, '<': 71, '<?xml version=': 0, '>': 75, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3856020', 'n_tokens_mistral': 4351, 'n_tokens_neox': 4178, 'n_words': 1253}