<commit_before>//License MIT 2017 <NAME>

#include <iostream>
#include <cstdlib>
#include <string>
#include <vector>
#include <cmath>
#include <map>
#include "UnrestrictedMultiShiftAnd.hpp"

using namespace std;

/**
 * @constructor An alphabet must be supplied of the letters that will be used in the patterns
 *
 * @param alphabet A list of characters used in the text/patterns e.g. ACGT
 * @param reportPatternPositions Report positions when matches found?
 */
UnrestrictedMultiShiftAnd::UnrestrictedMultiShiftAnd(const string & alphabet, bool reportPatternPositions)
{
    this->alphabet = alphabet;
    this->reportPatterns = reportPatternPositions;
    this->N = 0;
    this->M = 0;
    this->L = 1;
    this->Sv.push_back(0ul);
    this->Ev.push_back(0ul);
    this->D.push_back(0ul);
    vector<WORD> v;
    unsigned int i;
    for (i = 0; i < this->alphabet.length(); i++)
    {
        this->Sigma[(int)alphabet[i]] = i + 1;
        v.push_back(0ul);
    }
    this->Bv.push_back(v);
}

/**
 * Add a pattern
 *
 * @param pattern
 */
void UnrestrictedMultiShiftAnd::addPattern(const string & pattern)
{
    unsigned int i, j, m = pattern.length();

    //if we need more memory to hold the new pattern then create and initialize new words
    unsigned int l = (unsigned int) ceil((double)(this->M + m) / (double)BITSINWORD);
    unsigned int numWordsDiff = l - this->L;
    if (numWordsDiff > 0)
    {
        for (i = 0; i < numWordsDiff; i++)
        {
            this->Sv.push_back(0ul);
            this->Ev.push_back(0ul);
            this->D.push_back(0ul);
        }
        for (i = 0; i < numWordsDiff; i++)
        {
            vector<WORD> v;
            for (j = 0; j < this->alphabet.length(); j++)
            {
                v.push_back(0ul);
            }
            this->Bv.push_back(v);
        }
    }

    //process the pattern for Bitvector Bv
    int charIdx;
    unsigned int currWordIdx = (unsigned int) ((float)this->M / (float)BITSINWORD);
    unsigned int currBitIdx = this->M % BITSINWORD;
    for (i = 0; i < m; i++)
    {
        charIdx = (int) this->Sigma[(int)pattern[i]];
        if (charIdx > 0) {
            this->Bv[currWordIdx][charIdx - 1] = this->Bv[currWordIdx][charIdx - 1] | (1ul << currBitIdx);
        }
        currBitIdx = ++currBitIdx % BITSINWORD;
        if (currBitIdx == 0) {
            currWordIdx++;
        }
    }

    //process the pattern to set Start bit in Sv
    currWordIdx = (unsigned int) ((float)this->M / (float)BITSINWORD);
    currBitIdx = this->M % BITSINWORD;
    this->Sv[currWordIdx] = this->Sv[currWordIdx] | (1ul << currBitIdx);

    //process the pattern to set End bit in Ev and update the class variables
    this->L = this->L + numWordsDiff;
    this->M = this->M + m;
    this->N = this->N + 1;
    this->Ev[this->L - 1] = this->Ev[this->L - 1] | (1ul << ((this->M % BITSINWORD) - 1));

    //keep a record of position the pattern is stored and pattern id - for search results
    if (this->reportPatterns) {
        this->positions.insert(pair<int,int>(this->M - 1, this->N - 1));
    }
}

/**
 * Search a text<commit_msg>bug fix / refactoring<commit_after>//License MIT 2017 <NAME>

#include <iostream>
#include <cstdlib>
#include <string>
#include <vector>
#include <cmath>
#include <map>
#include "UnrestrictedMultiShiftAnd.hpp"

using namespace std;

/**
 * @constructor An alphabet must be supplied of the letters that will be used in the patterns
 *
 * @param alphabet A list of characters used in the text/patterns e.g. ACGT
 * @param reportPatternPositions Report positions when matches found?
 */
UnrestrictedMultiShiftAnd::UnrestrictedMultiShiftAnd(const string & alphabet, bool reportPatternPositions)
{
    this->alphabet = alphabet;
    this->reportPatterns = reportPatternPositions;
    this->N = 0;
    this->M = 0;
    this->L = 1;
    this->Sv.push_back(0ul);
    this->Ev.push_back(0ul);
    this->D.push_back(0ul);
    vector<WORD> v;
    unsigned int i;
    for (i = 0; i < this->alphabet.length(); i++)
    {
        this->Sigma[(int)alphabet[i]] = i + 1;
        v.push_back(0ul);
    }
    this->Bv.push_back(v);
}

/**
 * Add a pattern
 *
 * @param pattern
 */
void UnrestrictedMultiShiftAnd::addPattern(const string & pattern)
{
    int charIdx;
    unsigned int wordIdx, bitIdx, i, j, m = pattern.length();
    unsigned int lastIdx = m - 1;

    for (i = 0; i < m; i++)
    {
        charIdx = (int) this->Sigma[(int)pattern[i]];
        wordIdx = (unsigned int) ((double)(this->M + i) / (double)BITSINWORD);
        bitIdx = (unsigned int) ((this->M + i) % BITSINWORD);

        //expand memory if required
        if (this->L == wordIdx)
        {
            this->Sv.push_back(0ul);
            this->Ev.push_back(0ul);
            this->D.push_back(0ul);
            vector<WORD> v;
            for (j = 0; j < this->alphabet.length(); j++)
            {
                v.push_back(0ul);
            }
            this->Bv.push_back(v);
            this->L = this->L + 1;
        }

        //mark letter position of pattern
        if (charIdx > 0) {
            this->Bv[wordIdx][charIdx - 1] = this->Bv[wordIdx][charIdx - 1] | (1ul << bitIdx);
        }
        //mark start position of pattern
        if (i == 0) {
            this->Sv[wordIdx] = this->Sv[wordIdx] | (1ul << bitIdx);
        }
        //mark end position of pattern
        if (i == lastIdx) {
            this->Ev[wordIdx] = this->Ev[wordIdx] | (1ul << bitIdx);
        }
    }

    this->M = this->M + m;

    //keep a record of position the pattern is stored and pattern id - for search results
    if (this->reportPatterns) {
        this->positions.insert(pair<int,int>(this->M - 1, this->N));
    }

    this->N = this->N + 1;
}

/**
 * Search a text
{'fraction_non_alphanumeric': 0.1464304325169359, 'fraction_numerical': 0.011290602744484974, 'mean_word_length': 2.3379710144927537, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 104, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3802831', 'n_tokens_mistral': 1983, 'n_tokens_neox': 1903, 'n_words': 509}