<commit_before>#ifndef BIT_BLOCK_RECOMBINER_HPP_
#define BIT_BLOCK_RECOMBINER_HPP_

#include "clotho/utility/bit_block_iterator.hpp"
#include "clotho/utility/bit_fill.hpp"

namespace clotho {
namespace recombine {

struct copy_matching_classify_mismatch {};

template < class Classifier, class Tag = copy_matching_classify_mismatch >
class bit_block_recombiner {
public:
    typedef Classifier classifier_type;

    bit_block_recombiner( const classifier_type & cfier ) : m_cfier( cfier ) {}

    template < class Block, class ElementIterator >
    Block operator()( Block b0, Block b1, ElementIterator first ) {
        typedef Block block_type;
//        typedef clotho::utility::bit_block_iterator< block_type > iterator;
/*
        // b0 = 01001
        // b1 = 10101
        block_type matching = (b0 & b1); // = 00001
        block_type mismatch = (b0 ^ b1); // = 11100

        block_type b0_mask = (block_type)0;
        if( mismatch ) {
            iterator bit_it( mismatch ), bit_end;
            do { 
                unsigned int bit_idx = (*bit_it);
                if( m_cfier(first, bit_idx) ) {
                    b0_mask |= ((block_type)1 << (bit_idx));
                }
            } while( ++bit_it != bit_end);
            // assume:
            // classify( bit_idx = 2 ) true
            // classify( bit_idx = 3 ) true
            // classify( bit_idx = 4 ) false
            // => b0_mask = 01100
        }

        // ~b0_mask & mismatch = ~01100 & 11100 = 10011 & 11100 = 10000
        // b1_mismatch = (10000 & 10101) = 10000
        // res = (matching | b1_mismatch) = (00001 | 10000) = 10001
        // res = res | (b0_mask & b0) = (10001 | (01100 & 01001)) = (10001 | (01000)) = 11001
        block_type b1_mismatch = ((~b0_mask & mismatch) & b1);
        block_type res = (matching | b1_mismatch);
        res |= (b0_mask & b0);
        return  res;
*/
        // Simplified Logic:
        // b0 = 01001
        // b1 = 10101
        block_type hets = (b0 ^ b1);    // = 11100
        block_type b0_mask = (block_type)0;
        if( hets ) {
//            iterator bit_it( hets ), bit_end;
//            do {
//               unsigned int bit_idx = (*bit_it);
//                if( m_cfier(first, bit_idx ) ) {
//                    b0_mask |= ((block_type)1 << bit_idx);
//                }
//            } while( ++bit_it != bit_end );

//            unsigned int indices[ sizeof( block_type ) * 8];

            // preprocess set bits 
            //
//            unsigned int max_index = fill_set_bits( hets, indices );
//
//            for( unsigned int offset = 0; offset < max_index; ++offset ) {
//                unsigned int idx = indices[offset];
//                if( m_cfier( *(first + idx ) ) ) {
//                    b0_mask |= ((block_type) 1 << idx);
//                }
//            }
            // assume:
            // classify( bit_idx = 2 ) true
            // classify( bit_idx = 3 ) true
            // classify( bit_idx = 4 ) false
            // => b0_mask = 01100

            b0_mask = block_logic256( hets, first );
        }
        // b0 & b0_mask = (01001  &  01100) = 01000
        // b1 & ~b0_mask = (10101 & ~01100) = (10101 & 10011) = 10001
        // res = (01000 | 10001) = 11001
        block_type res = ((b0 & b0_mask) | (b1 & ~b0_mask));

        return res;
    }

    virtual ~bit_block_recombiner() {}

protected:
    classifier_type m_cfier;

#define BIT_SET_EVEN_1( z )                     \<commit_msg>clean<commit_after>#ifndef BIT_BLOCK_RECOMBINER_HPP_
#define BIT_BLOCK_RECOMBINER_HPP_

//#include "clotho/utility/bit_block_iterator.hpp"
//#include "clotho/utility/bit_fill.hpp"

namespace clotho {
namespace recombine {

struct copy_matching_classify_mismatch {};

template < class Classifier, class Tag = copy_matching_classify_mismatch >
class bit_block_recombiner {
public:
    typedef Classifier classifier_type;

    bit_block_recombiner( const classifier_type & cfier ) : m_cfier( cfier ) {}

    template < class Block, class ElementIterator >
    Block operator()( Block b0, Block b1, ElementIterator first ) {
        typedef Block block_type;
        // Simplified Logic:
        // b0 = 01001
        // b1 = 10101
        block_type hets = (b0 ^ b1);    // = 11100
        block_type b0_mask = (block_type)0;
        if( hets ) {
            // assume:
            // classify( bit_idx = 2 ) true
            // classify( bit_idx = 3 ) true
            // classify( bit_idx = 4 ) false
            // => b0_mask = 01100

            b0_mask = block_logic256( hets, first );
        }
        // b0 & b0_mask = (01001  &  01100) = 01000
        // b1 & ~b0_mask = (10101 & ~01100) = (10101 & 10011) = 10001
        // res = (01000 | 10001) = 11001
        block_type res = ((b0 & b0_mask) | (b1 & ~b0_mask));

        return res;
    }

    virtual ~bit_block_recombiner() {}

protected:
    classifier_type m_cfier;

#define BIT_SET_EVEN_1( z )                     \
{'fraction_non_alphanumeric': 0.12077294685990338, 'fraction_numerical': 0.06863929146537842, 'mean_word_length': 1.7947131608548932, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4503444', 'n_tokens_mistral': 1895, 'n_tokens_neox': 1613, 'n_words': 380}