<commit_before>// Copyright <NAME> 2015 hughperkins at gmail
//
// This Source Code Form is subject to the terms of the Mozilla Public License, 
// v. 2.0. If a copy of the MPL was not distributed with this file, You can 
// obtain one at http://mozilla.org/MPL/2.0/.

#include <algorithm>
#include <iostream>
#include <stdexcept>

#include "GenericLoader.h"

#include "BatchProcess.h"

using namespace std;

template< typename T>
void BatchProcess::run(std::string filepath, int startN, int batchSize, int totalN, int cubeSize, BatchAction<T> *batchAction) {
    int numBatches = ( totalN + batchSize - 1 ) / batchSize;
    int thisBatchSize = batchSize;
//    cout << "batchProcess::run batchsize " << batchSize << " startN " << startN << " totalN " << totalN << " numBatches " << numBatches << endl;
    for( int batch = 0; batch < numBatches; batch++ ) {
        int batchStart = batch * batchSize;
        if( batch == numBatches - 1 ) {
            thisBatchSize = totalN - batchStart;
//            cout << "size of last batch: " << thisBatchSize << endl;
        }
//        cout << "   batchStart " << batchStart << " thisBatchSize " << thisBatchSize << endl;
        GenericLoader::load( filepath, batchAction->data, batchAction->labels, batchStart, thisBatchSize );
        batchAction->processBatch( thisBatchSize, cubeSize );
    }
}

template void BatchProcess::run<unsigned char>( std::string filepath, int startN, int batchSize, int totalN, int cubeSize, BatchAction<unsigned char> *batchAction);
<commit_msg>exports for windows<commit_after>// Copyright <NAME> 2015 hugh<EMAIL>kins at gmail
//
// This Source Code Form is subject to the terms of the Mozilla Public License, 
// v. 2.0. If a copy of the MPL was not distributed with this file, You can 
// obtain one at http://mozilla.org/MPL/2.0/.

#include <algorithm>
#include <iostream>
#include <stdexcept>

#include "GenericLoader.h"

#include "BatchProcess.h"

#include "DllImportExport.h"

using namespace std;

template< typename T>
void BatchProcess::run(std::string filepath, int startN, int batchSize, int totalN, int cubeSize, BatchAction<T> *batchAction) {
    int numBatches = ( totalN + batchSize - 1 ) / batchSize;
    int thisBatchSize = batchSize;
//    cout << "batchProcess::run batchsize " << batchSize << " startN " << startN << " totalN " << totalN << " numBatches " << numBatches << endl;
    for( int batch = 0; batch < numBatches; batch++ ) {
        int batchStart = batch * batchSize;
        if( batch == numBatches - 1 ) {
            thisBatchSize = totalN - batchStart;
//            cout << "size of last batch: " << thisBatchSize << endl;
        }
//        cout << "   batchStart " << batchStart << " thisBatchSize " << thisBatchSize << endl;
        GenericLoader::load( filepath, batchAction->data, batchAction->labels, batchStart, thisBatchSize );
        batchAction->processBatch( thisBatchSize, cubeSize );
    }
}

template ClConvolve_EXPORT void BatchProcess::run<unsigned char>( std::string filepath, int startN, int batchSize, int totalN, int cubeSize, BatchAction<unsigned char> *batchAction);

{'fraction_non_alphanumeric': 0.12612612612612611, 'fraction_numerical': 0.007078507078507079, 'mean_word_length': 3.4798270893371757, 'pattern_counts': {'":': 0, '<': 90, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2317590', 'n_tokens_mistral': 967, 'n_tokens_neox': 924, 'n_words': 299}