<commit_before>#ifndef UTENSOR_VM_TESTS
#define UTENSOR_VM_TESTS

#include "test.hpp"
#include "tensorIdxImporter.hpp"

class vmTest : public Test {
  public:
    TensorIdxImporter t_import;
    void loadTest() {
        testStart("load test");
        Tensor* t = t_import.ubyte_import("/fs/testData/idxImport/uint8_4d_power2.idx", "uchar1");
        unsigned char* data = t->write<unsigned char>(0, 0);
        string tmp_file = "/fs/testData/tmp.txt";
        vm g;
        unsigned char* data_g = (unsigned char*)malloc(t->unit_size() * t->getSize());
        g.load_data<unsigned char>(tmp_file, t->unit_size(), 30, t->getSize(), 0,  data_g);
        uint32_t res_x = 0;
        uint32_t res_y = 0;
        for (unsigned int i = 0; i < 30; i++) {
           res_x += data_g[i];
           res_y += data[i];
        }
        g.load_data<unsigned char>(tmp_file, t->unit_size(), 30, t->getSize(), 30,  data_g);
        for (unsigned int i = 0; i < 30; i++) {
           res_x += data_g[i];
           res_y += data[i + 30];
        }

        passed(res_x == res_y);
        free(data_g);
    }
    void writeTest(void) {
      
        testStart("write test");
        Tensor* t = t_import.ubyte_import("/fs/testData/idxImport/uint8_4d_power2.idx", "uchar1");
        unsigned char* data = t->write<unsigned char>(0, 0);
        string tmp_file = "/fs/testData/tmp2.txt";
        vm g;
        FILE *buf = g.createFile(tmp_file);  
        uint8_t size = (uint8_t)t->unit_size();
        uint32_t totalsize = t->getSize();
        g.flush_data<unsigned char>(tmp_file, t->unit_size(), 30, t->getSize(), 0,  data);
        g.flush_data<unsigned char>(tmp_file, t->unit_size(), 30, t->getSize(), 30,  data + 30);
        unsigned char* data_g = (unsigned char*)malloc(t->unit_size() * 30);<commit_msg>  1. modify test case in vm class, which will generate tmp data file automatically.<commit_after>#ifndef UTENSOR_VM_TESTS
#define UTENSOR_VM_TESTS

#include "test.hpp"
#include "tensorIdxImporter.hpp"

class vmTest : public Test {
  public:
    TensorIdxImporter t_import;
    void loadTest() {
        testStart("load test");
        Tensor* t = t_import.ubyte_import("/fs/testData/idxImport/uint8_4d_power2.idx", "uchar1");
        unsigned char* data = t->write<unsigned char>(0, 0);
        string tmp_file = "/fs/tmp/tmp.txt";
        vm g;
        unsigned char* data_g = (unsigned char*)malloc(t->unit_size() * t->getSize());
        g.createFile(tmp_file);
        g.flush_data<unsigned char>(tmp_file, t->unit_size(), 30, t->getSize(), 0,  data);
        g.load_data<unsigned char>(tmp_file, t->unit_size(), 30, t->getSize(), 0,  data_g);
        uint32_t res_x = 0;
        uint32_t res_y = 0;
        for (unsigned int i = 0; i < 30; i++) {
           res_x += data_g[i];
           res_y += data[i];
        }
        g.flush_data<unsigned char>(tmp_file, t->unit_size(), 30, t->getSize(), 30,  data + 30);
        g.load_data<unsigned char>(tmp_file, t->unit_size(), 30, t->getSize(), 30,  data_g);
        for (unsigned int i = 0; i < 30; i++) {
           res_x += data_g[i];
           res_y += data[i + 30];
        }

        passed(res_x == res_y);
        free(data_g);
    }
    void writeTest(void) {
      
        testStart("write test");
        Tensor* t = t_import.ubyte_import("/fs/testData/idxImport/uint8_4d_power2.idx", "uchar1");
        unsigned char* data = t->write<unsigned char>(0, 0);
        string tmp_file = "/fs/tmp/tmp2.txt";
        vm g;
        FILE *buf = g.createFile(tmp_file);  
        uint8_t size = (uint8_t)t->unit_size();
        uint32_t totalsize = t->getSize();
        g.flush_data<unsigned char>(tmp_file, t->unit_size(), 30, t->getSize(), 0,  data);
        g.flush_data<unsigned char>(tmp_file, t->unit_size(), 30, t->getSize(), 30,  data + 30);
        unsigned char* data_g = (unsigned char*)malloc(t->unit_size() * 30);
{'fraction_non_alphanumeric': 0.15792181069958847, 'fraction_numerical': 0.027777777777777776, 'mean_word_length': 2.5942698706099816, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6380286', 'n_tokens_mistral': 1451, 'n_tokens_neox': 1369, 'n_words': 303}