<commit_before>/*

  This file contains specialization for strings and boolean values.

  The generic handle for numbers can be found in: attribute.h
*/
#include <cstring>
#include "attribute.h"

//#include <boost/log/trivial.hpp>

//namespace logging = boost::log;

// this is the definition of null(s) for values that are empty (for example in SciDB binary format)
const char Attribute::nullValues[Attribute::nullValuesSize] = {'\0','\0','\0','\0','\0','\0','\0','\0','\0','\0','\0','\0','\0','\0','\0','\0','\0','\0','\0','\0'};

Attribute* new_clone(Attribute const& other) {
    return other.clone();
}

Attribute::~Attribute() {}

//######################################################################################################
// Specialization for strings

GenericAttribute<char*>::GenericAttribute(bool isNullable) {
    this->value=NULL;
    this->isNullable=isNullable;
    this->bytesNumber=-1; // this is specific for every string
}

GenericAttribute<char*>::~GenericAttribute() {

}

void GenericAttribute<char*>::freeValue() {
    if (this->value != NULL) {
        delete [] this->value;
    }
    this->value = NULL;
}

int GenericAttribute<char*>::postgresReadBinary(FILE *fp) {
// string is handled differently than other types
    fread(&this->bytesNumber,4,1,fp);
//std::cout << "value bytes number before endianness: " << valueBytesNumber << std::endl;
    this->bytesNumber = be32toh(this->bytesNumber);
//std::cout << "value bytes number after endianness: " << valueBytesNumber << std::endl;
    if (this->bytesNumber == -1) {
// this is null and there are no bytes for the value
        this->isNull=true;
        return 0;
    }
    this->isNull=false;
    this->value = new char[this->bytesNumber+1]; // +1 is for NULL \0 value at the end of the string: SciDB adds \0 at the end of each string
    this->value[this->bytesNumber]='\0';
//std::cout << "value bytes number: " << valueBytesNumber << std::endl;
    fread(this->value,this->bytesNumber,1,fp);
// std::cout << "value read: " << value << std::endl;
    return 0; // success
}

int GenericAttribute<char*>::writeCsv(std::ofstream & ofile) {
// write nothing if the value is NULL
    if (this->isNull) return 0;
// write the value only if it's not NULL
// fprintf(fp,"%" PRId64,this->value);
//printf("%.*s",this->bytesNumber,value);
    ofile.width(this->bytesNumber);
    ofile << this->value;
    freeValue();
    return 0;
}

int GenericAttribute<char*>::scidbWriteBinary(FILE *fp) {
    if(this->isNullable) {
// we have to add one byte with info it it is null or not
        if(this->isNull) {
// we don't know the reason why it is null so we'll write byte 0
            char nullReason=0;
            fwrite(&nullReason,1,1,fp);
// this is indeed null, so only write -1 as the number of bytes
// and don't write the NULL value
            uint32_t scidbBytesNumber = 0;
            fwrite(&scidbBytesNumber,4,1,fp);
            return 0;
        } else {<commit_msg>data migrator generates data in csv,postgres binary,scidb binary formats from s-store<commit_after>/*

  This file contains specialization for strings and boolean values.

  The generic handle for numbers can be found in: attribute.h
*/
#include <cstring>
#include "attribute.h"

//#include <boost/log/trivial.hpp>

//namespace logging = boost::log;

// this is the definition of null(s) for values that are empty (for example in SciDB binary format)
const char Attribute::nullValues[Attribute::nullValuesSize] = {'\0','\0','\0','\0','\0','\0','\0','\0','\0','\0','\0','\0','\0','\0','\0','\0','\0','\0','\0','\0'};

Attribute* new_clone(Attribute const& other) {
    return other.clone();
}

Attribute::~Attribute() {}

//######################################################################################################
// Specialization for strings

GenericAttribute<char*>::GenericAttribute(bool isNullable) {
    this->value=NULL;
    this->isNullable=isNullable;
    this->bytesNumber=-1; // this is specific for every string
}

GenericAttribute<char*>::~GenericAttribute() {

}

void GenericAttribute<char*>::freeValue() {
    if (this->value != NULL) {
        delete [] this->value;
    }
    this->value = NULL;
}

int GenericAttribute<char*>::postgresReadBinary(FILE *fp) {
// string is handled differently than other types
    fread(&this->bytesNumber,4,1,fp);
//std::cout << "value bytes number before endianness: " << valueBytesNumber << std::endl;
    this->bytesNumber = be32toh(this->bytesNumber);
//std::cout << "value bytes number after endianness: " << valueBytesNumber << std::endl;
    if (this->bytesNumber == -1) {
// this is null and there are no bytes for the value
        this->isNull=true;
        return 0;
    }
    this->isNull=false;
    this->value = new char[this->bytesNumber+1]; // +1 is for NULL \0 value at the end of the string: SciDB adds \0 at the end of each string
    this->value[this->bytesNumber]='\0';
//std::cout << "value bytes number: " << valueBytesNumber << std::endl;
    fread(this->value,this->bytesNumber,1,fp);
// std::cout << "value read: " << value << std::endl;
    return 0; // success
}

int GenericAttribute<char*>::writeCsv(std::ofstream & ofile) {
// write nothing if the value is NULL
    if (this->isNull) return 0;
// write the value only if it's not NULL
// fprintf(fp,"%" PRId64,this->value);
//printf("%.*s",this->bytesNumber,value);
    //ofile.width(this->bytesNumber);
    ofile << this->value;
    freeValue();
    return 0;
}

int GenericAttribute<char*>::scidbWriteBinary(FILE *fp) {
    if(this->isNullable) {
// we have to add one byte with info it it is null or not
        if(this->isNull) {
// we don't know the reason why it is null so we'll write byte 0
            char nullReason=0;
            fwrite(&nullReason,1,1,fp);
// this is indeed null, so only write -1 as the number of bytes
// and don't write the NULL value
            uint32_t scidbBytesNumber = 0;
            fwrite(&scidbBytesNumber,4,1,fp);
            return 0;
        } else {
{'fraction_non_alphanumeric': 0.1930379746835443, 'fraction_numerical': 0.01632245169886742, 'mean_word_length': 3.6478328173374615, 'pattern_counts': {'":': 0, '<': 71, '<?xml version=': 0, '>': 69, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1784594', 'n_tokens_mistral': 1873, 'n_tokens_neox': 1827, 'n_words': 532}