<commit_before>// Sietronics Sieray CPI format
// Licence: Lesser GNU Public License 2.1 (LGPL)

#define BUILDING_XYLIB
#include "cpi.h"
#include "util.h"

using namespace std;
using namespace xylib::util;

namespace xylib {


const FormatInfo CpiDataSet::fmt_info(
    "cpi",
    "Sietronics Sieray CPI",
    "cpi",
    false,                      // whether binary
    false,                      // whether has multi-blocks
    &CpiDataSet::ctor,
    &CpiDataSet::check
);

bool CpiDataSet::check(istream &f, string*)
{
    string line;
    getline(f, line);
    return str_startwith(line, "SIETRONICS XRD SCAN");
}

void CpiDataSet::load_data(std::istream &f)
{
    /* format example:
        SIETRONICS XRD SCAN
        10.00
        155.00
        0.010
        Cu
        1.54056
        1-1-1900
        0.600
        HH117 CaO:Nb2O5 neutron batch .0
        SCANDATA
        8992
        9077
        9017
        9018
        9129
        9057
        ...
    */

    Block* blk = new Block;

    string s;
    getline (f, s); // first line
    getline (f, s);//xmin
    double xmin = my_strtod(s);
    getline (f, s); //xmax
    getline (f, s); //xstep
    double xstep = my_strtod(s);
    StepColumn *xcol = new StepColumn(xmin, xstep);
    blk->add_column(xcol);

    // ignore the rest of the header
    while (getline(f, s))
        if (str_startwith(s, "SCANDATA"))
            break;
    format_assert(this, f, "missing SCANDATA");

    // data
    VecColumn *ycol = new VecColumn();
    while (getline(f, s))
        ycol->add_val(my_strtod(s));

    blk->add_column(ycol);
    add_block(blk);
}

} // namespace xylib
<commit_msg>fix compilation with MSVC++<commit_after>// Sietronics Sieray CPI format
// Licence: Lesser GNU Public License 2.1 (LGPL)

#define BUILDING_XYLIB
#include "cpi.h"
#include "util.h"

using namespace std;
using namespace xylib::util;

namespace xylib {


const FormatInfo CpiDataSet::fmt_info(
    "cpi",
    "Sietronics Sieray CPI",
    "cpi",
    false,                      // whether binary
    false,                      // whether has multi-blocks
    &CpiDataSet::ctor,
    &CpiDataSet::check
);

bool CpiDataSet::check(istream &f, string*)
{
    string line;
    getline(f, line);
    return str_startwith(line, "SIETRONICS XRD SCAN");
}

void CpiDataSet::load_data(std::istream &f)
{
    /* format example:
        SIETRONICS XRD SCAN
        10.00
        155.00
        0.010
        Cu
        1.54056
        1-1-1900
        0.600
        HH117 CaO:Nb2O5 neutron batch .0
        SCANDATA
        8992
        9077
        9017
        9018
        9129
        9057
        ...
    */

    Block* blk = new Block;

    string s;
    getline (f, s); // first line
    getline (f, s);//xmin
    double xmin = my_strtod(s);
    getline (f, s); //xmax
    getline (f, s); //xstep
    double xstep = my_strtod(s);
    StepColumn *xcol = new StepColumn(xmin, xstep);
    blk->add_column(xcol);

    // ignore the rest of the header
    while (getline(f, s))
        if (str_startwith(s, "SCANDATA"))
            break;
    format_assert(this, !f.eof(), "missing SCANDATA");

    // data
    VecColumn *ycol = new VecColumn();
    while (getline(f, s))
        ycol->add_val(my_strtod(s));

    blk->add_column(ycol);
    add_block(blk);
}

} // namespace xylib

{'fraction_non_alphanumeric': 0.12119384986433525, 'fraction_numerical': 0.03678022309315647, 'mean_word_length': 2.002714932126697, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5724638', 'n_tokens_mistral': 1272, 'n_tokens_neox': 1187, 'n_words': 297}