<commit_before>// Copyright <NAME> 2015 h<EMAIL>ins at gmail
//
// This Source Code Form is subject to the terms of the Mozilla Public License, 
// v. 2.0. If a copy of the MPL was not distributed with this file, You can 
// obtain one at http://mozilla.org/MPL/2.0/.

#include <iostream>
#include <string>
#include <cstring>
#include <fstream>
#include <stdexcept>

#include "util/FileHelper.h"
#include "util/stringhelper.h"
#include "ManifestLoaderv1.h"
#include "util/JpegHelper.h"

#include "DeepCLDllExport.h"

using namespace std;

#undef STATIC
#undef VIRTUAL
#define STATIC
#define VIRTUAL

PUBLIC STATIC bool ManifestLoaderv1::isFormatFor(std::string imagesFilepath) {
    cout << "ManifestLoaderv1 checking format for " << imagesFilepath << endl;
    string sigString = "# format=deepcl-jpeg-list-v1 ";
    char *headerBytes = FileHelper::readBinaryChunk(imagesFilepath, 0, sigString.length() + 1);
    headerBytes[sigString.length()] = 0;
    bool matched = string(headerBytes) == sigString;
    cout << "matched: " << matched << endl;
    return matched;
}
PUBLIC ManifestLoaderv1::ManifestLoaderv1(std::string imagesFilepath) {
    init(imagesFilepath);
}
PRIVATE void ManifestLoaderv1::init(std::string imagesFilepath) {
    this->imagesFilepath = imagesFilepath;
    // by reading the number of lines in the manifest, we can get the number of examples, *p_N
    // number of planes is .... 1
    // imageSize is ...

    if(!isFormatFor(imagesFilepath) ) {
        throw runtime_error("file " + imagesFilepath + " is not a deepcl-jpeg-list-v1 manifest file");
    }

    ifstream infile(imagesFilepath);
    char lineChars[1024];
    infile.getline(lineChars, 1024); // skip first, header, line
    string firstLine = string(lineChars);
//    cout << "firstline: [" << firstLine << "]" << endl;
    vector<string> splitLine = split(firstLine, " ");
    N = readIntValue(splitLine, "N");
    planes = readIntValue(splitLine, "planes");
    size = readIntValue(splitLine, "width");
    int imageSizeRepeated = readIntValue(splitLine, "height");
    if(size != imageSizeRepeated) {
        throw runtime_error("file " + imagesFilepath + " contains non-square images.  Not handled for now.");
    }
    // now we should load into memory, since the file is not fixed-size records, and cannot be loaded partially easily

    files = new string[N];
    labels = 0;

    int n = 0;
    hasLabels = false;
    while(infile) {
        infile.getline(lineChars, 1024);
        if(!infile) {
            break;
        }
        string line = string(lineChars);
        if(line == "") {
            continue;
        }
        vector<string> splitLine = split(line, " ");
        if((int)splitLine.size() == 0) {
            continue;
        }
        if(n == 0) {
            int lineSize = (int)splitLine.size();
            if(lineSize == 2) {
                hasLabels = true;
                labels = new int[N];
            }
        }
        if(!hasLabels && (int)splitLine.size() != 1) { 
            throw runtime_error("Error reading " + imagesFilepath + ".  Following line not parseable:\n" + line);
        }
        if(hasLabels && (int)splitLine.size() != 2) { 
            throw runtime_error("Error reading " + imagesFilepath + ".  Following line not parseable:\n" + line);
        }
        string jpegFile = splitLine[0];
        #ifdef _WIN32
        if(jpegFile[1] != ':' && jpegFile[0] != '\\') {  // I guess this means its a relative path?
            vector<string> splitManifestPath = split(imagesFilepath, "\\");
            string dirPath = replace(imagesFilepath, splitManifestPath[splitManifestPath.size()-1], "");
            jpegFile = dirPath + jpegFile;
        }
        #else
        if(jpegFile[0] != '/') {  // this is a bit hacky, but at least handles linux and mac for now...
            vector<string> splitManifestPath = split(imagesFilepath, "/");
            string dirPath = replace(imagesFilepath, splitManifestPath[splitManifestPath.size()-1], "");
            jpegFile = dirPath + jpegFile;
        }
        #endif
        files[n] = jpegFile;
        if(hasLabels) {
            int label = atoi(splitLine[1]);
            labels[n] = label;
        }
//        cout << "file " << jpegFile << " label=" << label << endl;
        n++;
    }
    infile.close();
    if(n != N) {
        throw runtime_error("Error: number of images declared in manifest " + toString(N) + " doesnt match number actually in manifest " + to_string(n));
    }

    cout << "manifest " << imagesFilepath << " read. N=" << N << " planes=" << planes << " size=" << size << " labels? " << hasLabels << endl;
}
PUBLIC VIRTUAL std::string ManifestLoaderv1::getType() {
    return "ManifestLoaderv1";
}
PUBLIC VIRTUAL int ManifestLoaderv1::getImageCubeSize() {
    return planes * size * size;
}<commit_msg>another tostring<commit_after>// Copyright <NAME> 2015 hughperkins at gmail
//
// This Source Code Form is subject to the terms of the Mozilla Public License, 
// v. 2.0. If a copy of the MPL was not distributed with this file, You can 
// obtain one at http://mozilla.org/MPL/2.0/.

#include <iostream>
#include <string>
#include <cstring>
#include <fstream>
#include <stdexcept>

#include "util/FileHelper.h"
#include "util/stringhelper.h"
#include "ManifestLoaderv1.h"
#include "util/JpegHelper.h"

#include "DeepCLDllExport.h"

using namespace std;

#undef STATIC
#undef VIRTUAL
#define STATIC
#define VIRTUAL

PUBLIC STATIC bool ManifestLoaderv1::isFormatFor(std::string imagesFilepath) {
    cout << "ManifestLoaderv1 checking format for " << imagesFilepath << endl;
    string sigString = "# format=deepcl-jpeg-list-v1 ";
    char *headerBytes = FileHelper::readBinaryChunk(imagesFilepath, 0, sigString.length() + 1);
    headerBytes[sigString.length()] = 0;
    bool matched = string(headerBytes) == sigString;
    cout << "matched: " << matched << endl;
    return matched;
}
PUBLIC ManifestLoaderv1::ManifestLoaderv1(std::string imagesFilepath) {
    init(imagesFilepath);
}
PRIVATE void ManifestLoaderv1::init(std::string imagesFilepath) {
    this->imagesFilepath = imagesFilepath;
    // by reading the number of lines in the manifest, we can get the number of examples, *p_N
    // number of planes is .... 1
    // imageSize is ...

    if(!isFormatFor(imagesFilepath) ) {
        throw runtime_error("file " + imagesFilepath + " is not a deepcl-jpeg-list-v1 manifest file");
    }

    ifstream infile(imagesFilepath);
    char lineChars[1024];
    infile.getline(lineChars, 1024); // skip first, header, line
    string firstLine = string(lineChars);
//    cout << "firstline: [" << firstLine << "]" << endl;
    vector<string> splitLine = split(firstLine, " ");
    N = readIntValue(splitLine, "N");
    planes = readIntValue(splitLine, "planes");
    size = readIntValue(splitLine, "width");
    int imageSizeRepeated = readIntValue(splitLine, "height");
    if(size != imageSizeRepeated) {
        throw runtime_error("file " + imagesFilepath + " contains non-square images.  Not handled for now.");
    }
    // now we should load into memory, since the file is not fixed-size records, and cannot be loaded partially easily

    files = new string[N];
    labels = 0;

    int n = 0;
    hasLabels = false;
    while(infile) {
        infile.getline(lineChars, 1024);
        if(!infile) {
            break;
        }
        string line = string(lineChars);
        if(line == "") {
            continue;
        }
        vector<string> splitLine = split(line, " ");
        if((int)splitLine.size() == 0) {
            continue;
        }
        if(n == 0) {
            int lineSize = (int)splitLine.size();
            if(lineSize == 2) {
                hasLabels = true;
                labels = new int[N];
            }
        }
        if(!hasLabels && (int)splitLine.size() != 1) { 
            throw runtime_error("Error reading " + imagesFilepath + ".  Following line not parseable:\n" + line);
        }
        if(hasLabels && (int)splitLine.size() != 2) { 
            throw runtime_error("Error reading " + imagesFilepath + ".  Following line not parseable:\n" + line);
        }
        string jpegFile = splitLine[0];
        #ifdef _WIN32
        if(jpegFile[1] != ':' && jpegFile[0] != '\\') {  // I guess this means its a relative path?
            vector<string> splitManifestPath = split(imagesFilepath, "\\");
            string dirPath = replace(imagesFilepath, splitManifestPath[splitManifestPath.size()-1], "");
            jpegFile = dirPath + jpegFile;
        }
        #else
        if(jpegFile[0] != '/') {  // this is a bit hacky, but at least handles linux and mac for now...
            vector<string> splitManifestPath = split(imagesFilepath, "/");
            string dirPath = replace(imagesFilepath, splitManifestPath[splitManifestPath.size()-1], "");
            jpegFile = dirPath + jpegFile;
        }
        #endif
        files[n] = jpegFile;
        if(hasLabels) {
            int label = atoi(splitLine[1]);
            labels[n] = label;
        }
//        cout << "file " << jpegFile << " label=" << label << endl;
        n++;
    }
    infile.close();
    if(n != N) {
        throw runtime_error("Error: number of images declared in manifest " + toString(N) + " doesnt match number actually in manifest " + toString(n));
    }

    cout << "manifest " << imagesFilepath << " read. N=" << N << " planes=" << planes << " size=" << size << " labels? " << hasLabels << endl;
}
PUBLIC VIRTUAL std::string ManifestLoaderv1::getType() {
    return "ManifestLoaderv1";
}
PUBLIC VIRTUAL int ManifestLoaderv1::getImageCubeSize() {
    return planes * size * size;
}
{'fraction_non_alphanumeric': 0.13514072847682118, 'fraction_numerical': 0.010554635761589404, 'mean_word_length': 2.700229709035222, 'pattern_counts': {'":': 0, '<': 128, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6624534', 'n_tokens_mistral': 3010, 'n_tokens_neox': 2906, 'n_words': 779}