<commit_before>#ifdef USE_ALLEGRO
#include <allegro.h>

/* FIXME: replace with <winalleg.h> */
#ifdef _WIN32
#define BITMAP dummyBITMAP
#include <windows.h>
#undef BITMAP
#endif

#endif
#include <algorithm>
#include "funcs.h"
#include "file-system.h"
#include "thread.h"
#include "system.h"
#include "globals.h"
#include <dirent.h>
#include <sstream>
#include <exception>
#include <string>
#include <fstream>
#include <ostream>

#include "zip/unzip.h"
#include "zip/ioapi.h"

#ifndef USE_ALLEGRO
/* some sfl symbols conflict with allegro */
#include "sfl/sfl.h"
#include "sfl/sfldir.h"
#endif

#ifdef _WIN32
#define _WIN32_IE 0x400
#include <shlobj.h>
#endif

using namespace std;

/* some filesystem access can only be done by one thread at a time. specifically, sfl
 * has its own allocator that is meant to be used in a single-threaded manner.
 * rather than try to add locks to sfl we just wrap all sfl calls with a lock.
 *
 * initialize() must be called to initialize this lock
 */
// Util::Thread::Lock lock;



namespace Path{

/* remove extra path separators (/) */
string sanitize(string path){
    size_t double_slash = path.find("//");
    while (double_slash != string::npos){
        path.erase(double_slash, 1);
        double_slash = path.find("//");
    }
    return path;
}

static int invert(int c){
    if (c == '\\'){
        return '/';
    }
    return c;
}

std::string invertSlashes(string str){
    transform(str.begin(), str.end(), str.begin(), invert);
    return str;
}

const string & Path::path() const {
    return mypath;
}
        
const string Path::getExtension() const {
    size_t dot = mypath.rfind('.');
    if (dot == string::npos){
        return "";
    } else {
        return mypath.substr(dot + 1);
    }
}

bool Path::isEmpty() const {
    return mypath.empty();
}
        
Path::~Path(){
}

Path::Path(){
}

Path::Path(const std::string & path):
mypath(sanitize(invertSlashes(path))){
}

Path::Path(const Path & path):
mypath(sanitize(invertSlashes(path.path()))){
}

RelativePath::RelativePath(){
}

RelativePath::RelativePath(const std::string & path):
Path(path){
    if (! path.empty() && path[0] == '/'){
        ostringstream out;
        out << "Relative path '" << path << "' cannot start with a /. Only absolute paths can start with /";
        throw Storage::IllegalPath(__FILE__, __LINE__, out.str());
    }
}

/* a/b/c/d -> b/c/d */
std::string stripFirstDir(const std::string & str){
    if (str.find("/") != std::string::npos || str.find( "\\") != std::string::npos){
        std::string temp = str;
        size_t rem = temp.find("/");
        if (rem != std::string::npos){
            return str.substr(rem+1,str.size());
        }
        rem = temp.find("\\");
        if( rem != std::string::npos ){
            return str.substr(rem+1,str.size());
        }
    }
    return str; 
}

static vector<string> splitPath(string path){
    vector<string> all;
    if (path.size() > 0 && path[0] == '/'){
        all.push_back("/");
    }
    size_t found = path.find('/');
    while (found != string::npos){
        if (found > 0){
            all.push_back(path.substr(0, found));
        }
        path.erase(0, found + 1);
        found = path.find('/');
    }   
    if (path.size() != 0){
        all.push_back(path);
    }   
    return all;
}

static string joinPath(const vector<string> & paths){
    ostringstream out;
    bool first = true;
    for (vector<string>::const_iterator it = paths.begin(); it != paths.end(); it++){
        if (!first){
            out << '/';
        }
        out << *it;
        first = false;
    }
    return out.str();
}

/* a/b/c/d -> a/b/c
 * a/b/c/d/ -> a/b/c
 */
static string dirname(string path){
    vector<string> paths = splitPath(path);
    if (paths.size() > 1){
        paths.pop_back();
        return joinPath(paths);
    } else if (paths.size() == 1){
        if (paths[0] == "/"){
            return "/";
        }
        return ".";
    } else {
        return ".";
    }
    /*
    while (path.size() > 0 && path[path.size() - 1] == '/'){
        path.erase(path.size() - 1);
    }

    if (path.find("/") != string::npos ||
        path.find("\\") != string::npos){
        size_t rem = path.find_last_of("/");
        if (rem != string::npos){
            return path.substr(0, rem + 1);
        }

        rem = path.find_last_of("\\");
        if (rem != string::npos){
            return path.substr(0, rem + 1);
        }
    }

    return "";
    */
}

RelativePath::RelativePath(const RelativePath & path):
Path(path){
}
        
RelativePath RelativePath::removeFirstDirectory() const {
    return RelativePath(stripFirstDir(path()));
}
        
bool RelativePath::isFile() const {
    vector<string> paths = splitPath(path());
    return paths.size() == 1;
}
        
RelativePath RelativePath::firstDirectory() const {
    vector<string> paths = splitPath(path());
    if (paths.size() > 1){
        return RelativePath(paths[0]);
    }
    return RelativePath();
}

RelativePath RelativePath::getDirectory() const {
    return RelativePath(dirname(path()));
}

RelativePath RelativePath::getFilename() const {
    return RelativePath(stripDir(path()));
}

bool RelativePath::operator<(const RelativePath & path) const {
    return this->path() < path.path();
}
        
bool RelativePath::operator==(const RelativePath & path) const {
    return this->path() == path.path();
}
        
bool RelativePath::operator!=(const RelativePath & path) const {
    return !(*this == path);
}

RelativePath RelativePath::join(const RelativePath & him) const {
    return RelativePath(path() + "/" + him.path());
}

RelativePath & RelativePath::operator=(const RelativePath & copy){
    setPath(copy.path());
    return *this;
}

AbsolutePath::AbsolutePath(){
}

AbsolutePath::AbsolutePath(const std::string & path):
Path(path){
}

AbsolutePath::AbsolutePath(const AbsolutePath & path):
Path(path){
}

AbsolutePath & AbsolutePath::operator=(const AbsolutePath & copy){
    setPath(copy.path());
    return *this;
}

bool AbsolutePath::operator==(const AbsolutePath & path) const {
    return this->path() == path.path();
}
        
bool AbsolutePath::operator!=(const AbsolutePath & path) const {
    return !(*this == path);
}
        
bool AbsolutePath::operator<(const AbsolutePath & path) const {
    return this->path() < path.path();
}
        
RelativePath AbsolutePath::remove(const AbsolutePath & what) const {
    string real = path();
    real.erase(0, what.path().size());
    while (real.find("/") == 0){
        real.erase(0, 1);
    }
    return RelativePath(real);
}
        
AbsolutePath AbsolutePath::getDirectory() const {
    return AbsolutePath(dirname(path()));
}

AbsolutePath AbsolutePath::getFilename() const {
    return AbsolutePath(stripDir(path()));
}
        
AbsolutePath AbsolutePath::join(const RelativePath & path) const {
    return AbsolutePath(this->path() + "/" + path.path());
}
        
InsensitivePath::InsensitivePath(const Path & what):
Path(what){
}

bool InsensitivePath::operator==(const Path & path) const {
    return Util::upperCaseAll(this->path()) == Util::upperCaseAll(path.path());
}

std::string removeExtension(const std::string & str){
    if (str.find(".") != std::string::npos){
        return str.substr(0, str.find_last_of("."));
    }
    return str;
}

/* a/b/c/d -> d */
std::string stripDir(const std::string & str){
    if (str.find("/") != std::string::npos || str.find("\\") != std::string::npos){
        std::string temp = str;
        size_t rem = temp.find_last_of( "/" );
        if (rem != std::string::npos){
            return str.substr(rem+1,str.size());
        }
        rem = temp.find_last_of( "\\" );
        if( rem != std::string::npos ){
            return str.substr(rem+1,str.size());
        }
    }
    return str; 
}

/* a/b/c/d -> a/b/c/ */
std::string stripFilename(const std::string & str){
    std::string temp = str;
    if( str.find( "/") != std::string::npos || str.find( "\\") != std::string::npos ){
        size_t rem = temp.find_last_of( "/" );
        if( rem != std::string::npos ){
            return str.substr(0,rem+1);
        }
        rem = temp.find_last_of( "\\" );
        if( rem != std::string::npos ){
            return str.substr(0,rem+1);
        }
    }
    return "";
}

}

namespace Storage{
        
Exception::Exception(const std::string & where, int line, const std::string & file):
Exc::Base(where, line),
reason(file){
}

Exception::Exception(const std::string & where, int line, const Exc::Base & nested, const std::string & file):
Exc::Base(where, line, nested),
reason(file){
}
        
Exception::Exception(const Exception & copy):
Exc::Base(copy),
reason(copy.reason){
}

Exception::~Exception() throw (){
}
        
const std::string Exception::getReason() const {
    return reason;
}

NotFound::NotFound(const std::string & where, int line, const std::string & file):
Exception(where, line, file){
}

NotFound::NotFound(const std::string & where, int line, const Exc::Base & nested, const std::string & file):
Exception(where, line, nested, file){
}
        
NotFound::NotFound(const NotFound & copy):
Exception(copy){
}

NotFound::~NotFound() throw (){
}

IllegalPath::IllegalPath(const std::string & where, int line, const std::string & file):
Exception(where, line, file){
}

IllegalPath::IllegalPath(const std::string & where, int line, const Exc::Base & nested, const std::string & file):
Exception(where, line, nested, file){
}
        
IllegalPath::IllegalPath(const IllegalPath & copy):
Exception(copy){
}

IllegalPath::~IllegalPath() throw(){
}

System::System(){
}

System::~System(){
}

vector<Filesystem::AbsolutePath> System::getFiles(const Filesystem::AbsolutePath & dataPath, const Filesystem::RelativePath & find, bool caseInsensitive){
    if (find.isFile()){
        return getFiles(dataPath, find.path(), caseInsensitive);
    }

    /* split the path into its consituent parts
     * a/b/c -> a/b and c
     * search for a/b, then search for c in the results
     */
    Filesystem::RelativePath directory = find.getDirectory();
    Filesystem::RelativePath file = find.getFilename();

    vector<Filesystem::AbsolutePath> more = getFiles(dataPath, directory, caseInsensitive);
    vector<Filesystem::AbsolutePath> out;
    for (vector<Filesystem::AbsolutePath>::iterator it = more.begin(); it != more.end(); it++){
        Filesystem::AbsolutePath path = *it;
        /* if its not a directory then we can't keep searching */
        if (::System::isDirectory(path.path())){
            vector<Filesystem::AbsolutePath> findMore = getFiles(path, file, caseInsensitive);
            out.insert(out.end(), findMore.begin(), findMore.end());
        }
    }
    return out;
}

Util::ReferenceCount<System> self;
System & instance(){
    if (self != NULL){
        return *self;
    }
    self = new Filesystem(Util::getDataPath2());
    return *self;
}
        
File::File(const Path::AbsolutePath & path, Access mode):
path(path){
}
        
File::~File(){
}
        
int File::readLine(char * output, int size){
    return 0;
}

/* overlays:
 * add x/y.zip
 * y.zip contains
 *   example.txt
 * try to read x/example.txt, goto y.zip
 * or y.zip could contain y/
 * 
 * z.zip contains
 *   example.txt
 *
 * how can you prevent two zip files from providing the same files?
 */
class ZipContainer{
public:
    ZipContainer(const string & path, const Filesystem::AbsolutePath & start):
    path(path),
    start(start){
        zipFile = unzOpen(path.c_str());
        if (zipFile == NULL){
            throw Exception(__FILE__, __LINE__, "Could not open zip file");
        }

        if (unzGoToFirstFile(zipFile) != UNZ_OK){
            throw Exception(__FILE__, __LINE__, "Could not get to first file");
        }
<commit_msg>implement normal i/o class on top of the file abstraction<commit_after>#ifdef USE_ALLEGRO
#include <allegro.h>

/* FIXME: replace with <winalleg.h> */
#ifdef _WIN32
#define BITMAP dummyBITMAP
#include <windows.h>
#undef BITMAP
#endif

#endif
#include <algorithm>
#include "funcs.h"
#include "file-system.h"
#include "thread.h"
#include "system.h"
#include "globals.h"
#include <dirent.h>
#include <sstream>
#include <exception>
#include <string>
#include <fstream>
#include <ostream>

#include "zip/unzip.h"
#include "zip/ioapi.h"

#ifndef USE_ALLEGRO
/* some sfl symbols conflict with allegro */
#include "sfl/sfl.h"
#include "sfl/sfldir.h"
#endif

#ifdef _WIN32
#define _WIN32_IE 0x400
#include <shlobj.h>
#endif

using namespace std;

/* some filesystem access can only be done by one thread at a time. specifically, sfl
 * has its own allocator that is meant to be used in a single-threaded manner.
 * rather than try to add locks to sfl we just wrap all sfl calls with a lock.
 *
 * initialize() must be called to initialize this lock
 */
// Util::Thread::Lock lock;



namespace Path{

/* remove extra path separators (/) */
string sanitize(string path){
    size_t double_slash = path.find("//");
    while (double_slash != string::npos){
        path.erase(double_slash, 1);
        double_slash = path.find("//");
    }
    return path;
}

static int invert(int c){
    if (c == '\\'){
        return '/';
    }
    return c;
}

std::string invertSlashes(string str){
    transform(str.begin(), str.end(), str.begin(), invert);
    return str;
}

const string & Path::path() const {
    return mypath;
}
        
const string Path::getExtension() const {
    size_t dot = mypath.rfind('.');
    if (dot == string::npos){
        return "";
    } else {
        return mypath.substr(dot + 1);
    }
}

bool Path::isEmpty() const {
    return mypath.empty();
}
        
Path::~Path(){
}

Path::Path(){
}

Path::Path(const std::string & path):
mypath(sanitize(invertSlashes(path))){
}

Path::Path(const Path & path):
mypath(sanitize(invertSlashes(path.path()))){
}

RelativePath::RelativePath(){
}

RelativePath::RelativePath(const std::string & path):
Path(path){
    if (! path.empty() && path[0] == '/'){
        ostringstream out;
        out << "Relative path '" << path << "' cannot start with a /. Only absolute paths can start with /";
        throw Storage::IllegalPath(__FILE__, __LINE__, out.str());
    }
}

/* a/b/c/d -> b/c/d */
std::string stripFirstDir(const std::string & str){
    if (str.find("/") != std::string::npos || str.find( "\\") != std::string::npos){
        std::string temp = str;
        size_t rem = temp.find("/");
        if (rem != std::string::npos){
            return str.substr(rem+1,str.size());
        }
        rem = temp.find("\\");
        if( rem != std::string::npos ){
            return str.substr(rem+1,str.size());
        }
    }
    return str; 
}

static vector<string> splitPath(string path){
    vector<string> all;
    if (path.size() > 0 && path[0] == '/'){
        all.push_back("/");
    }
    size_t found = path.find('/');
    while (found != string::npos){
        if (found > 0){
            all.push_back(path.substr(0, found));
        }
        path.erase(0, found + 1);
        found = path.find('/');
    }   
    if (path.size() != 0){
        all.push_back(path);
    }   
    return all;
}

static string joinPath(const vector<string> & paths){
    ostringstream out;
    bool first = true;
    for (vector<string>::const_iterator it = paths.begin(); it != paths.end(); it++){
        if (!first){
            out << '/';
        }
        out << *it;
        first = false;
    }
    return out.str();
}

/* a/b/c/d -> a/b/c
 * a/b/c/d/ -> a/b/c
 */
static string dirname(string path){
    vector<string> paths = splitPath(path);
    if (paths.size() > 1){
        paths.pop_back();
        return joinPath(paths);
    } else if (paths.size() == 1){
        if (paths[0] == "/"){
            return "/";
        }
        return ".";
    } else {
        return ".";
    }
    /*
    while (path.size() > 0 && path[path.size() - 1] == '/'){
        path.erase(path.size() - 1);
    }

    if (path.find("/") != string::npos ||
        path.find("\\") != string::npos){
        size_t rem = path.find_last_of("/");
        if (rem != string::npos){
            return path.substr(0, rem + 1);
        }

        rem = path.find_last_of("\\");
        if (rem != string::npos){
            return path.substr(0, rem + 1);
        }
    }

    return "";
    */
}

RelativePath::RelativePath(const RelativePath & path):
Path(path){
}
        
RelativePath RelativePath::removeFirstDirectory() const {
    return RelativePath(stripFirstDir(path()));
}
        
bool RelativePath::isFile() const {
    vector<string> paths = splitPath(path());
    return paths.size() == 1;
}
        
RelativePath RelativePath::firstDirectory() const {
    vector<string> paths = splitPath(path());
    if (paths.size() > 1){
        return RelativePath(paths[0]);
    }
    return RelativePath();
}

RelativePath RelativePath::getDirectory() const {
    return RelativePath(dirname(path()));
}

RelativePath RelativePath::getFilename() const {
    return RelativePath(stripDir(path()));
}

bool RelativePath::operator<(const RelativePath & path) const {
    return this->path() < path.path();
}
        
bool RelativePath::operator==(const RelativePath & path) const {
    return this->path() == path.path();
}
        
bool RelativePath::operator!=(const RelativePath & path) const {
    return !(*this == path);
}

RelativePath RelativePath::join(const RelativePath & him) const {
    return RelativePath(path() + "/" + him.path());
}

RelativePath & RelativePath::operator=(const RelativePath & copy){
    setPath(copy.path());
    return *this;
}

AbsolutePath::AbsolutePath(){
}

AbsolutePath::AbsolutePath(const std::string & path):
Path(path){
}

AbsolutePath::AbsolutePath(const AbsolutePath & path):
Path(path){
}

AbsolutePath & AbsolutePath::operator=(const AbsolutePath & copy){
    setPath(copy.path());
    return *this;
}

bool AbsolutePath::operator==(const AbsolutePath & path) const {
    return this->path() == path.path();
}
        
bool AbsolutePath::operator!=(const AbsolutePath & path) const {
    return !(*this == path);
}
        
bool AbsolutePath::operator<(const AbsolutePath & path) const {
    return this->path() < path.path();
}
        
RelativePath AbsolutePath::remove(const AbsolutePath & what) const {
    string real = path();
    real.erase(0, what.path().size());
    while (real.find("/") == 0){
        real.erase(0, 1);
    }
    return RelativePath(real);
}
        
AbsolutePath AbsolutePath::getDirectory() const {
    return AbsolutePath(dirname(path()));
}

AbsolutePath AbsolutePath::getFilename() const {
    return AbsolutePath(stripDir(path()));
}
        
AbsolutePath AbsolutePath::join(const RelativePath & path) const {
    return AbsolutePath(this->path() + "/" + path.path());
}
        
InsensitivePath::InsensitivePath(const Path & what):
Path(what){
}

bool InsensitivePath::operator==(const Path & path) const {
    return Util::upperCaseAll(this->path()) == Util::upperCaseAll(path.path());
}

std::string removeExtension(const std::string & str){
    if (str.find(".") != std::string::npos){
        return str.substr(0, str.find_last_of("."));
    }
    return str;
}

/* a/b/c/d -> d */
std::string stripDir(const std::string & str){
    if (str.find("/") != std::string::npos || str.find("\\") != std::string::npos){
        std::string temp = str;
        size_t rem = temp.find_last_of( "/" );
        if (rem != std::string::npos){
            return str.substr(rem+1,str.size());
        }
        rem = temp.find_last_of( "\\" );
        if( rem != std::string::npos ){
            return str.substr(rem+1,str.size());
        }
    }
    return str; 
}

/* a/b/c/d -> a/b/c/ */
std::string stripFilename(const std::string & str){
    std::string temp = str;
    if( str.find( "/") != std::string::npos || str.find( "\\") != std::string::npos ){
        size_t rem = temp.find_last_of( "/" );
        if( rem != std::string::npos ){
            return str.substr(0,rem+1);
        }
        rem = temp.find_last_of( "\\" );
        if( rem != std::string::npos ){
            return str.substr(0,rem+1);
        }
    }
    return "";
}

}

namespace Storage{
        
Exception::Exception(const std::string & where, int line, const std::string & file):
Exc::Base(where, line),
reason(file){
}

Exception::Exception(const std::string & where, int line, const Exc::Base & nested, const std::string & file):
Exc::Base(where, line, nested),
reason(file){
}
        
Exception::Exception(const Exception & copy):
Exc::Base(copy),
reason(copy.reason){
}

Exception::~Exception() throw (){
}
        
const std::string Exception::getReason() const {
    return reason;
}

NotFound::NotFound(const std::string & where, int line, const std::string & file):
Exception(where, line, file){
}

NotFound::NotFound(const std::string & where, int line, const Exc::Base & nested, const std::string & file):
Exception(where, line, nested, file){
}
        
NotFound::NotFound(const NotFound & copy):
Exception(copy){
}

NotFound::~NotFound() throw (){
}

IllegalPath::IllegalPath(const std::string & where, int line, const std::string & file):
Exception(where, line, file){
}

IllegalPath::IllegalPath(const std::string & where, int line, const Exc::Base & nested, const std::string & file):
Exception(where, line, nested, file){
}
        
IllegalPath::IllegalPath(const IllegalPath & copy):
Exception(copy){
}

IllegalPath::~IllegalPath() throw(){
}

System::System(){
}

System::~System(){
}

vector<Filesystem::AbsolutePath> System::getFiles(const Filesystem::AbsolutePath & dataPath, const Filesystem::RelativePath & find, bool caseInsensitive){
    if (find.isFile()){
        return getFiles(dataPath, find.path(), caseInsensitive);
    }

    /* split the path into its consituent parts
     * a/b/c -> a/b and c
     * search for a/b, then search for c in the results
     */
    Filesystem::RelativePath directory = find.getDirectory();
    Filesystem::RelativePath file = find.getFilename();

    vector<Filesystem::AbsolutePath> more = getFiles(dataPath, directory, caseInsensitive);
    vector<Filesystem::AbsolutePath> out;
    for (vector<Filesystem::AbsolutePath>::iterator it = more.begin(); it != more.end(); it++){
        Filesystem::AbsolutePath path = *it;
        /* if its not a directory then we can't keep searching */
        if (::System::isDirectory(path.path())){
            vector<Filesystem::AbsolutePath> findMore = getFiles(path, file, caseInsensitive);
            out.insert(out.end(), findMore.begin(), findMore.end());
        }
    }
    return out;
}

Util::ReferenceCount<System> self;
System & instance(){
    if (self != NULL){
        return *self;
    }
    self = new Filesystem(Util::getDataPath2());
    return *self;
}
        
File::File(const Path::AbsolutePath & path, Access mode):
path(path){
}
        
File::~File(){
}
        
int File::readLine(char * output, int size){
    return 0;
}

class NormalFile: public File {
public:
    NormalFile(const Path::AbsolutePath & path, Access mode = Read):
    File(path, mode){
        ios_base::openmode iosMode = fstream::in;
        switch (mode){
            case Read: iosMode = fstream::in; break;
            case Write: iosMode = fstream::out; break;
            case ReadWrite: iosMode = fstream::in | fstream::out; break;
        }

        in.open(path.path().c_str(), iosMode | fstream::binary);
    }
        
    virtual int readLine(char * output, int size){
        in.read(output, size);
        return in.gcount();
    }

    virtual ~NormalFile(){
        in.close();
    }

protected:
    std::fstream in;
};

/* overlays:
 * add x/y.zip
 * y.zip contains
 *   example.txt
 * try to read x/example.txt, goto y.zip
 * or y.zip could contain y/
 * 
 * z.zip contains
 *   example.txt
 *
 * how can you prevent two zip files from providing the same files?
 */
class ZipContainer{
public:
    ZipContainer(const string & path, const Filesystem::AbsolutePath & start):
    path(path),
    start(start){
        zipFile = unzOpen(path.c_str());
        if (zipFile == NULL){
            throw Exception(__FILE__, __LINE__, "Could not open zip file");
        }

        if (unzGoToFirstFile(zipFile) != UNZ_OK){
            throw Exception(__FILE__, __LINE__, "Could not get to first file");
        }

{'fraction_non_alphanumeric': 0.1664680990920669, 'fraction_numerical': 0.003943962861016392, 'mean_word_length': 3.0009861932938855, 'pattern_counts': {'":': 0, '<': 79, '<?xml version=': 0, '>': 85, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1041152', 'n_tokens_mistral': 7844, 'n_tokens_neox': 7703, 'n_words': 1870}