<commit_before>//
// mysociety_file.h:
// Various file related functions.
//
// Copyright (c) 2010 UK Citizens Online Democracy. All rights reserved.
// Email: <EMAIL>; WWW: http://www.mysociety.org/
//
// $Id: mysociety_error.h,v 1.4 2009-09-24 22:00:29 francis Exp $
//

#include <sys/stat.h>

// Test if a file (or directory) exists
bool file_exists(const std::string& file_name) {
    struct stat file_info;
    if (stat(file_name.c_str(),&file_info) < 0) {
        return false;
    }
    return true;
}

/////////////////////////////////////////////////////////////////////
// Memory mapped files

// Wrapper for mmap.
class MemoryMappedFile {
    public:

    MemoryMappedFile() {
        this->f_h = -1;
        this->ptr = (void*)MAP_FAILED;
    }
    
    // this is done in a separate function, rather than the constructor, so
    // it does not always have to be called (e.g. when MemoryMappedFile is used
    // as a member of another class, and is optional)
    void map_file(const std::string& f_name, unsigned int f_size, bool f_write = false) {
        this->unmap_file();

        this->f_write = f_write;
        assert(this->ptr == MAP_FAILED);

        if (!this->f_write) {
            // check the file is present and the right size
            if (stat(f_name.c_str(), &this->f_stat) < 0) {
                throw Exception((boost::format("map_file: failed to stat file %s: %s") % f_name.c_str() % strerror(errno)).str());
            }
            assert((unsigned int)this->f_stat.st_size == f_size);
        }

        // map it into RAM
        this->f_h = open(f_name.c_str(), this->f_write ? (O_RDWR | O_CREAT) : O_RDONLY, 0644);
        if (this->f_h == -1) {
            throw Exception((boost::format("map_file: failed to fopen file %s: %s") % f_name % strerror(errno)).str());
        }
        if (this->f_write) {
            if (lseek(this->f_h, f_size - 1, SEEK_SET) == -1) {
                throw Exception((boost::format("map_file: failed to lseek file: %s") % strerror(errno)).str());
            }
            if (write(this->f_h, "", 1) == -1) {
                throw Exception((boost::format("map_file: failed to write byte at end of file: %s") % strerror(errno)).str());
            }
        }
        this->ptr = mmap(NULL, f_size, this->f_write ? (PROT_READ | PROT_WRITE) : PROT_READ, MAP_SHARED, this->f_h, 0);
        if (this->ptr == MAP_FAILED) {
            throw Exception((boost::format("map_file: failed to mmap file: %s") % strerror(errno)).str());
        }
    }

    void unmap_file() {
        if (this->ptr != MAP_FAILED) {
            munmap(this->ptr, this->f_stat.st_size);
            this->ptr = MAP_FAILED;
        }
        if (this->f_h != -1) {
            close(this->f_h);
            this->f_h = -1;
        }
    }

    ~MemoryMappedFile() {
        this->unmap_file();
    }
 
    public:
    void *ptr;
    private:
    struct stat f_stat;
    int f_h;
    bool f_write;
};



<commit_msg>msync file, error check munmap, store size as not in stat on write<commit_after>//
// mysociety_file.h:
// Various file related functions.
//
// Copyright (c) 2010 UK Citizens Online Democracy. All rights reserved.
// Email: <EMAIL>; WWW: http://www.mysociety.org/
//
// $Id: mysociety_error.h,v 1.4 2009-09-24 22:00:29 francis Exp $
//

#include <sys/stat.h>

// Test if a file (or directory) exists
bool file_exists(const std::string& file_name) {
    struct stat file_info;
    if (stat(file_name.c_str(),&file_info) < 0) {
        return false;
    }
    return true;
}

/////////////////////////////////////////////////////////////////////
// Memory mapped files

// Wrapper for mmap.
class MemoryMappedFile {
    public:

    MemoryMappedFile() {
        this->f_h = -1;
        this->ptr = (void*)MAP_FAILED;
    }
    
    // this is done in a separate function, rather than the constructor, so
    // it does not always have to be called (e.g. when MemoryMappedFile is used
    // as a member of another class, and is optional)
    void map_file(const std::string& f_name, unsigned int f_size, bool f_write = false) {
        this->unmap_file();

        this->f_write = f_write;
        assert(this->ptr == MAP_FAILED);

        this->f_size = f_size;

        if (!this->f_write) {
            // check the file is present and the right size
            if (stat(f_name.c_str(), &this->f_stat) < 0) {
                throw Exception((boost::format("map_file: failed to stat file %s: %s") % f_name.c_str() % strerror(errno)).str());
            }
            assert((unsigned int)this->f_stat.st_size == f_size);
        }

        // map it into RAM
        this->f_h = open(f_name.c_str(), this->f_write ? (O_RDWR | O_CREAT) : O_RDONLY, 0644);
        if (this->f_h == -1) {
            throw Exception((boost::format("map_file: failed to fopen file %s: %s") % f_name % strerror(errno)).str());
        }
        if (this->f_write) {
            if (lseek(this->f_h, f_size - 1, SEEK_SET) == -1) {
                throw Exception((boost::format("map_file: failed to lseek file: %s") % strerror(errno)).str());
            }
            if (write(this->f_h, "", 1) == -1) {
                throw Exception((boost::format("map_file: failed to write byte at end of file: %s") % strerror(errno)).str());
            }
        }
        this->ptr = mmap(NULL, f_size, this->f_write ? (PROT_READ | PROT_WRITE) : PROT_READ, MAP_SHARED, this->f_h, 0);
        if (this->ptr == MAP_FAILED) {
            throw Exception((boost::format("map_file: failed to mmap file: %s") % strerror(errno)).str());
        }
    }

    void unmap_file() {
        if (this->ptr != MAP_FAILED) {
            int ret1 = msync(this->ptr, this->f_size, MS_SYNC);
            if (ret1 == -1) {
                throw Exception((boost::format("map_file: failed to msync file: %s") % strerror(errno)).str());
            }
            int ret2 = munmap(this->ptr, this->f_size);
            if (ret2 == -1) {
                throw Exception((boost::format("map_file: failed to munmap file: %s") % strerror(errno)).str());
            }
            this->ptr = MAP_FAILED;
        }
        if (this->f_h != -1) {
            close(this->f_h);
            this->f_h = -1;
        }
    }

    ~MemoryMappedFile() {
        this->unmap_file();
    }
 
    public:
    void *ptr;
    private:
    unsigned int f_size;
    struct stat f_stat;
    int f_h;
    bool f_write;
};




{'fraction_non_alphanumeric': 0.16869158878504673, 'fraction_numerical': 0.011838006230529595, 'mean_word_length': 1.9768196569309227, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 62, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4703176', 'n_tokens_mistral': 2197, 'n_tokens_neox': 2103, 'n_words': 538}