<commit_before>/*
This file is part of Groho, a simulator for inter-planetary travel and warfare.
Copyright (c) 2017-2018 by <NAME>. Some rights reserved, see LICENSE

This file contains some common functions used in parsing scenario files
*/
#pragma once

#include <optional>
#include <string>

namespace sim {

struct KeyValue {
    std::string key, value;
};

const std::string wspace = " \t\n\r\f\v";

inline std::string trim_whitespace(std::string line)
{
    line.erase(0, line.find_first_not_of(wspace));
    line.erase(line.find_last_not_of(wspace) + 1);
    return line;
}

inline std::string trim_comments(std::string line)
{
    return line.erase(std::min(line.find_first_of(';'), line.size()));
}

inline std::optional<KeyValue> get_key_value(std::string line)
{
    KeyValue kv;
    size_t   n0 = line.find_first_of('=');

    if (n0 > line.size()) {
        return {};
    }

    kv.key   = trim_whitespace(line.substr(0, n0 - 1));
    kv.value = trim_whitespace(line.substr(n0 + 1));
    return kv;
}
}<commit_msg>Add utility functions for text processing and file modification time<commit_after>/*
This file is part of Groho, a simulator for inter-planetary travel and warfare.
Copyright (c) 2017-2018 by <NAME>. Some rights reserved, see LICENSE

This file contains some common functions used in parsing scenario files
*/
#pragma once

#include <iterator>
#include <optional>
#include <sstream>
#include <string>
#include <vector>

// For file modification time
#include <sys/stat.h>
#include <sys/types.h>
#ifndef WIN32
#include <unistd.h>
#endif

#ifdef WIN32
#define stat _stat
#endif

#include "vector.hpp"

#define LOGURU_WITH_STREAMS 1
#include "loguru.hpp"

namespace sim {

struct KeyValue {
    std::string key, value;
};

const std::string wspace = " \t\n\r\f\v";

inline std::string trim_whitespace(std::string line)
{
    line.erase(0, line.find_first_not_of(wspace));
    line.erase(line.find_last_not_of(wspace) + 1);
    return line;
}

inline std::string trim_comments(std::string line)
{
    return line.erase(std::min(line.find_first_of(';'), line.size()));
}

inline std::optional<KeyValue> get_key_value(std::string line)
{
    KeyValue kv;
    size_t   n0 = line.find_first_of('=');

    if (n0 > line.size()) {
        return {};
    }

    kv.key   = trim_whitespace(line.substr(0, n0 - 1));
    kv.value = trim_whitespace(line.substr(n0 + 1));
    return kv;
}

inline std::vector<std::string> split(std::string line)
{
    std::istringstream iss(line);
    return std::vector<std::string>(
        std::istream_iterator<std::string>(iss),
        std::istream_iterator<std::string>());
}

inline Vector stoV(std::string v)
{
    std::vector<std::string> tokens = split(v);
    if (tokens.size() < 3) {
        LOG_S(ERROR) << "Vector needs three components: Got " << v;
        return {};
    }
    Vector out{ stof(tokens[0]), stof(tokens[1]), stof(tokens[2]) };
    return out;
}

inline time_t file_modification_time(std::string fname)
{
    struct stat result;
    if (stat(fname.c_str(), &result) == 0) {
        return result.st_mtime;
    } else {
        return 0;
    }
}

// Given a list of files, give us the time of latest change among them all
inline time_t file_modification_time(std::vector<std::string> fnames)
{
    time_t fmod_time = 0;
    for (auto& fn : fnames) {
        time_t fn_ft = file_modification_time(fn);
        if (fn_ft > fmod_time) {
            fmod_time = fn_ft;
        }
    }
    return fmod_time;
}
}
{'fraction_non_alphanumeric': 0.13924777490668963, 'fraction_numerical': 0.01320700545506747, 'mean_word_length': 3.682795698924731, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2107553', 'n_tokens_mistral': 1244, 'n_tokens_neox': 1212, 'n_words': 310}