<commit_before>#include <iostream>
#include <iomanip>
#include <vector>
#include "../StringUtils.h"

using namespace std;

int main()
{
    // conversion functions
    {
        int i;
        double e;
        string_to_int("123456", i);
        string_to_double("2.718281828459045", e);

        cout << setprecision(16);
        cout << "i = " << i << endl;
        cout << "e = " << e << endl;
        cout << endl;
    }

    // path functions
    {
        string vtkpath = "/path/to/file.vtk:temperature";
        string hdfpath = "/path/to/file.h5:/path/to/dataset";
        string testpath1 = "/path/to/some/file.ext:foo:bar";
        string testpath2 = "/path/to/file.ext:";

        vector<string> paths;
        paths.push_back(vtkpath);
        paths.push_back(hdfpath);
        paths.push_back(testpath1);
        paths.push_back(testpath2);

        string path;
        string location, filename, ext;
        string locdir, locbase;

        vector<string>::iterator i;
        for (i = paths.begin(); i != paths.end(); ++i)
        {
            path = *i;

            parse_dataset_path(path, location, filename, ext);
            cout << "path      = '" << path << "'\n";
            cout << "filename  = '" << filename << "'\n";
            cout << "extension = '" << ext << "'\n";
            cout << "location  = '" << location << "'\n";

            path_dirname(location, locdir);
            path_basename(location, locbase);
            cout << "location dirname  = '" << locdir << "'\n";<commit_msg>Added more pathological cases for parse_dataset_path(). This function still needs to be fixed when one of the two arguments is missing.<commit_after>#include <iostream>
#include <iomanip>
#include <vector>
#include "../StringUtils.h"

using namespace std;

int main()
{
    // conversion functions
    {
        int i;
        double e;
        string_to_int("123456", i);
        string_to_double("2.718281828459045", e);

        cout << setprecision(16);
        cout << "i = " << i << endl;
        cout << "e = " << e << endl;
        cout << endl;
    }

    // path functions
    {
        string vtkpath = "/path/to/file.vtk:temperature";
        string hdfpath = "/path/to/file.h5:/path/to/dataset";
        string testpath0 = ".:/path/to/file.ext";
        string testpath1 = "/path/to/file.ext";
        string testpath2 = "/path/to/file.ext:";
        string testpath3 = "/path/to/file.ext:foo";
        string testpath4 = "/path/to/file.ext:foo:";
        string testpath5 = "/path/to/file.ext:foo:bar";

        vector<string> paths;
        paths.push_back(vtkpath);
        paths.push_back(hdfpath);
        paths.push_back(testpath0);
        paths.push_back(testpath1);
        paths.push_back(testpath2);
        paths.push_back(testpath3);
        paths.push_back(testpath4);
        paths.push_back(testpath5);

        string path;
        string location, filename, ext;
        string locdir, locbase;

        vector<string>::iterator i;
        for (i = paths.begin(); i != paths.end(); ++i)
        {
            path = *i;

            parse_dataset_path(path, location, filename, ext);
            cout << "path      = '" << path << "'\n";
            cout << "location  = '" << location << "'\n";
            cout << "filename  = '" << filename << "'\n";
            cout << "extension = '" << ext << "'\n";

            path_dirname(location, locdir);
            path_basename(location, locbase);
            cout << "location dirname  = '" << locdir << "'\n";
{'fraction_non_alphanumeric': 0.1590909090909091, 'fraction_numerical': 0.01875, 'mean_word_length': 2.0068317677198975, 'pattern_counts': {'":': 0, '<': 105, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4713535', 'n_tokens_mistral': 1134, 'n_tokens_neox': 1086, 'n_words': 241}