<commit_before>Getting Started with string_theory
==================================

Building string_theory
----------------------

string_theory uses the [CMake](http://cmake.org) build system.  To get started
on a Unix-like system (Linux, Mac, MSYS, etc), you can just build string_theory
from the command line after installing CMake:

~~~bash
$ cd /path/to/string_theory
$ mkdir build && cd build
$ cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local ..
$ make && sudo make install
~~~

On Windows, you can also generate Visual Studio projects, either from the
command line or from the CMake GUI:

~~~bat
> cd C:\Path\to\string_theory
> mkdir build-msvc
> cmake -G "Visual Studio 14 2015" ..
(Open the generated .sln file and build the Release target)
~~~

If you want to build string_theory as a static library, you can also add
`-DST_BUILD_STATIC=ON` to the CMake command line (or set it in the GUI).


Using string_theory in your project
-----------------------------------

If your project is built using CMake, it should be easy to add string_theory:

~~~CMake
# CMakeLists.txt
# ...
find_package(string_theory REQUIRED)
include_directories("${STRING_THEORY_INCLUDE_DIRS}")
# ...
add_executable(myapp ${SOURCES} ${HEADERS})
target_link_libraries(myapp ${STRING_THEORY_LIBRARIES})
~~~

Then be sure to specify or locate the cmake module path when configuring your
project (e.g. `-Dstring_theory_DIR=/usr/local/lib/cmake/string_theory`).

When using other build systems, you can just specify string_theory's include
and library paths for your build system.  For GNU makefiles, this may look
something like:

~~~make
STRING_THEORY_DIR = /usr/local
CPPFLAGS += -I$(STRING_THEORY_DIR)/include
LDFLAGS += -L$(STRING_THEORY_DIR)/lib -lstring_theory
~~~

If you create a useful module for your favorite build system, you can also
contribute the necessary files to string_theory via a Pull Request.


Examples
--------

**Basic ST::string features**

~~~c++
#include <string_theory/string>
#include <string_theory/exceptions>
#include <string_theory/stdio>

int main(int argc, char *argv[])
{
    ST::string greeting = ST_LITERAL("Hello");

    // Literals can also use the _st user-literal operator if your compiler
    // supports user-defined literals
    ST::string name = "world"_st;
    if (argc > 1) {
        try {
            // Alternatively, you can assume UTF-8 and substitute invalid
            // byte sequences with
            // ST::string::from_utf8(argv[1], ST_SIZE_AUTO, ST::substitute_invalid)
            // The above method will also avoid throwing any exceptions.
            name = ST::string::from_utf8(argv[1]);
        } catch (ST::unicode_error) {
            name = ST::string::from_latin_1(argv[1]);
        }
    }
    ST::printf("{}, {}\n", greeting, name);

    return 0;
}
~~~

**String formatting**

~~~c++
#include <string_theory/format>

extern void log_line(const ST::string &line);

ST::string status_line(const ST::string &filename, int file_num, int total_files)
{
    return ST::format("Processing file {>3}: {<40} - {3.1f}%", file_num,
                      filename, double(file_num / total_files));
}

int main(int argc, char *argv[])
{
    for (int i = 1; i < argc; ++i) {
        log_line(status_line(argv[i]));
        // Process file...
    }
    return 0;
}
~~~

**Codecs**

~~~c++
#include <string_theory/codecs>
#include <string_theory/stdio>

int main(int argc, char *argv)
{
    if (argc < 2) {
        ST::printf(stderr, "Usage: {} <mode> input\n", argv[0]);
        fputs("<mode>:\n", stderr);
        fputs("  base64_encode\n", stderr);
        fputs("  base64_decode\n", stderr);
        fputs("  hex_encode\n", stderr);
        fputs("  hex_decode\n", stderr);
        return 1;
    }

    ST::string mode = argv[1];
    ST::string input = argv[2];
    ST::string result;
    if (mode == "base64_encode") {
        result = ST::base64_encode(input.to_utf8());
    } else if (mode == "base64_decode") {
        ST::char_buffer data = ST::base64_decode(input);
        result = ST::string::from_utf8(data);
    } else if (mode == "hex_encode") {
        result = ST::hex_encode(input.to_utf8());
    } else if (mode == "hex_decode") {
        ST::char_buffer data = ST::hex_decode(input);
        result = ST::string::from_utf8()
    } else {
        fputs("ERROR: Unsupported mode.\n", stderr);
        return 2;
    }
    puts(result.c_str());

    return 0;
}
~~~
<commit_msg>Fix format example a bit<commit_after>Getting Started with string_theory
==================================

Building string_theory
----------------------

string_theory uses the [CMake](http://cmake.org) build system.  To get started
on a Unix-like system (Linux, Mac, MSYS, etc), you can just build string_theory
from the command line after installing CMake:

~~~bash
$ cd /path/to/string_theory
$ mkdir build && cd build
$ cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local ..
$ make && sudo make install
~~~

On Windows, you can also generate Visual Studio projects, either from the
command line or from the CMake GUI:

~~~bat
> cd C:\Path\to\string_theory
> mkdir build-msvc
> cmake -G "Visual Studio 14 2015" ..
(Open the generated .sln file and build the Release target)
~~~

If you want to build string_theory as a static library, you can also add
`-DST_BUILD_STATIC=ON` to the CMake command line (or set it in the GUI).


Using string_theory in your project
-----------------------------------

If your project is built using CMake, it should be easy to add string_theory:

~~~CMake
# CMakeLists.txt
# ...
find_package(string_theory REQUIRED)
include_directories("${STRING_THEORY_INCLUDE_DIRS}")
# ...
add_executable(myapp ${SOURCES} ${HEADERS})
target_link_libraries(myapp ${STRING_THEORY_LIBRARIES})
~~~

Then be sure to specify or locate the cmake module path when configuring your
project (e.g. `-Dstring_theory_DIR=/usr/local/lib/cmake/string_theory`).

When using other build systems, you can just specify string_theory's include
and library paths for your build system.  For GNU makefiles, this may look
something like:

~~~make
STRING_THEORY_DIR = /usr/local
CPPFLAGS += -I$(STRING_THEORY_DIR)/include
LDFLAGS += -L$(STRING_THEORY_DIR)/lib -lstring_theory
~~~

If you create a useful module for your favorite build system, you can also
contribute the necessary files to string_theory via a Pull Request.


Examples
--------

**Basic ST::string features**

~~~c++
#include <string_theory/string>
#include <string_theory/exceptions>
#include <string_theory/stdio>

int main(int argc, char *argv[])
{
    ST::string greeting = ST_LITERAL("Hello");

    // Literals can also use the _st user-literal operator if your compiler
    // supports user-defined literals
    ST::string name = "world"_st;
    if (argc > 1) {
        try {
            // Alternatively, you can assume UTF-8 and substitute invalid
            // byte sequences with
            // ST::string::from_utf8(argv[1], ST_SIZE_AUTO, ST::substitute_invalid)
            // The above method will also avoid throwing any exceptions.
            name = ST::string::from_utf8(argv[1]);
        } catch (ST::unicode_error) {
            name = ST::string::from_latin_1(argv[1]);
        }
    }
    ST::printf("{}, {}\n", greeting, name);

    return 0;
}
~~~

**String formatting**

~~~c++
#include <string_theory/format>

extern void log_line(const ST::string &line);

ST::string status_line(const ST::string &filename, int file_num, int total_files)
{
    // "Processing file 004: Foobar.txt          - 12.5%"
    return ST::format("Processing file {>03}: {<20} - {4.1f}%", file_num,
                      filename, double(file_num) / double(total_files));
}

int main(int argc, char *argv[])
{
    for (int i = 1; i < argc; ++i) {
        log_line(status_line(argv[i]));
        // Process file...
    }
    return 0;
}
~~~

**Codecs**

~~~c++
#include <string_theory/codecs>
#include <string_theory/stdio>

int main(int argc, char *argv)
{
    if (argc < 2) {
        ST::printf(stderr, "Usage: {} <mode> input\n", argv[0]);
        fputs("<mode>:\n", stderr);
        fputs("  base64_encode\n", stderr);
        fputs("  base64_decode\n", stderr);
        fputs("  hex_encode\n", stderr);
        fputs("  hex_decode\n", stderr);
        return 1;
    }

    ST::string mode = argv[1];
    ST::string input = argv[2];
    ST::string result;
    if (mode == "base64_encode") {
        result = ST::base64_encode(input.to_utf8());
    } else if (mode == "base64_decode") {
        ST::char_buffer data = ST::base64_decode(input);
        result = ST::string::from_utf8(data);
    } else if (mode == "hex_encode") {
        result = ST::hex_encode(input.to_utf8());
    } else if (mode == "hex_decode") {
        ST::char_buffer data = ST::hex_decode(input);
        result = ST::string::from_utf8()
    } else {
        fputs("ERROR: Unsupported mode.\n", stderr);
        return 2;
    }
    puts(result.c_str());

    return 0;
}
~~~

{'fraction_non_alphanumeric': 0.1523820124666073, 'fraction_numerical': 0.010796972395369546, 'mean_word_length': 3.1946778711484596, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7020682', 'n_tokens_mistral': 3030, 'n_tokens_neox': 2830, 'n_words': 851}