<commit_before>#ifndef utilities_hpp
#define utilities_hpp

#include "basic_types.hpp"
#include "miniz.h"
#include <oauth/urlencode.h>  // For urlen/decode
#include <json/json.hpp>      // 2016 JSON handling

// 2017/01/05 tired of typing
using namespace std;
using json = nlohmann::json;

//  INDEX
// ~~~~~~~
// STRING HELPERS
//    static bool strings_are_equal(const string& s1, const string& s2, bool b_case_insensitive = true)
//    static bool replace(string& str, const string& from, const string& to)
//    static bool replace_once(string& str, const string& from, const string& to)
//    static bool replace_with_regex(string& str, const string& from, const string& to)
//    static bool replace_once_with_regex(string& str, const string& from, const string& to)
//    static bool b_string_ends_in(const string& source, const string& search)
//    static vector<int64_t> parse_csv_ints(const std::string& csvdata)
//    static string escape_doublequotes(string& string_to_change)
//    static bool escape_doublequotes(string& string_to_change)
//    static bool escape_backslash(string& string_to_change)
//    static bool double_doublequotes(string& string_to_change)
//    static string raw_to_Javascript(const string& input)
//    split any string: vector<string> strs; boost::split(strs,line,boost::is_any_of("\t"));
//   WEB
//    static bool url_decode(const std::string& in, std::string& out)
//    static string urlencode(const string &value) (from oauth code)
//    static std::map<const std::string,std::string> parse_cookies(const std::string& cookiedata)     { return parse_html(cookiedata,"; "); }
//    static std::map<const std::string,std::string> parse_url_params(const std::string& urldata)     { return parse_html(urldata   ,"?&"); }
//    static std::map<const std::string,std::string> parse_form(const std::string& formdata)          { return parse_html(formdata  ,"&" ); }
//    static std::map<const std::string,std::string> parse_html(...)
//    For more powerful curl-like helpers, see: http/HttpsClientUtilities.hpp
//   JSON
//    //      #include <json/json.hpp>                                // 2016 JSON handling
//    //      using json = nlohmann::json;
//   VERSIONING
//    // class SemVer
//   JWT
//    class JWT;
//    string url_encode_JWT(const JWT& j_input);
//    void url_decode_JWT(string input, JWT& jwt);
// TIME
//    static ptime get_current_time()
//    static ptime string_to_ptime(const string& str_time, const string& str_format)
//    static ptime iso_string_to_ptime(const string& str_time)
//    static string ptime_to_string(const ptime& pt, const string& str_format)
//    static time_t ptime_to_time_t(const ptime& pt)
//    static ptime time_t_to_ptime(const time_t& tt)
//    static string time_t_to_string(const time_t& tt, const string& str_format)
//    static time_t get_current_time_t()
//    static time_t get_today_midnight()<commit_msg>Full case insensitive email handling<commit_after>#ifndef utilities_hpp
#define utilities_hpp

#include "basic_types.hpp"
#include "miniz.h"
#include <oauth/urlencode.h>  // For urlen/decode
#include <json/json.hpp>      // 2016 JSON handling

// 2017/01/05 tired of typing
using namespace std;
using json = nlohmann::json;

//  INDEX
// ~~~~~~~
// STRING HELPERS
//    static bool strings_are_equal(const string& s1, const string& s2, bool b_case_insensitive = true)
//    static bool replace(string& str, const string& from, const string& to)
//    static bool replace_once(string& str, const string& from, const string& to)
//    static bool replace_with_regex(string& str, const string& from, const string& to)
//    static bool replace_once_with_regex(string& str, const string& from, const string& to)
//    static bool b_string_ends_in(const string& source, const string& search)
//    static vector<int64_t> parse_csv_ints(const std::string& csvdata)
//    static string escape_doublequotes(string& string_to_change)
//    static bool escape_doublequotes(string& string_to_change)
//    static bool escape_backslash(string& string_to_change)
//    static bool double_doublequotes(string& string_to_change)
//    static string raw_to_Javascript(const string& input)
//    split any string: vector<string> strs; boost::split(strs,line,boost::is_any_of("\t"));
//    make lower case: boost::algorithm::to_lower()
//   WEB
//    static bool url_decode(const std::string& in, std::string& out)
//    static string urlencode(const string &value) (from oauth code)
//    static std::map<const std::string,std::string> parse_cookies(const std::string& cookiedata)     { return parse_html(cookiedata,"; "); }
//    static std::map<const std::string,std::string> parse_url_params(const std::string& urldata)     { return parse_html(urldata   ,"?&"); }
//    static std::map<const std::string,std::string> parse_form(const std::string& formdata)          { return parse_html(formdata  ,"&" ); }
//    static std::map<const std::string,std::string> parse_html(...)
//    For more powerful curl-like helpers, see: http/HttpsClientUtilities.hpp
//   JSON
//    //      #include <json/json.hpp>                                // 2016 JSON handling
//    //      using json = nlohmann::json;
//   VERSIONING
//    // class SemVer
//   JWT
//    class JWT;
//    string url_encode_JWT(const JWT& j_input);
//    void url_decode_JWT(string input, JWT& jwt);
// TIME
//    static ptime get_current_time()
//    static ptime string_to_ptime(const string& str_time, const string& str_format)
//    static ptime iso_string_to_ptime(const string& str_time)
//    static string ptime_to_string(const ptime& pt, const string& str_format)
//    static time_t ptime_to_time_t(const ptime& pt)
//    static ptime time_t_to_ptime(const time_t& tt)
//    static string time_t_to_string(const time_t& tt, const string& str_format)
//    static time_t get_current_time_t()
//    static time_t get_today_midnight()
{'fraction_non_alphanumeric': 0.12688463493139082, 'fraction_numerical': 0.00677621548365238, 'mean_word_length': 3.7805668016194334, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '302259', 'n_tokens_mistral': 1951, 'n_tokens_neox': 1895, 'n_words': 497}