<commit_before>// STL
#include <iostream>
// Boost String
#include <boost/locale.hpp>

// ////////////// M A I N //////////////
int main (int argc, char* argv[]) {

  // Get the global localisation backend
  boost::locale::localization_backend_manager lLocBEMgr =
    boost::locale::localization_backend_manager::global(); 

  // Select ICU backend as default
  lLocBEMgr.select ("icu");

  // Set this backend globally
  boost::locale::localization_backend_manager::global (lLocBEMgr);

  // Create a generator that uses this backend.
  boost::locale::generator locGen (lLocBEMgr);

  // Create locale generator with the system default locale
  std::locale::global (locGen ("")); 

  // With STL locale feature
  std::string mucDE ("München");
  std::string mucRU ("Мюнхен");
  std::string pacaFR ("Provence Alpes Côtes d'Azur");
  std::string pvgZH ("上海浦东国际机场");

  // Test the Boost Locale string conversions
  std::cout << "Original: " << mucDE << std::endl
            <<"Upper " << boost::locale::to_upper (mucDE) << std::endl
            <<"Lower " << boost::locale::to_lower (mucDE) << std::endl
            <<"Title " << boost::locale::to_title (mucDE) << std::endl
            <<"Fold  " << boost::locale::fold_case (mucDE) << std::endl
            << "Normalise: "
            << boost::locale::normalize (mucDE, boost::locale::norm_nfkc)
            << std::endl;

  std::cout << "Original: " << mucRU << std::endl
            <<"Upper " << boost::locale::to_upper (mucRU) << std::endl
            <<"Lower " << boost::locale::to_lower (mucRU) << std::endl
            <<"Title " << boost::locale::to_title (mucRU) << std::endl
            <<"Fold  " << boost::locale::fold_case (mucRU) << std::endl
            << "Normalise: "
            << boost::locale::normalize (mucRU, boost::locale::norm_nfkc)
            << std::endl;

  std::cout << "Original: " << pacaFR << std::endl
            <<"Upper " << boost::locale::to_upper (pacaFR) << std::endl
            <<"Lower " << boost::locale::to_lower (pacaFR) << std::endl
            <<"Title " << boost::locale::to_title (pacaFR) << std::endl
            <<"Fold  " << boost::locale::fold_case (pacaFR) << std::endl
            << "Normalise: "
            << boost::locale::normalize (pacaFR, boost::locale::norm_nfkc)
            << std::endl;
  
  std::cout << "Original: " << pvgZH << std::endl
            <<"Upper " << boost::locale::to_upper (pvgZH) << std::endl
            <<"Lower " << boost::locale::to_lower (pvgZH) << std::endl
            <<"Title " << boost::locale::to_title (pvgZH) << std::endl
            <<"Fold  " << boost::locale::fold_case (pvgZH) << std::endl
            << "Normalise: "
            << boost::locale::normalize (pvgZH, boost::locale::norm_nfkc)
            << std::endl;
  
  return 0;<commit_msg>[Test] Took a better string example for accentuated words (côté rather than Côte).<commit_after>// STL
#include <iostream>
// Boost Locale
#include <boost/locale.hpp>

// ////////////// M A I N //////////////
int main (int argc, char* argv[]) {

  // Get the global localisation backend
  boost::locale::localization_backend_manager lLocBEMgr =
    boost::locale::localization_backend_manager::global(); 

  // Select ICU backend as default
  lLocBEMgr.select ("icu");

  // Set this backend globally
  boost::locale::localization_backend_manager::global (lLocBEMgr);

  // Create a generator that uses this backend.
  boost::locale::generator locGen (lLocBEMgr);

  // Create locale generator with the system default locale
  std::locale::global (locGen ("")); 

  // A few instances of languages
  std::string mucDE ("München");
  std::string mucRU ("Мюнхен");
  std::string sideFR ("Côté");
  std::string pvgZH ("上海浦东国际机场");

  // Test the Boost Locale string conversions
  std::cout << "Original: " << mucDE << std::endl
            <<"Upper " << boost::locale::to_upper (mucDE) << std::endl
            <<"Lower " << boost::locale::to_lower (mucDE) << std::endl
            <<"Title " << boost::locale::to_title (mucDE) << std::endl
            <<"Fold  " << boost::locale::fold_case (mucDE) << std::endl
            << "Normalise: "
            << boost::locale::normalize (mucDE, boost::locale::norm_nfkc)
            << std::endl;

  std::cout << "Original: " << mucRU << std::endl
            <<"Upper " << boost::locale::to_upper (mucRU) << std::endl
            <<"Lower " << boost::locale::to_lower (mucRU) << std::endl
            <<"Title " << boost::locale::to_title (mucRU) << std::endl
            <<"Fold  " << boost::locale::fold_case (mucRU) << std::endl
            << "Normalise: "
            << boost::locale::normalize (mucRU, boost::locale::norm_nfkc)
            << std::endl;

  std::cout << "Original: " << sideFR << std::endl
            <<"Upper " << boost::locale::to_upper (sideFR) << std::endl
            <<"Lower " << boost::locale::to_lower (sideFR) << std::endl
            <<"Title " << boost::locale::to_title (sideFR) << std::endl
            <<"Fold  " << boost::locale::fold_case (sideFR) << std::endl
            << "Normalised - [NFD]: "
            << boost::locale::normalize (sideFR, boost::locale::norm_nfd)
            << "; [NFC]: "
            << boost::locale::normalize (sideFR, boost::locale::norm_nfc)
            << ";  [NFKD]: "
            << boost::locale::normalize (sideFR, boost::locale::norm_nfkd)
            << "; [NFKC]: "
            << boost::locale::normalize (sideFR, boost::locale::norm_nfkc)
            << std::endl;
  
  std::cout << "Original: " << pvgZH << std::endl
            <<"Upper " << boost::locale::to_upper (pvgZH) << std::endl
            <<"Lower " << boost::locale::to_lower (pvgZH) << std::endl
            <<"Title " << boost::locale::to_title (pvgZH) << std::endl
            <<"Fold  " << boost::locale::fold_case (pvgZH) << std::endl
            << "Normalise: "
            << boost::locale::normalize (pvgZH, boost::locale::norm_nfkc)
            << std::endl;
  
  return 0;
{'fraction_non_alphanumeric': 0.19374268027438515, 'fraction_numerical': 0.00033461602810774636, 'mean_word_length': 2.7787610619469025, 'pattern_counts': {'":': 0, '<': 307, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1849972', 'n_tokens_mistral': 1950, 'n_tokens_neox': 1865, 'n_words': 394}