<commit_before>// Copyright (c) 2014-2017 Dr. <NAME> and <NAME>
// Please see LICENSE for license or visit https://github.com/taocpp/PEGTL/

#ifndef TAOCPP_PEGTL_INCLUDE_INTERNAL_DEMANGLE_CXXABI_HPP
#define TAOCPP_PEGTL_INCLUDE_INTERNAL_DEMANGLE_CXXABI_HPP

#include <cstdlib>
#include <cxxabi.h>
#include <memory>
#include <string>

#include "../config.hpp"

namespace tao
{
   namespace TAOCPP_PEGTL_NAMESPACE
   {
      namespace internal
      {
         inline std::string demangle( const char* symbol )
         {
            const std::unique_ptr< char, decltype( &std::free ) > demangled( abi::__cxa_demangle( symbol, nullptr, nullptr, nullptr ), &std::free );
            return demangled ? demangled.get() : symbol;
         }

      }  // namespace internal

   }  // namespace TAOCPP_PEGTL_NAMESPACE

}  // namespace tao

#endif
<commit_msg>Sanitise chars.<commit_after>// Copyright (c) 2014-2017 Dr. <NAME> and <NAME>
// Please see LICENSE for license or visit https://github.com/taocpp/PEGTL/

#ifndef TAOCPP_PEGTL_INCLUDE_INTERNAL_DEMANGLE_CXXABI_HPP
#define TAOCPP_PEGTL_INCLUDE_INTERNAL_DEMANGLE_CXXABI_HPP

#include <cstdlib>
#include <cxxabi.h>
#include <memory>
#include <string>

#include "../config.hpp"

namespace tao
{
   namespace TAOCPP_PEGTL_NAMESPACE
   {
      namespace internal
      {
         inline void demangle_sanitise_chars( std::string & s )
         {
            std::string::size_type p;
            while( ( p = s.find( "(char)" ) ) != std::string::npos ) {
               int c = 0;
               std::string::size_type q;
               for( q = p + 6; ( q < s.size() ) && ( s[ q ] >= '0' ) && ( s[ q ] <= '9' ); ++q ) {
                  c *= 10;
                  c += s[ q ] - '0';
               }
               if( c == '\'' ) {
                  s.replace( p, q - p, "'\''" );
               }
               else if( c == '\\' ) {
                  s.replace( p, q - p, "'\\'" );
               }
               else if( ( c < 32 ) || ( c > 126 ) ) {
                  s.replace( p, p + 6, "" );
               }
               else {
                  s.replace( p, q - p, std::string( 1, '\'' ) + char( c ) + '\'' );
               }
            }
         }

         inline std::string demangle( const char* symbol )
         {
            const std::unique_ptr< char, decltype( &std::free ) > demangled( abi::__cxa_demangle( symbol, nullptr, nullptr, nullptr ), &std::free );
            if( !demangled ) {
               return symbol;
            }
            std::string result( demangled.get() );
            demangle_sanitise_chars( result );
            return result;
         }

      }  // namespace internal

   }  // namespace TAOCPP_PEGTL_NAMESPACE

}  // namespace tao

#endif

{'fraction_non_alphanumeric': 0.13818181818181818, 'fraction_numerical': 0.01090909090909091, 'mean_word_length': 1.8014256619144602, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 19, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6807198', 'n_tokens_mistral': 943, 'n_tokens_neox': 895, 'n_words': 206}