<commit_before>#include "parser.h"

/*
* Debugging function
*/




std::list<std::string> keywords = {"norrk\x94ping","norrkoping",
"spongebob","britney","spears", "paris", "hilton","ipad", "kunskap"};
void printString(std::string s){

  for(char& c : s) {
    printchar((unsigned char)c);
  }
}

void printchar(unsigned char theChar){

  switch (theChar) {

    case '\n':
    printf("\\n\n");
    break;
    case '\r':
    printf("\\r");
    break;
    case '\t':
    printf("\\t");
    break;
    default:
    if ((theChar < 0x20) || (theChar > 0x7f)) {
      printf("\\%03o", (unsigned char)theChar);
    } else {
      printf("%c", theChar);
    }
    break;
  }
}


<commit_msg>Added thanks to relevant sources  # Please enter the commit message for your changes. Lines starting<commit_after>#include "parser.h"

/*
* Debugging function
*/




std::list<std::string> keywords = {"norrk\x94ping","norrkoping",
"spongebob","britney","spears", "paris", "hilton","ipad", "kunskap"};
void printString(std::string s){

  for(char& c : s) {
    printchar((unsigned char)c);
  }
}

//
// Thanks to http://stackoverflow.com/questions/1079748/how-to-print-n-instead-of-a-newline
//

void printchar(unsigned char theChar){

  switch (theChar) {

    case '\n':
    printf("\\n\n");
    break;
    case '\r':
    printf("\\r");
    break;
    case '\t':
    printf("\\t");
    break;
    default:
    if ((theChar < 0x20) || (theChar > 0x7f)) {
      printf("\\%03o", (unsigned char)theChar);
    } else {
      printf("%c", theChar);
    }
    break;
  }
}



{'fraction_non_alphanumeric': 0.20077220077220076, 'fraction_numerical': 0.016087516087516088, 'mean_word_length': 2.946700507614213, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6136506', 'n_tokens_mistral': 607, 'n_tokens_neox': 566, 'n_words': 117}