<commit_before>/*
    AutoCorpus: automatically extracts clean natural language corpora from
    publicly available datasets.

    Tokenizer.cpp: splits input lines into sequences of words separated
                   by exactly one space. Ignores all punctuation by default,
                   but can be easily configured to keep selected characters.

                   Can and by default does ignore all text inside parantheses,
                   in order to make the output easier to use in building NLP
                   language models.

                   

    Copyright (C) 2011 <NAME>


    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <string.h>
#include <ctype.h>
#include <iostream>
#include <string>

#include "PCREMatcher.h"

#define PUNCTUATION ".,!?()&@()[]{}/\\\"'#:;<>^”*=-−—\x93"

using namespace std;

class Tokenizer
{
private:
  const char* keep; // punctuation to include in output  
  bool includeParens;
  char lastChar;
  PCREMatcher* abbreviationMatcher;
  
  
  inline bool isPunctuation(char ch) 
  {
    if(ch == '\'')
      return isWS(lastChar);

    return strchr(PUNCTUATION, ch);
  }

  inline bool isDigit(char ch) 
  {
    return isdigit(ch);
  }

  inline bool isKeep(char ch) 
  {
    return strchr(keep, ch);
  }

  inline bool isWS(char ch)
  {
    return isspace(ch);
  }

  inline void printSpace()
  {
    if(!isWS(lastChar)) {
      fputc(' ', stdout);
      lastChar = ' ';
    }
  }

  inline void printChar(char ch)
  {
    fputc(ch, stdout);
    lastChar = ch;
   }
  
  inline void printString(string& input) {
    for(size_t i = 0; i < input.length(); i++) {
      const char ch = input[i];
      if(ch == ' ')
        printSpace();
      else
        printChar(ch);
    }
  }

public:
  Tokenizer(const char* keep, bool includeParens) {
    this->keep = keep;
    this->includeParens = includeParens;
    this->lastChar = '\0';
    this->abbreviationMatcher = new PCREMatcher("^\\w\\.(\\s*\\w\\.)+(\\s|$)+", 0);
  }<commit_msg>downcase abbreviations in tokenizer<commit_after>/*
    AutoCorpus: automatically extracts clean natural language corpora from
    publicly available datasets.

    Tokenizer.cpp: splits input lines into sequences of words separated
                   by exactly one space. Ignores all punctuation by default,
                   but can be easily configured to keep selected characters.

                   Can and by default does ignore all text inside parantheses,
                   in order to make the output easier to use in building NLP
                   language models.

                   

    Copyright (C) 2011 <NAME>


    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <string.h>
#include <ctype.h>
#include <iostream>
#include <string>

#include "PCREMatcher.h"

#define PUNCTUATION ".,!?()&@()[]{}/\\\"'#:;<>^”*=-−—\x93"

using namespace std;

class Tokenizer
{
private:
  const char* keep; // punctuation to include in output  
  bool includeParens;
  char lastChar;
  PCREMatcher* abbreviationMatcher;  
  
  inline bool isPunctuation(char ch) 
  {
    if(ch == '\'')
      return isWS(lastChar);

    return strchr(PUNCTUATION, ch);
  }

  inline bool isDigit(char ch) 
  {
    return isdigit(ch);
  }

  inline bool isKeep(char ch) 
  {
    return strchr(keep, ch);
  }

  inline bool isWS(char ch)
  {
    return isspace(ch);
  }

  inline void printSpace()
  {
    if(!isWS(lastChar)) {
      fputc(' ', stdout);
      lastChar = ' ';
    }
  }

  inline void printChar(char ch)
  {
    fputc(ch, stdout);
    lastChar = ch;
   }
  
  inline void printString(string& input) {
    for(size_t i = 0; i < input.length(); i++) {
      const char ch = input[i];
      if(ch == ' ')
        printSpace();
      else
        printChar(tolower(ch));
    }
  }

public:
  Tokenizer(const char* keep, bool includeParens) {
    this->keep = keep;
    this->includeParens = includeParens;
    this->lastChar = '\0';
    this->abbreviationMatcher = new PCREMatcher("^\\w\\.(\\s*\\w\\.)+(\\s|$)+", 0);
  }
{'fraction_non_alphanumeric': 0.10246679316888045, 'fraction_numerical': 0.003795066413662239, 'mean_word_length': 2.539959704499664, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5867328', 'n_tokens_mistral': 1649, 'n_tokens_neox': 1569, 'n_words': 563}