<commit_before>/* This file is part of the KDE project

   Copyright (C) 1999, 2000 <NAME> <<EMAIL>>
             (C) 1999, 2000 <NAME> <<EMAIL>>

   This library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Library General Public
   License as published by the Free Software Foundation; either
   version 2 of the License, or (at your option) any later version.

   This library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Library General Public License for more details.

   You should have received a copy of the GNU Library General Public License
   along with this library; see the file COPYING.LIB.  If not, write to
   the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
   Boston, MA 02111-1307, USA.
*/

#include <string.h>
#include <stddef.h>
#include <qstring.h>
#include <qstrlist.h>

#include <RMM_Defines.h>
#include <RMM_Enum.h>

namespace RMM {

    Q_UINT32
RTokenise(
    const char * str,
    const char * delim,
    QStrList & l,
    bool skipComments,
    bool quotedTokens)
{
    l.clear();
    
    if (!delim || !str || strlen(delim) == 0 || strlen(str) == 0) return 0;
    
    char * len = const_cast<char *>(str + strlen(str));    // End of string.
    
    int rl = strlen(str);
    char * rstart = new char[rl + 1];
    char * r = rstart;
    
    
    const char * i = str;        // Cursor.
    
    while (i <= len) {

        // I'm ignoring all non-printable chars.
        // Note that 127 (DEL) is ignored here. So what ? RFC822 is fucked up.
        // I'm not going to pander to its self-contradictory crap.
        if ((*i < 32 || *i == 127) && *i != '\t') {
            ++i;
            continue;
        }

        if (*i == '\\') { // Escaped chars go straight through.
            *r++ = *i++;
            if (i <= len)
                *r++ = *i++;
            continue;
        }
        
        if (strchr(delim, *i) != 0) {
            // We hit a delimiter. If we have some text, make a new token.
            // This has the effect that multiple delimiters are collapsed.
            *r = '\0';
            if (r != rstart) {
                l.append(rstart);
            }
            r = rstart;
            ++i;
            continue;
        }
        
        // If we find a quote, we need to continue through until
        // the next quote, ignoring delimiters
        if (*i == '"') {
          
            if (quotedTokens) {

                // If there's anything in the buffer, make a token from<commit_msg>No need to call r = rstart, if we already know that r == rstart.<commit_after>/* This file is part of the KDE project

   Copyright (C) 1999, 2000 <NAME> <<EMAIL>>
             (C) 1999, 2000 <NAME> <<EMAIL>>

   This library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Library General Public
   License as published by the Free Software Foundation; either
   version 2 of the License, or (at your option) any later version.

   This library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Library General Public License for more details.

   You should have received a copy of the GNU Library General Public License
   along with this library; see the file COPYING.LIB.  If not, write to
   the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
   Boston, MA 02111-1307, USA.
*/

#include <string.h>
#include <stddef.h>
#include <qstring.h>
#include <qstrlist.h>

#include <RMM_Defines.h>
#include <RMM_Enum.h>

namespace RMM {

    Q_UINT32
RTokenise(
    const char * str,
    const char * delim,
    QStrList & l,
    bool skipComments,
    bool quotedTokens)
{
    l.clear();
    
    if (!delim || !str || strlen(delim) == 0 || strlen(str) == 0) return 0;
    
    char * len = const_cast<char *>(str + strlen(str));    // End of string.
    
    int rl = strlen(str);
    char * rstart = new char[rl + 1];
    char * r = rstart;
    
    
    const char * i = str;        // Cursor.
    
    while (i <= len) {

        // I'm ignoring all non-printable chars.
        // Note that 127 (DEL) is ignored here. So what ? RFC822 is fucked up.
        // I'm not going to pander to its self-contradictory crap.
        if ((*i < 32 || *i == 127) && *i != '\t') {
            ++i;
            continue;
        }

        if (*i == '\\') { // Escaped chars go straight through.
            *r++ = *i++;
            if (i <= len)
                *r++ = *i++;
            continue;
        }
        
        if (strchr(delim, *i) != 0) {
            // We hit a delimiter. If we have some text, make a new token.
            // This has the effect that multiple delimiters are collapsed.
            *r = '\0';
            if (r != rstart) {
                l.append(rstart);
                r = rstart;
            }   
            ++i;
            continue;
        }
        
        // If we find a quote, we need to continue through until
        // the next quote, ignoring delimiters
        if (*i == '"') {
          
            if (quotedTokens) {

                // If there's anything in the buffer, make a token from
{'fraction_non_alphanumeric': 0.10611111111111111, 'fraction_numerical': 0.018518518518518517, 'mean_word_length': 1.826268969126112, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5646477', 'n_tokens_mistral': 1694, 'n_tokens_neox': 1566, 'n_words': 655}