<commit_before>#include "utils.h"

#include <ctype.h> /* isprint(), isdigit(), tolower(), isalnum() */
#include <string.h> /* strlen() */

char *tu_strtok(char **str, unsigned int *match_length, const char *delimiters) {

  char *start_ptr;
  unsigned int cur_delimiter, delimiters_length, delimiter_found;

  *match_length = 0;
  if ( str == NULL || *str == NULL )
    return NULL;

  delimiters_length = strlen(delimiters);
  if ( delimiters_length == 0 )
    return NULL;

  start_ptr = *str;

  while ( **str ) {

    delimiter_found = 0;
    for ( cur_delimiter = 0; cur_delimiter < delimiters_length; cur_delimiter++)
      if ( **str == *(delimiters + cur_delimiter) ) {
        delimiter_found = 1;
        break;
      }
    (*str)++;

    if ( delimiter_found == 1 )
      break;

    (*match_length)++;
  }

  if ( delimiter_found == 0 )
    *match_length = 0;

  return start_ptr;
}

char *tu_strbtok(char **str, unsigned int *match_length, const char *delimiter) {

  char *start_ptr;
  unsigned int cur_delimiter, delimiter_length;

  *match_length = 0;
  if ( str == NULL || *str == NULL )<commit_msg>utils: delimiter_found could not be anything else at this point<commit_after>#include "utils.h"

#include <ctype.h> /* isprint(), isdigit(), tolower(), isalnum() */
#include <string.h> /* strlen() */

char *tu_strtok(char **str, unsigned int *match_length, const char *delimiters) {

  char *start_ptr;
  unsigned int cur_delimiter, delimiters_length, delimiter_found;

  *match_length = 0;
  if ( str == NULL || *str == NULL )
    return NULL;

  delimiters_length = strlen(delimiters);
  if ( delimiters_length == 0 )
    return NULL;

  start_ptr = *str;
  delimiter_found = 0;
  while ( **str ) {

    for ( cur_delimiter = 0; cur_delimiter < delimiters_length; cur_delimiter++)
      if ( **str == *(delimiters + cur_delimiter) ) {
        delimiter_found = 1;
        break;
      }
    (*str)++;

    if ( delimiter_found == 1 )
      break;

    (*match_length)++;
  }

  if ( delimiter_found == 0 )
    *match_length = 0;

  return start_ptr;
}

char *tu_strbtok(char **str, unsigned int *match_length, const char *delimiter) {

  char *start_ptr;
  unsigned int cur_delimiter, delimiter_length;

  *match_length = 0;
  if ( str == NULL || *str == NULL )
{'fraction_non_alphanumeric': 0.13997367266344887, 'fraction_numerical': 0.007898200965335674, 'mean_word_length': 3.3263757115749524, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5341038', 'n_tokens_mistral': 831, 'n_tokens_neox': 785, 'n_words': 203}