<commit_before>// Copyright 2013 <NAME>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

#include "common.h"
#include "logging.h"

namespace kernel {

bool Tokenize(const char *line, fv *fv, short *y, std::size_t *maxid) {
  if (line[0] == '#') return false;

  *y = static_cast<short>(atoi(line));
  if (*y != 1 && *y != -1) {
    LOG(ERROR) << "Invalid label. A label must be +1 or -1.";
    return false;
  }

  while (!isspace(*line) && *line) line++;
  while (isspace(*line) && *line) line++;
  while (1) {
    const char *begin = line;
    unsigned int index = std::atol(begin);
    while (*line != ':' && *line) line++;
    float val = std::atof(++line);

    while (*line != ' ' && *line) line++;
    fv->push_back(std::make_pair(index, val));
    if (index > *maxid) *maxid = index;
    if (*line++ == 0) break;
  }
  return true;
}
} // namespace kernel
<commit_msg>Remove redundant checkpoint<commit_after>// Copyright 2013 <NAME>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

#include "common.h"
#include "logging.h"

namespace kernel {

bool Tokenize(const char *line, fv *fv, short *y, std::size_t *maxid) {
  *y = static_cast<short>(atoi(line));
  if (*y != 1 && *y != -1) {
    LOG(ERROR) << "Invalid label. A label must be +1 or -1.";
    return false;
  }

  while (!isspace(*line) && *line) line++;
  while (isspace(*line) && *line) line++;
  while (1) {
    const char *begin = line;
    unsigned int index = std::atol(begin);
    while (*line != ':' && *line) line++;
    float val = std::atof(++line);

    while (*line != ' ' && *line) line++;
    fv->push_back(std::make_pair(index, val));
    if (index > *maxid) *maxid = index;
    if (*line++ == 0) break;
  }
  return true;
}
} // namespace kernel

{'fraction_non_alphanumeric': 0.18673740053050397, 'fraction_numerical': 0.011140583554376658, 'mean_word_length': 3.2477477477477477, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4387476', 'n_tokens_mistral': 675, 'n_tokens_neox': 650, 'n_words': 216}