<commit_before>// Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/tools/convert_dict/aff_reader.h"

#include <algorithm>

#include "base/file_util.h"
#include "base/string_util.h"
#include "chrome/tools/convert_dict/hunspell_reader.h"

namespace convert_dict {

namespace {

// Returns true if the given line begins with the given case-sensitive
// NULL-terminated ASCII string.
bool StringBeginsWith(const std::string& str, const char* with) {
  size_t cur = 0;
  while (cur < str.size() && with[cur] != 0) {
    if (str[cur] != with[cur])
      return false;
    cur++;
  }
  return with[cur] == 0;
}

// Collapses runs of spaces to only one space.
void CollapseDuplicateSpaces(std::string* str) {
  int prev_space = false;
  for (size_t i = 0; i < str->length(); i++) {
    if ((*str)[i] == ' ') {
      if (prev_space) {
        str->erase(str->begin() + i);
        i--;
      }
      prev_space = true;
    } else {
      prev_space = false;
    }
  }
}

}  // namespace

AffReader::AffReader(const std::string& filename) {
  file_ = file_util::OpenFile(filename, "r");<commit_msg>Coverity: Initialize has_indexed_affixes_ in the constructor.<commit_after>// Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/tools/convert_dict/aff_reader.h"

#include <algorithm>

#include "base/file_util.h"
#include "base/string_util.h"
#include "chrome/tools/convert_dict/hunspell_reader.h"

namespace convert_dict {

namespace {

// Returns true if the given line begins with the given case-sensitive
// NULL-terminated ASCII string.
bool StringBeginsWith(const std::string& str, const char* with) {
  size_t cur = 0;
  while (cur < str.size() && with[cur] != 0) {
    if (str[cur] != with[cur])
      return false;
    cur++;
  }
  return with[cur] == 0;
}

// Collapses runs of spaces to only one space.
void CollapseDuplicateSpaces(std::string* str) {
  int prev_space = false;
  for (size_t i = 0; i < str->length(); i++) {
    if ((*str)[i] == ' ') {
      if (prev_space) {
        str->erase(str->begin() + i);
        i--;
      }
      prev_space = true;
    } else {
      prev_space = false;
    }
  }
}

}  // namespace

AffReader::AffReader(const std::string& filename)
    : has_indexed_affixes_(false) {
  file_ = file_util::OpenFile(filename, "r");
{'fraction_non_alphanumeric': 0.14066193853427897, 'fraction_numerical': 0.009456264775413711, 'mean_word_length': 3.252931323283082, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4110154', 'n_tokens_mistral': 887, 'n_tokens_neox': 857, 'n_words': 245}