<commit_before>////////////////////////////////////////////////////////////////////////////////
/// @brief Utf8/Utf16 Helper class
///
/// @file
///
/// DISCLAIMER
///
/// Copyright 2014 ArangoDB GmbH, Cologne, Germany
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
///     http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author <NAME>
/// @author <NAME>
/// @author Copyright 2014, ArangoDB GmbH, Cologne, Germany
/// @author Copyright 2010-2013, triAGENS GmbH, Cologne, Germany
////////////////////////////////////////////////////////////////////////////////

#include "Utf8Helper.h"

#include "unicode/normalizer2.h"
#include "unicode/ucasemap.h"
#include "unicode/brkiter.h"
#include "unicode/ustdio.h"

#include "Basics/logging.h"
#include "Basics/tri-strings.h"
#include "Basics/utf8-helper.h"

#ifdef _WIN32
#include "Basics/win-utils.h"
#endif

using namespace triagens::basics;
using namespace std;


Utf8Helper Utf8Helper::DefaultUtf8Helper;

// -----------------------------------------------------------------------------
// --SECTION--                                      constructors and destructors
// -----------------------------------------------------------------------------

Utf8Helper::Utf8Helper () : _coll(0) {
  setCollatorLanguage("");
}

Utf8Helper::Utf8Helper (std::string const& lang) : _coll(0) {
  setCollatorLanguage(lang);
}

Utf8Helper::~Utf8Helper () {
  if (_coll) {
    delete _coll;
  }
}

int Utf8Helper::compareUtf8 (const char* left, const char* right) {
  if (!_coll) {
    LOG_ERROR("no Collator in Utf8Helper::compareUtf8()!");
    return (strcmp(left, right));
  }
<commit_msg>Properly shut down ICU to make valgrind happy.<commit_after>////////////////////////////////////////////////////////////////////////////////
/// @brief Utf8/Utf16 Helper class
///
/// @file
///
/// DISCLAIMER
///
/// Copyright 2014 ArangoDB GmbH, Cologne, Germany
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
///     http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author <NAME>
/// @author <NAME>
/// @author Copyright 2014, ArangoDB GmbH, Cologne, Germany
/// @author Copyright 2010-2013, triAGENS GmbH, Cologne, Germany
////////////////////////////////////////////////////////////////////////////////

#include "Utf8Helper.h"

#include "unicode/normalizer2.h"
#include "unicode/ucasemap.h"
#include "unicode/brkiter.h"
#include "unicode/ustdio.h"
#include "unicode/uclean.h"

#include "Basics/logging.h"
#include "Basics/tri-strings.h"
#include "Basics/utf8-helper.h"

#ifdef _WIN32
#include "Basics/win-utils.h"
#endif

using namespace triagens::basics;
using namespace std;


Utf8Helper Utf8Helper::DefaultUtf8Helper;

// -----------------------------------------------------------------------------
// --SECTION--                                      constructors and destructors
// -----------------------------------------------------------------------------

Utf8Helper::Utf8Helper () : _coll(0) {
  setCollatorLanguage("");
}

Utf8Helper::Utf8Helper (std::string const& lang) : _coll(0) {
  setCollatorLanguage(lang);
}

Utf8Helper::~Utf8Helper () {
  if (_coll) {
    delete _coll;
    if (this == &DefaultUtf8Helper) {
      u_cleanup();
    }
  }
}

int Utf8Helper::compareUtf8 (const char* left, const char* right) {
  if (!_coll) {
    LOG_ERROR("no Collator in Utf8Helper::compareUtf8()!");
    return (strcmp(left, right));
  }

{'fraction_non_alphanumeric': 0.2595520421607378, 'fraction_numerical': 0.022617479139218268, 'mean_word_length': 1.9654947916666667, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6855047', 'n_tokens_mistral': 1352, 'n_tokens_neox': 1267, 'n_words': 385}