<commit_before>/* * keyvi - A key value store.
 *
 * Copyright 2015 <NAME><<EMAIL>>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
 * json_value_store.h
 *
 *  Created on: October 13, 2015
 *      Author: <NAME><<EMAIL>>
 */

#ifndef JSON_VALUE_STORE_H_
#define JSON_VALUE_STORE_H_

#include "dictionary/fsa/internal/intrinsics.h"
#if defined(KEYVI_SSE42)
#define RAPIDJSON_SSE42
#endif

#include <functional>
#include <boost/functional/hash.hpp>
#include <boost/lexical_cast.hpp>
#include "rapidjson/document.h"
#include "rapidjson/writer.h"
#include "rapidjson/stringbuffer.h"

#include "dictionary/fsa/internal/ivalue_store.h"
#include "dictionary/fsa/internal/serialization_utils.h"
#include "dictionary/fsa/internal/lru_generation_cache.h"
#include "dictionary/fsa/internal/memory_map_manager.h"
#include "dictionary/util/vint.h"

#include "dictionary/util/json_to_msgpack_fwd.h"
#include "msgpack.hpp"
#include "dictionary/util/json_to_msgpack.h"
#include "msgpack/zbuffer.hpp"

#include "dictionary/fsa/internal/constants.h"
#include "compression/compression_selector.h"

//#define ENABLE_TRACING
#include "dictionary/util/trace.h"

namespace keyvi {
namespace dictionary {
namespace fsa {
namespace internal {

/** Decompresses (if needed) and decodes a json value stored in a JsonValueStore. */
inline std::string DecodeJsonValue(const std::string& encoded_value) {
  compression::decompress_func_t decompressor =
      compression::decompressor_by_code(encoded_value);
  std::string packed_string = decompressor(encoded_value);
  TRACE("unpacking %s", packed_string.c_str());

  msgpack::unpacked doc;
  msgpack::unpack(&doc, packed_string.data(), packed_string.size());
  rapidjson::Document json_document;

  doc.get().convert(&json_document);

  rapidjson::StringBuffer buffer;
  rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
  json_document.Accept(writer);
  return buffer.GetString();
}

/**
 * Encodes @p raw_value with msgpack and compresses it, if it is longer than
 * the specified threshold.
 */
inline std::string EncodeJsonValue(
    std::function<std::string(const char*, size_t)> long_compress,
    std::function<std::string(const char*, size_t)> short_compress,
    msgpack::sbuffer& msgpack_buffer,
    const std::string& raw_value, size_t compression_threshold=32) {
  std::string packed_value;
  
  rapidjson::Document json_document;
  json_document.Parse(raw_value.c_str());

  if (!json_document.HasParseError()) {
    TRACE("Got json");
    msgpack::pack(&msgpack_buffer, json_document);
  } else {
    TRACE("Got a normal string");
    msgpack::pack(&msgpack_buffer, raw_value);
  }

  // compression
  if (msgpack_buffer.size() > compression_threshold) {
    packed_value = long_compress(
        msgpack_buffer.data(), msgpack_buffer.size());
  } else {
    packed_value = short_compress(
        msgpack_buffer.data(), msgpack_buffer.size());
  }

  TRACE("Packed value: %s", packed_value.c_str());
  return packed_value;
}

/**
 * Encodes @p raw_value with msgpack and compresses it, if it is longer than
 * the specified threshold.
 *
 * @note This is a default implementation that uses snappy for string longer
 *       than 32 characters.
 */
inline std::string EncodeJsonValue(const std::string& raw_value,
                                   size_t compression_threshold=32) {
  msgpack::sbuffer msgpack_buffer;
  return EncodeJsonValue(static_cast<std::string(const char*, size_t)>(&compression::SnappyCompressionStrategy::DoCompress),
                         static_cast<std::string(const char*, size_t)>(&compression::RawCompressionStrategy::DoCompress),
                         msgpack_buffer, raw_value, compression_threshold);
}
  

/**
 * Value store where the value consists of a single integer.
 */
class JsonValueStore final : public IValueStoreWriter {
 public:

  struct RawPointer final {
   public:
    RawPointer() : RawPointer(0, 0, 0) {}

    RawPointer(uint64_t offset, int hashcode, size_t length)
        : offset_(offset), hashcode_(hashcode), length_(length) {
      if (length > USHRT_MAX) {
        length_ = USHRT_MAX;
      }
    }<commit_msg>Function POINTER. :)<commit_after>/* * keyvi - A key value store.
 *
 * Copyright 2015 <NAME><<EMAIL>>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
 * json_value_store.h
 *
 *  Created on: October 13, 2015
 *      Author: <NAME><<EMAIL>>
 */

#ifndef JSON_VALUE_STORE_H_
#define JSON_VALUE_STORE_H_

#include "dictionary/fsa/internal/intrinsics.h"
#if defined(KEYVI_SSE42)
#define RAPIDJSON_SSE42
#endif

#include <functional>
#include <boost/functional/hash.hpp>
#include <boost/lexical_cast.hpp>
#include "rapidjson/document.h"
#include "rapidjson/writer.h"
#include "rapidjson/stringbuffer.h"

#include "dictionary/fsa/internal/ivalue_store.h"
#include "dictionary/fsa/internal/serialization_utils.h"
#include "dictionary/fsa/internal/lru_generation_cache.h"
#include "dictionary/fsa/internal/memory_map_manager.h"
#include "dictionary/util/vint.h"

#include "dictionary/util/json_to_msgpack_fwd.h"
#include "msgpack.hpp"
#include "dictionary/util/json_to_msgpack.h"
#include "msgpack/zbuffer.hpp"

#include "dictionary/fsa/internal/constants.h"
#include "compression/compression_selector.h"

//#define ENABLE_TRACING
#include "dictionary/util/trace.h"

namespace keyvi {
namespace dictionary {
namespace fsa {
namespace internal {

/** Decompresses (if needed) and decodes a json value stored in a JsonValueStore. */
inline std::string DecodeJsonValue(const std::string& encoded_value) {
  compression::decompress_func_t decompressor =
      compression::decompressor_by_code(encoded_value);
  std::string packed_string = decompressor(encoded_value);
  TRACE("unpacking %s", packed_string.c_str());

  msgpack::unpacked doc;
  msgpack::unpack(&doc, packed_string.data(), packed_string.size());
  rapidjson::Document json_document;

  doc.get().convert(&json_document);

  rapidjson::StringBuffer buffer;
  rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
  json_document.Accept(writer);
  return buffer.GetString();
}

/**
 * Encodes @p raw_value with msgpack and compresses it, if it is longer than
 * the specified threshold.
 */
inline std::string EncodeJsonValue(
    std::function<std::string(const char*, size_t)> long_compress,
    std::function<std::string(const char*, size_t)> short_compress,
    msgpack::sbuffer& msgpack_buffer,
    const std::string& raw_value, size_t compression_threshold=32) {
  std::string packed_value;
  
  rapidjson::Document json_document;
  json_document.Parse(raw_value.c_str());

  if (!json_document.HasParseError()) {
    TRACE("Got json");
    msgpack::pack(&msgpack_buffer, json_document);
  } else {
    TRACE("Got a normal string");
    msgpack::pack(&msgpack_buffer, raw_value);
  }

  // compression
  if (msgpack_buffer.size() > compression_threshold) {
    packed_value = long_compress(
        msgpack_buffer.data(), msgpack_buffer.size());
  } else {
    packed_value = short_compress(
        msgpack_buffer.data(), msgpack_buffer.size());
  }

  TRACE("Packed value: %s", packed_value.c_str());
  return packed_value;
}

/**
 * Encodes @p raw_value with msgpack and compresses it, if it is longer than
 * the specified threshold.
 *
 * @note This is a default implementation that uses snappy for string longer
 *       than 32 characters.
 */
inline std::string EncodeJsonValue(const std::string& raw_value,
                                   size_t compression_threshold=32) {
  msgpack::sbuffer msgpack_buffer;
  return EncodeJsonValue(static_cast<std::string(*)(const char*, size_t)>(
                          &compression::SnappyCompressionStrategy::DoCompress),
                         static_cast<std::string(*)(const char*, size_t)>(
                           &compression::RawCompressionStrategy::DoCompress),
                         msgpack_buffer, raw_value, compression_threshold);
}
  

/**
 * Value store where the value consists of a single integer.
 */
class JsonValueStore final : public IValueStoreWriter {
 public:

  struct RawPointer final {
   public:
    RawPointer() : RawPointer(0, 0, 0) {}

    RawPointer(uint64_t offset, int hashcode, size_t length)
        : offset_(offset), hashcode_(hashcode), length_(length) {
      if (length > USHRT_MAX) {
        length_ = USHRT_MAX;
      }
    }
{'fraction_non_alphanumeric': 0.11497326203208556, 'fraction_numerical': 0.006203208556149733, 'mean_word_length': 4.346483704974271, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1368342', 'n_tokens_mistral': 2955, 'n_tokens_neox': 2865, 'n_words': 708}