<commit_before>/**
 * @file   array_schema.cc
 *
 * @section LICENSE
 *
 * The MIT License
 *
 * @copyright Copyright (c) 2018-2022 TileDB, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 *
 * @section DESCRIPTION
 *
 * This file defines serialization functions for ArraySchema.
 */
#ifdef TILEDB_SERIALIZATION
#include <capnp/compat/json.h>
#include <capnp/serialize.h>
#include "tiledb/sm/serialization/capnp_utils.h"
#endif

#include "tiledb/common/heap_memory.h"
#include "tiledb/common/logger.h"
#include "tiledb/sm/array/array.h"
#include "tiledb/sm/array_schema/attribute.h"
#include "tiledb/sm/array_schema/dimension.h"
#include "tiledb/sm/array_schema/domain.h"
#include "tiledb/sm/enums/array_type.h"
#include "tiledb/sm/enums/compressor.h"
#include "tiledb/sm/enums/datatype.h"
#include "tiledb/sm/enums/filter_option.h"
#include "tiledb/sm/enums/filter_type.h"
#include "tiledb/sm/enums/layout.h"
#include "tiledb/sm/enums/serialization_type.h"
#include "tiledb/sm/filter/bit_width_reduction_filter.h"
#include "tiledb/sm/filter/compression_filter.h"
#include "tiledb/sm/filter/filter_create.h"
#include "tiledb/sm/filter/positive_delta_filter.h"
#include "tiledb/sm/misc/constants.h"
#include "tiledb/sm/serialization/array_schema.h"

#include <set>
#include <string>

using namespace tiledb::common;

namespace tiledb {
namespace sm {
namespace serialization {

#ifdef TILEDB_SERIALIZATION

Status filter_pipeline_to_capnp(
    const FilterPipeline* filter_pipeline,
    capnp::FilterPipeline::Builder* filter_pipeline_builder) {
  if (filter_pipeline == nullptr)
    return LOG_STATUS(Status_SerializationError(
        "Error serializing filter pipeline; filter pipeline is null."));

  const unsigned num_filters = filter_pipeline->size();
  if (num_filters == 0)
    return Status::Ok();

  auto filter_list_builder = filter_pipeline_builder->initFilters(num_filters);
  for (unsigned i = 0; i < num_filters; i++) {
    const auto* filter = filter_pipeline->get_filter(i);
    auto filter_builder = filter_list_builder[i];
    filter_builder.setType(filter_type_str(filter->type()));

    switch (filter->type()) {
      case FilterType::FILTER_BIT_WIDTH_REDUCTION: {
        uint32_t window;
        RETURN_NOT_OK(
            filter->get_option(FilterOption::BIT_WIDTH_MAX_WINDOW, &window));
        auto data = filter_builder.initData();
        data.setUint32(window);
        break;
      }
      case FilterType::FILTER_POSITIVE_DELTA: {
        uint32_t window;
        RETURN_NOT_OK(filter->get_option(
            FilterOption::POSITIVE_DELTA_MAX_WINDOW, &window));
        auto data = filter_builder.initData();
        data.setUint32(window);
        break;
      }
      case FilterType::FILTER_GZIP:
      case FilterType::FILTER_ZSTD:
      case FilterType::FILTER_LZ4:
      case FilterType::FILTER_RLE:
      case FilterType::FILTER_BZIP2:
      case FilterType::FILTER_DOUBLE_DELTA:
      case FilterType::FILTER_DICTIONARY: {
        int32_t level;
        RETURN_NOT_OK(
            filter->get_option(FilterOption::COMPRESSION_LEVEL, &level));
        auto data = filter_builder.initData();
        data.setInt32(level);
        break;
      }
      default:
        break;
    }
  }

  return Status::Ok();
}

static tuple<Status, optional<shared_ptr<Filter>>> filter_constructor(
    capnp::Filter::Reader reader) {
  FilterType type = FilterType::FILTER_NONE;
  RETURN_NOT_OK_TUPLE(
      filter_type_enum(reader.getType().cStr(), &type), nullopt);

  switch (type) {
    case FilterType::FILTER_BIT_WIDTH_REDUCTION: {
      auto data = reader.getData();
      uint32_t window = data.getUint32();
      return {
          Status::Ok(),
          tiledb::common::make_shared<BitWidthReductionFilter>(HERE(), window)};
    }
    case FilterType::FILTER_POSITIVE_DELTA: {
      auto data = reader.getData();
      uint32_t window = data.getUint32();
      return {Status::Ok(),
              tiledb::common::make_shared<PositiveDeltaFilter>(HERE(), window)};
    }
    case FilterType::FILTER_GZIP:
    case FilterType::FILTER_ZSTD:
    case FilterType::FILTER_LZ4:
    case FilterType::FILTER_RLE:
    case FilterType::FILTER_BZIP2:
    case FilterType::FILTER_DOUBLE_DELTA:
    case FilterType::FILTER_DICTIONARY: {
      auto data = reader.getData();
      int32_t level = data.getInt32();
      return {
          Status::Ok(),
          tiledb::common::make_shared<CompressionFilter>(HERE(), type, level)};
    }
    default: {
      throw std::logic_error(
          "Invalid data received from filter pipeline capnp reader, unknown "
          "type");
    }
  }
}

tuple<Status, optional<shared_ptr<FilterPipeline>>> filter_pipeline_from_capnp(
    const capnp::FilterPipeline::Reader& filter_pipeline_reader) {
  if (!filter_pipeline_reader.hasFilters())
    return {Status::Ok(), make_shared<FilterPipeline>(HERE())};

  std::vector<shared_ptr<Filter>> filter_list;
  auto filter_list_reader = filter_pipeline_reader.getFilters();
  for (auto filter_reader : filter_list_reader) {
    auto&& [st_f, filter]{filter_constructor(filter_reader)};
    RETURN_NOT_OK_TUPLE(st_f, nullopt);
    filter_list.push_back(filter.value());
  }

  return {Status::Ok(),
          make_shared<FilterPipeline>(
              HERE(), constants::max_tile_chunk_size, filter_list)};
}

Status attribute_to_capnp(
    const Attribute* attribute, capnp::Attribute::Builder* attribute_builder) {<commit_msg>Add missing filters to switch case for Filter serialization (#3256)<commit_after>/**
 * @file   array_schema.cc
 *
 * @section LICENSE
 *
 * The MIT License
 *
 * @copyright Copyright (c) 2018-2022 TileDB, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 *
 * @section DESCRIPTION
 *
 * This file defines serialization functions for ArraySchema.
 */
#ifdef TILEDB_SERIALIZATION
#include <capnp/compat/json.h>
#include <capnp/serialize.h>
#include "tiledb/sm/serialization/capnp_utils.h"
#endif

#include "tiledb/common/heap_memory.h"
#include "tiledb/common/logger.h"
#include "tiledb/sm/array/array.h"
#include "tiledb/sm/array_schema/attribute.h"
#include "tiledb/sm/array_schema/dimension.h"
#include "tiledb/sm/array_schema/domain.h"
#include "tiledb/sm/enums/array_type.h"
#include "tiledb/sm/enums/compressor.h"
#include "tiledb/sm/enums/datatype.h"
#include "tiledb/sm/enums/filter_option.h"
#include "tiledb/sm/enums/filter_type.h"
#include "tiledb/sm/enums/layout.h"
#include "tiledb/sm/enums/serialization_type.h"
#include "tiledb/sm/filter/bit_width_reduction_filter.h"
#include "tiledb/sm/filter/bitshuffle_filter.h"
#include "tiledb/sm/filter/byteshuffle_filter.h"
#include "tiledb/sm/filter/checksum_md5_filter.h"
#include "tiledb/sm/filter/checksum_sha256_filter.h"
#include "tiledb/sm/filter/compression_filter.h"
#include "tiledb/sm/filter/encryption_aes256gcm_filter.h"
#include "tiledb/sm/filter/filter_create.h"
#include "tiledb/sm/filter/positive_delta_filter.h"
#include "tiledb/sm/misc/constants.h"
#include "tiledb/sm/serialization/array_schema.h"

#include <set>
#include <string>

using namespace tiledb::common;

namespace tiledb {
namespace sm {
namespace serialization {

#ifdef TILEDB_SERIALIZATION

Status filter_pipeline_to_capnp(
    const FilterPipeline* filter_pipeline,
    capnp::FilterPipeline::Builder* filter_pipeline_builder) {
  if (filter_pipeline == nullptr)
    return LOG_STATUS(Status_SerializationError(
        "Error serializing filter pipeline; filter pipeline is null."));

  const unsigned num_filters = filter_pipeline->size();
  if (num_filters == 0)
    return Status::Ok();

  auto filter_list_builder = filter_pipeline_builder->initFilters(num_filters);
  for (unsigned i = 0; i < num_filters; i++) {
    const auto* filter = filter_pipeline->get_filter(i);
    auto filter_builder = filter_list_builder[i];
    filter_builder.setType(filter_type_str(filter->type()));

    switch (filter->type()) {
      case FilterType::FILTER_BIT_WIDTH_REDUCTION: {
        uint32_t window;
        RETURN_NOT_OK(
            filter->get_option(FilterOption::BIT_WIDTH_MAX_WINDOW, &window));
        auto data = filter_builder.initData();
        data.setUint32(window);
        break;
      }
      case FilterType::FILTER_POSITIVE_DELTA: {
        uint32_t window;
        RETURN_NOT_OK(filter->get_option(
            FilterOption::POSITIVE_DELTA_MAX_WINDOW, &window));
        auto data = filter_builder.initData();
        data.setUint32(window);
        break;
      }
      case FilterType::FILTER_GZIP:
      case FilterType::FILTER_ZSTD:
      case FilterType::FILTER_LZ4:
      case FilterType::FILTER_RLE:
      case FilterType::FILTER_BZIP2:
      case FilterType::FILTER_DOUBLE_DELTA:
      case FilterType::FILTER_DICTIONARY: {
        int32_t level;
        RETURN_NOT_OK(
            filter->get_option(FilterOption::COMPRESSION_LEVEL, &level));
        auto data = filter_builder.initData();
        data.setInt32(level);
        break;
      }
      case FilterType::FILTER_NONE:
      case FilterType::FILTER_BITSHUFFLE:
      case FilterType::FILTER_BYTESHUFFLE:
      case FilterType::FILTER_CHECKSUM_MD5:
      case FilterType::FILTER_CHECKSUM_SHA256:
      case FilterType::INTERNAL_FILTER_AES_256_GCM:
        break;
    }
  }

  return Status::Ok();
}

static tuple<Status, optional<shared_ptr<Filter>>> filter_constructor(
    capnp::Filter::Reader reader) {
  FilterType type = FilterType::FILTER_NONE;
  RETURN_NOT_OK_TUPLE(
      filter_type_enum(reader.getType().cStr(), &type), nullopt);

  switch (type) {
    case FilterType::FILTER_BIT_WIDTH_REDUCTION: {
      auto data = reader.getData();
      uint32_t window = data.getUint32();
      return {
          Status::Ok(),
          tiledb::common::make_shared<BitWidthReductionFilter>(HERE(), window)};
    }
    case FilterType::FILTER_POSITIVE_DELTA: {
      auto data = reader.getData();
      uint32_t window = data.getUint32();
      return {Status::Ok(),
              tiledb::common::make_shared<PositiveDeltaFilter>(HERE(), window)};
    }
    case FilterType::FILTER_GZIP:
    case FilterType::FILTER_ZSTD:
    case FilterType::FILTER_LZ4:
    case FilterType::FILTER_RLE:
    case FilterType::FILTER_BZIP2:
    case FilterType::FILTER_DOUBLE_DELTA:
    case FilterType::FILTER_DICTIONARY: {
      auto data = reader.getData();
      int32_t level = data.getInt32();
      return {
          Status::Ok(),
          tiledb::common::make_shared<CompressionFilter>(HERE(), type, level)};
    }
    case FilterType::FILTER_NONE:
      break;
    case FilterType::FILTER_BITSHUFFLE: {
      return {Status::Ok(),
              tiledb::common::make_shared<BitshuffleFilter>(HERE())};
    }
    case FilterType::FILTER_BYTESHUFFLE: {
      return {Status::Ok(),
              tiledb::common::make_shared<BitshuffleFilter>(HERE())};
    }
    case FilterType::FILTER_CHECKSUM_MD5: {
      return {Status::Ok(),
              tiledb::common::make_shared<ChecksumMD5Filter>(HERE())};
    }
    case FilterType::FILTER_CHECKSUM_SHA256: {
      return {Status::Ok(),
              tiledb::common::make_shared<ChecksumSHA256Filter>(HERE())};
    }
    case FilterType::INTERNAL_FILTER_AES_256_GCM: {
      return {Status::Ok(),
              tiledb::common::make_shared<EncryptionAES256GCMFilter>(HERE())};
    }
    default: {
      throw std::logic_error(
          "Invalid data received from filter pipeline capnp reader, unknown "
          "type");
    }
  }
  return {Status_SerializationError(
              "Invalid data received from filter pipeline capnp reader, "
              "unknown type " +
              filter_type_str(type)),
          std::nullopt};
}

tuple<Status, optional<shared_ptr<FilterPipeline>>> filter_pipeline_from_capnp(
    const capnp::FilterPipeline::Reader& filter_pipeline_reader) {
  if (!filter_pipeline_reader.hasFilters())
    return {Status::Ok(), make_shared<FilterPipeline>(HERE())};

  std::vector<shared_ptr<Filter>> filter_list;
  auto filter_list_reader = filter_pipeline_reader.getFilters();
  for (auto filter_reader : filter_list_reader) {
    auto&& [st_f, filter]{filter_constructor(filter_reader)};
    RETURN_NOT_OK_TUPLE(st_f, nullopt);
    filter_list.push_back(filter.value());
  }

  return {Status::Ok(),
          make_shared<FilterPipeline>(
              HERE(), constants::max_tile_chunk_size, filter_list)};
}

Status attribute_to_capnp(
    const Attribute* attribute, capnp::Attribute::Builder* attribute_builder) {
{'fraction_non_alphanumeric': 0.11071280849303737, 'fraction_numerical': 0.0074451950916861985, 'mean_word_length': 3.9852233676975946, 'pattern_counts': {'":': 0, '<': 44, '<?xml version=': 0, '>': 58, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3818647', 'n_tokens_mistral': 4621, 'n_tokens_neox': 4537, 'n_words': 1000}