<commit_before>/* * keyvi - A key value store.
 *
 * Copyright 2015 <NAME><<EMAIL>>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
 * automata.h
 *
 *  Created on: May 12, 2014
 *      Author: hendrik
 */

#ifndef AUTOMATA_H_
#define AUTOMATA_H_

#include <sys/mman.h>
#include <boost/lexical_cast.hpp>
#include <boost/filesystem.hpp>
#include <boost/interprocess/file_mapping.hpp>
#include <boost/interprocess/mapped_region.hpp>
#include "dictionary/fsa/internal/constants.h"
#include "dictionary/fsa/internal/value_store_factory.h"
#include "dictionary/fsa/internal/serialization_utils.h"
#include "dictionary/fsa/internal/memory_map_flags.h"
#include "dictionary/fsa/traversal/traversal_base.h"
#include "dictionary/fsa/traversal/weighted_traversal.h"
#include "dictionary/util/vint.h"
#include "dictionary/util/endian.h"
#include "dictionary/fsa/internal/intrinsics.h"
#include "dictionary/dictionary_merger_fwd.h"
#include "dictionary/util/keyvi_file.h"

//#define ENABLE_TRACING
#include "dictionary/util/trace.h"

namespace keyvi {
namespace dictionary {
namespace fsa {

/// TODO: refator (split) class Automata, so there is no need for param "loadVS" and friend classes
class Automata
final {

public:
    explicit Automata(const std::string&  filename, loading_strategy_types loading_strategy = loading_strategy_types::lazy)
            : Automata(filename, loading_strategy, true)
    {}

private:
    template<typename , typename>
    friend class ::keyvi::dictionary::DictionaryMerger;

    explicit Automata(const std::string& filename, loading_strategy_types loading_strategy , const bool loadVS) {
        using namespace ::boost;
        using namespace ::boost::interprocess;
        using namespace internal;

        util::KeyViFile keyViFile(filename);

        automata_properties_ = keyViFile.automataProperties();
        start_state_ = lexical_cast<uint64_t> (automata_properties_.get<std::string>("start_state"));
        number_of_keys_ = lexical_cast<uint64_t> (automata_properties_.get<std::string>("number_of_keys"));

        std::istream& persistenceStream = keyViFile.persistenceStream();
        sparse_array_properties_ = SerializationUtils::ReadJsonRecord(persistenceStream);

        compact_size_ = lexical_cast<uint32_t> (sparse_array_properties_.get<std::string>("version")) == 2;
        const size_t bucket_size = compact_size_ ? sizeof(uint16_t) : sizeof(uint32_t);
        const size_t array_size = lexical_cast<size_t>(sparse_array_properties_.get<std::string>("size"));

        const std::streampos offset = persistenceStream.tellg();

        file_mapping_ = file_mapping(filename.c_str(), interprocess::read_only);
<commit_msg> - removed redundant friend declaration<commit_after>/* * keyvi - A key value store.
 *
 * Copyright 2015 <NAME><<EMAIL>>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
 * automata.h
 *
 *  Created on: May 12, 2014
 *      Author: hendrik
 */

#ifndef AUTOMATA_H_
#define AUTOMATA_H_

#include <sys/mman.h>
#include <boost/lexical_cast.hpp>
#include <boost/filesystem.hpp>
#include <boost/interprocess/file_mapping.hpp>
#include <boost/interprocess/mapped_region.hpp>
#include "dictionary/fsa/internal/constants.h"
#include "dictionary/fsa/internal/value_store_factory.h"
#include "dictionary/fsa/internal/serialization_utils.h"
#include "dictionary/fsa/internal/memory_map_flags.h"
#include "dictionary/fsa/traversal/traversal_base.h"
#include "dictionary/fsa/traversal/weighted_traversal.h"
#include "dictionary/util/vint.h"
#include "dictionary/util/endian.h"
#include "dictionary/fsa/internal/intrinsics.h"
#include "dictionary/dictionary_merger_fwd.h"
#include "dictionary/util/keyvi_file.h"

//#define ENABLE_TRACING
#include "dictionary/util/trace.h"

namespace keyvi {
namespace dictionary {
namespace fsa {

/// TODO: refator (split) class Automata, so there is no need for param "loadVS" and friend classes
class Automata
final {

public:
    explicit Automata(const std::string&  filename, loading_strategy_types loading_strategy = loading_strategy_types::lazy)
            : Automata(filename, loading_strategy, true)
    {}

private:

    explicit Automata(const std::string& filename, loading_strategy_types loading_strategy , const bool loadVS) {
        using namespace ::boost;
        using namespace ::boost::interprocess;
        using namespace internal;

        util::KeyViFile keyViFile(filename);

        automata_properties_ = keyViFile.automataProperties();
        start_state_ = lexical_cast<uint64_t> (automata_properties_.get<std::string>("start_state"));
        number_of_keys_ = lexical_cast<uint64_t> (automata_properties_.get<std::string>("number_of_keys"));

        std::istream& persistenceStream = keyViFile.persistenceStream();
        sparse_array_properties_ = SerializationUtils::ReadJsonRecord(persistenceStream);

        compact_size_ = lexical_cast<uint32_t> (sparse_array_properties_.get<std::string>("version")) == 2;
        const size_t bucket_size = compact_size_ ? sizeof(uint16_t) : sizeof(uint32_t);
        const size_t array_size = lexical_cast<size_t>(sparse_array_properties_.get<std::string>("size"));

        const std::streampos offset = persistenceStream.tellg();

        file_mapping_ = file_mapping(filename.c_str(), interprocess::read_only);

{'fraction_non_alphanumeric': 0.10771907624169566, 'fraction_numerical': 0.007908889591901298, 'mean_word_length': 4.7639015496809485, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7543851', 'n_tokens_mistral': 1987, 'n_tokens_neox': 1902, 'n_words': 461}