<commit_before>/**
 * Copyright (c) 2015 <NAME> <<EMAIL>>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "queryprocessor.h"
#include "seriesparser.h"
#include "util.h"
#include "datetime.h"
#include "status_util.h"

#include <string>
#include <map>
#include <algorithm>
#include <regex>

#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/json_parser.hpp>

namespace Akumuli {

//                        //
//     SeriesMatcher      //
//                        //

static const StringT EMPTY = std::make_pair(nullptr, 0);

SeriesMatcher::SeriesMatcher(u64 starting_id)
    : table(StringTools::create_table(0x1000))
    , series_id(starting_id)
{
    if (starting_id == 0u) {
        AKU_PANIC("Bad series ID");
    }
}

u64 SeriesMatcher::add(const char* begin, const char* end) {
    std::lock_guard<std::mutex> guard(mutex);
    auto id = series_id++;
    aku_Status status;
    StringT sname;
    std::tie(status, sname) = index.append(begin, end);
    if (status != AKU_SUCCESS) {
        series_id = id;
        return 0;
    }
    auto tup = std::make_tuple(std::get<0>(sname), std::get<1>(sname), id);
    table[sname] = id;
    inv_table[id] = sname;
    names.push_back(tup);
    return id;
}

void SeriesMatcher::_add(std::string series, u64 id) {
    if (series.empty()) {
        return;
    }
    _add(series.data(), series.data() + series.size(), id);
}

void SeriesMatcher::_add(const char*  begin, const char* end, u64 id) {
    std::lock_guard<std::mutex> guard(mutex);
    aku_Status status;
    StringT sname;
    std::tie(status, sname) = index.append(begin, end);
    StatusUtil::throw_on_error(status);
    auto tup = std::make_tuple(std::get<0>(sname), std::get<1>(sname), id);
    table[sname] = id;
    inv_table[id] = sname;
    names.push_back(tup);
}

u64 SeriesMatcher::match(const char* begin, const char* end) const {
    int len = static_cast<int>(end - begin);
    StringT str = std::make_pair(begin, len);

    std::lock_guard<std::mutex> guard(mutex);
    auto it = table.find(str);
    if (it == table.end()) {
        return 0ul;
    }
    return it->second;
}

StringT SeriesMatcher::id2str(u64 tokenid) const {
    std::lock_guard<std::mutex> guard(mutex);
    auto it = inv_table.find(tokenid);
    if (it == inv_table.end()) {
        return EMPTY;
    }
    return it->second;
}

void SeriesMatcher::pull_new_names(std::vector<PlainSeriesMatcher::SeriesNameT> *buffer) {
    std::lock_guard<std::mutex> guard(mutex);
    std::swap(names, *buffer);
}

std::vector<u64> SeriesMatcher::get_all_ids() const {<commit_msg>Fix error in series parser<commit_after>/**
 * Copyright (c) 2015 <NAME> <<EMAIL>>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "queryprocessor.h"
#include "seriesparser.h"
#include "util.h"
#include "datetime.h"
#include "status_util.h"

#include <string>
#include <map>
#include <algorithm>
#include <regex>

#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/json_parser.hpp>

namespace Akumuli {

//                        //
//     SeriesMatcher      //
//                        //

static const StringT EMPTY = std::make_pair(nullptr, 0);

SeriesMatcher::SeriesMatcher(u64 starting_id)
    : table(StringTools::create_table(0x1000))
    , series_id(starting_id)
{
    if (starting_id == 0u) {
        AKU_PANIC("Bad series ID");
    }
}

u64 SeriesMatcher::add(const char* begin, const char* end) {
    std::lock_guard<std::mutex> guard(mutex);
    auto id = series_id++;
    aku_Status status;
    StringT sname;
    std::tie(status, sname) = index.append(begin, end);
    if (status != AKU_SUCCESS) {
        series_id = id;
        return 0;
    }
    auto tup = std::make_tuple(std::get<0>(sname), std::get<1>(sname), id);
    table[sname] = id;
    inv_table[id] = sname;
    names.push_back(tup);
    return id;
}

void SeriesMatcher::_add(std::string series, u64 id) {
    if (series.empty()) {
        return;
    }
    _add(series.data(), series.data() + series.size(), id);
}

void SeriesMatcher::_add(const char*  begin, const char* end, u64 id) {
    std::lock_guard<std::mutex> guard(mutex);
    aku_Status status;
    StringT sname;
    std::tie(status, sname) = index.append(begin, end);
    StatusUtil::throw_on_error(status);
    table[sname] = id;
    inv_table[id] = sname;
}

u64 SeriesMatcher::match(const char* begin, const char* end) const {
    int len = static_cast<int>(end - begin);
    StringT str = std::make_pair(begin, len);

    std::lock_guard<std::mutex> guard(mutex);
    auto it = table.find(str);
    if (it == table.end()) {
        return 0ul;
    }
    return it->second;
}

StringT SeriesMatcher::id2str(u64 tokenid) const {
    std::lock_guard<std::mutex> guard(mutex);
    auto it = inv_table.find(tokenid);
    if (it == inv_table.end()) {
        return EMPTY;
    }
    return it->second;
}

void SeriesMatcher::pull_new_names(std::vector<PlainSeriesMatcher::SeriesNameT> *buffer) {
    std::lock_guard<std::mutex> guard(mutex);
    std::swap(names, *buffer);
}

std::vector<u64> SeriesMatcher::get_all_ids() const {
{'fraction_non_alphanumeric': 0.14082666231008006, 'fraction_numerical': 0.011436039862767521, 'mean_word_length': 3.5047829286239884, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4492009', 'n_tokens_mistral': 2080, 'n_tokens_neox': 2025, 'n_words': 518}