<commit_before>/*
 * Copyright (C) 2018 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "src/trace_processor/sqlite/span_join_operator_table.h"

#include <sqlite3.h>
#include <string.h>

#include <algorithm>
#include <set>
#include <utility>

#include "perfetto/base/logging.h"
#include "perfetto/ext/base/string_splitter.h"
#include "perfetto/ext/base/string_utils.h"
#include "perfetto/ext/base/string_view.h"
#include "src/trace_processor/sqlite/sqlite_utils.h"

namespace perfetto {
namespace trace_processor {

namespace {

constexpr char kTsColumnName[] = "ts";
constexpr char kDurColumnName[] = "dur";

bool IsRequiredColumn(const std::string& name) {
  return name == kTsColumnName || name == kDurColumnName;
}

base::Optional<std::string> HasDuplicateColumns(
    const std::vector<SqliteTable::Column>& cols) {
  std::set<std::string> names;
  for (const auto& col : cols) {
    if (names.count(col.name()) > 0)
      return col.name();
    names.insert(col.name());
  }
  return base::nullopt;
}

}  // namespace

SpanJoinOperatorTable::SpanJoinOperatorTable(sqlite3* db, const TraceStorage*)
    : db_(db) {}

void SpanJoinOperatorTable::RegisterTable(sqlite3* db,
                                          const TraceStorage* storage) {
  SqliteTable::Register<SpanJoinOperatorTable>(db, storage, "span_join",
                                               /* read_write */ false,
                                               /* requires_args */ true);

  SqliteTable::Register<SpanJoinOperatorTable>(db, storage, "span_left_join",
                                               /* read_write */ false,
                                               /* requires_args */ true);

  SqliteTable::Register<SpanJoinOperatorTable>(db, storage, "span_outer_join",
                                               /* read_write */ false,
                                               /* requires_args */ true);
}

util::Status SpanJoinOperatorTable::Init(int argc,
                                         const char* const* argv,
                                         Schema* schema) {
  // argv[0] - argv[2] are SQLite populated fields which are always present.
  if (argc < 5)
    return util::Status("SPAN_JOIN: expected at least 2 args");

  TableDescriptor t1_desc;
  auto status = TableDescriptor::Parse(
      std::string(reinterpret_cast<const char*>(argv[3])), &t1_desc);
  if (!status.ok())
    return status;

  TableDescriptor t2_desc;
  status = TableDescriptor::Parse(
      std::string(reinterpret_cast<const char*>(argv[4])), &t2_desc);
  if (!status.ok())
    return status;

  // Check that the partition columns match between the two tables.
  if (t1_desc.partition_col == t2_desc.partition_col) {
    partitioning_ = t1_desc.IsPartitioned()
                        ? PartitioningType::kSamePartitioning
                        : PartitioningType::kNoPartitioning;
    if (partitioning_ == PartitioningType::kNoPartitioning && IsOuterJoin()) {
      return util::ErrStatus(
          "SPAN_JOIN: Outer join not supported for no partition tables");
    }
  } else if (t1_desc.IsPartitioned() && t2_desc.IsPartitioned()) {
    return util::ErrStatus(
        "SPAN_JOIN: mismatching partitions between the two tables; "
        "(partition %s in table %s, partition %s in table %s)",
        t1_desc.partition_col.c_str(), t1_desc.name.c_str(),
        t2_desc.partition_col.c_str(), t2_desc.name.c_str());
  } else {
    if (IsOuterJoin()) {
      return util::ErrStatus(
          "SPAN_JOIN: Outer join not supported for mixed partitioned tables");
    }
    partitioning_ = PartitioningType::kMixedPartitioning;
  }

  bool t1_part_mixed = t1_desc.IsPartitioned() &&
                       partitioning_ == PartitioningType::kMixedPartitioning;
  bool t2_part_mixed = t2_desc.IsPartitioned() &&
                       partitioning_ == PartitioningType::kMixedPartitioning;

  EmitShadowType t1_shadow_type;
  if (IsOuterJoin()) {
    if (t1_part_mixed || partitioning_ == PartitioningType::kNoPartitioning) {
      t1_shadow_type = EmitShadowType::kPresentPartitionOnly;
    } else {
      t1_shadow_type = EmitShadowType::kAll;
    }
  } else {
    t1_shadow_type = EmitShadowType::kNone;
  }
  status = CreateTableDefinition(t1_desc, t1_shadow_type, &t1_defn_);
  if (!status.ok())
    return status;

  EmitShadowType t2_shadow_type;
  if (IsOuterJoin() || IsLeftJoin()) {
    if (t2_part_mixed || partitioning_ == PartitioningType::kNoPartitioning) {
      t2_shadow_type = EmitShadowType::kPresentPartitionOnly;
    } else {
      t2_shadow_type = EmitShadowType::kAll;
    }
  } else {
    t2_shadow_type = EmitShadowType::kNone;
  }
  status = CreateTableDefinition(t2_desc, t2_shadow_type, &t2_defn_);
  if (!status.ok())
    return status;

  std::vector<SqliteTable::Column> cols;
  // Ensure the shared columns are consistently ordered and are not
  // present twice in the final schema
  cols.emplace_back(Column::kTimestamp, kTsColumnName, SqlValue::Type::kLong);
  cols.emplace_back(Column::kDuration, kDurColumnName, SqlValue::Type::kLong);
  if (partitioning_ != PartitioningType::kNoPartitioning)
    cols.emplace_back(Column::kPartition, partition_col(),
                      SqlValue::Type::kLong);

  CreateSchemaColsForDefn(t1_defn_, &cols);
  CreateSchemaColsForDefn(t2_defn_, &cols);

  if (auto opt_dupe_col = HasDuplicateColumns(cols)) {
    return util::ErrStatus(
        "SPAN_JOIN: column %s present in both tables %s and %s",
        opt_dupe_col->c_str(), t1_defn_.name().c_str(),
        t2_defn_.name().c_str());
  }
  std::vector<size_t> primary_keys = {Column::kTimestamp};
  if (partitioning_ != PartitioningType::kNoPartitioning)
    primary_keys.push_back(Column::kPartition);
  *schema = Schema(cols, primary_keys);<commit_msg>tp: add error check for illegal characters in span join<commit_after>/*
 * Copyright (C) 2018 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "src/trace_processor/sqlite/span_join_operator_table.h"

#include <sqlite3.h>
#include <string.h>

#include <algorithm>
#include <set>
#include <utility>

#include "perfetto/base/logging.h"
#include "perfetto/ext/base/string_splitter.h"
#include "perfetto/ext/base/string_utils.h"
#include "perfetto/ext/base/string_view.h"
#include "src/trace_processor/sqlite/sqlite_utils.h"

namespace perfetto {
namespace trace_processor {

namespace {

constexpr char kTsColumnName[] = "ts";
constexpr char kDurColumnName[] = "dur";

bool IsRequiredColumn(const std::string& name) {
  return name == kTsColumnName || name == kDurColumnName;
}

base::Optional<std::string> HasDuplicateColumns(
    const std::vector<SqliteTable::Column>& cols) {
  std::set<std::string> names;
  for (const auto& col : cols) {
    if (names.count(col.name()) > 0)
      return col.name();
    names.insert(col.name());
  }
  return base::nullopt;
}

}  // namespace

SpanJoinOperatorTable::SpanJoinOperatorTable(sqlite3* db, const TraceStorage*)
    : db_(db) {}

void SpanJoinOperatorTable::RegisterTable(sqlite3* db,
                                          const TraceStorage* storage) {
  SqliteTable::Register<SpanJoinOperatorTable>(db, storage, "span_join",
                                               /* read_write */ false,
                                               /* requires_args */ true);

  SqliteTable::Register<SpanJoinOperatorTable>(db, storage, "span_left_join",
                                               /* read_write */ false,
                                               /* requires_args */ true);

  SqliteTable::Register<SpanJoinOperatorTable>(db, storage, "span_outer_join",
                                               /* read_write */ false,
                                               /* requires_args */ true);
}

util::Status SpanJoinOperatorTable::Init(int argc,
                                         const char* const* argv,
                                         Schema* schema) {
  // argv[0] - argv[2] are SQLite populated fields which are always present.
  if (argc < 5)
    return util::Status("SPAN_JOIN: expected at least 2 args");

  TableDescriptor t1_desc;
  auto status = TableDescriptor::Parse(
      std::string(reinterpret_cast<const char*>(argv[3])), &t1_desc);
  if (!status.ok())
    return status;

  TableDescriptor t2_desc;
  status = TableDescriptor::Parse(
      std::string(reinterpret_cast<const char*>(argv[4])), &t2_desc);
  if (!status.ok())
    return status;

  // Check that the partition columns match between the two tables.
  if (t1_desc.partition_col == t2_desc.partition_col) {
    partitioning_ = t1_desc.IsPartitioned()
                        ? PartitioningType::kSamePartitioning
                        : PartitioningType::kNoPartitioning;
    if (partitioning_ == PartitioningType::kNoPartitioning && IsOuterJoin()) {
      return util::ErrStatus(
          "SPAN_JOIN: Outer join not supported for no partition tables");
    }
  } else if (t1_desc.IsPartitioned() && t2_desc.IsPartitioned()) {
    return util::ErrStatus(
        "SPAN_JOIN: mismatching partitions between the two tables; "
        "(partition %s in table %s, partition %s in table %s)",
        t1_desc.partition_col.c_str(), t1_desc.name.c_str(),
        t2_desc.partition_col.c_str(), t2_desc.name.c_str());
  } else {
    if (IsOuterJoin()) {
      return util::ErrStatus(
          "SPAN_JOIN: Outer join not supported for mixed partitioned tables");
    }
    partitioning_ = PartitioningType::kMixedPartitioning;
  }

  bool t1_part_mixed = t1_desc.IsPartitioned() &&
                       partitioning_ == PartitioningType::kMixedPartitioning;
  bool t2_part_mixed = t2_desc.IsPartitioned() &&
                       partitioning_ == PartitioningType::kMixedPartitioning;

  EmitShadowType t1_shadow_type;
  if (IsOuterJoin()) {
    if (t1_part_mixed || partitioning_ == PartitioningType::kNoPartitioning) {
      t1_shadow_type = EmitShadowType::kPresentPartitionOnly;
    } else {
      t1_shadow_type = EmitShadowType::kAll;
    }
  } else {
    t1_shadow_type = EmitShadowType::kNone;
  }
  status = CreateTableDefinition(t1_desc, t1_shadow_type, &t1_defn_);
  if (!status.ok())
    return status;

  EmitShadowType t2_shadow_type;
  if (IsOuterJoin() || IsLeftJoin()) {
    if (t2_part_mixed || partitioning_ == PartitioningType::kNoPartitioning) {
      t2_shadow_type = EmitShadowType::kPresentPartitionOnly;
    } else {
      t2_shadow_type = EmitShadowType::kAll;
    }
  } else {
    t2_shadow_type = EmitShadowType::kNone;
  }
  status = CreateTableDefinition(t2_desc, t2_shadow_type, &t2_defn_);
  if (!status.ok())
    return status;

  std::vector<SqliteTable::Column> cols;
  // Ensure the shared columns are consistently ordered and are not
  // present twice in the final schema
  cols.emplace_back(Column::kTimestamp, kTsColumnName, SqlValue::Type::kLong);
  cols.emplace_back(Column::kDuration, kDurColumnName, SqlValue::Type::kLong);
  if (partitioning_ != PartitioningType::kNoPartitioning)
    cols.emplace_back(Column::kPartition, partition_col(),
                      SqlValue::Type::kLong);

  CreateSchemaColsForDefn(t1_defn_, &cols);
  CreateSchemaColsForDefn(t2_defn_, &cols);

  // Check if any column has : in its name. This often happens when SELECT *
  // is used to create a view with the same column name in two joined tables.
  for (const auto& col : cols) {
    if (col.name().find(':') != std::string::npos) {
      return util::ErrStatus("SPAN_JOIN: column %s has illegal character :",
                             col.name().c_str());
    }
  }

  if (auto opt_dupe_col = HasDuplicateColumns(cols)) {
    return util::ErrStatus(
        "SPAN_JOIN: column %s present in both tables %s and %s",
        opt_dupe_col->c_str(), t1_defn_.name().c_str(),
        t2_defn_.name().c_str());
  }
  std::vector<size_t> primary_keys = {Column::kTimestamp};
  if (partitioning_ != PartitioningType::kNoPartitioning)
    primary_keys.push_back(Column::kPartition);
  *schema = Schema(cols, primary_keys);
{'fraction_non_alphanumeric': 0.11354809230066255, 'fraction_numerical': 0.008377122839083085, 'mean_word_length': 3.0769947221359826, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 36, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1559502', 'n_tokens_mistral': 4010, 'n_tokens_neox': 3939, 'n_words': 918}