<commit_before>/*
 * Copyright 2014 Google Inc. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "flatbuffers/flatbuffers.h"
#include "flatbuffers/idl.h"
#include "flatbuffers/util.h"

static void Error(const char *err, const char *obj = nullptr,
                  bool usage = false);

namespace flatbuffers {

bool GenerateBinary(const Parser &parser,
                    const std::string &path,
                    const std::string &file_name,
                    const GeneratorOptions & /*opts*/) {
  auto ext = parser.file_extension_.length() ? parser.file_extension_ : "bin";
  return !parser.builder_.GetSize() ||
         flatbuffers::SaveFile(
           (path + file_name + "." + ext).c_str(),
           reinterpret_cast<char *>(parser.builder_.GetBufferPointer()),
           parser.builder_.GetSize(),
           true);
}

bool GenerateTextFile(const Parser &parser,
                      const std::string &path,
                      const std::string &file_name,
                      const GeneratorOptions &opts) {
  if (!parser.builder_.GetSize()) return true;
  if (!parser.root_struct_def) Error("root_type not set");
  std::string text;
  GenerateText(parser, parser.builder_.GetBufferPointer(), opts,
               &text);
  return flatbuffers::SaveFile((path + file_name + ".json").c_str(),
                               text,
                               false);

}

}

// This struct allows us to create a table of all possible output generators
// for the various programming languages and formats we support.
struct Generator {
  bool (*generate)(const flatbuffers::Parser &parser,
                   const std::string &path,
                   const std::string &file_name,
                   const flatbuffers::GeneratorOptions &opts);
  const char *extension;
  const char *name;
  const char *help;
};

const Generator generators[] = {
  { flatbuffers::GenerateBinary,   "b", "binary",
    "Generate wire format binaries for any data definitions" },
  { flatbuffers::GenerateTextFile, "t", "text",
    "Generate text output for any data definitions" },
  { flatbuffers::GenerateCPP,      "c", "C++",
    "Generate C++ headers for tables/structs" },
  { flatbuffers::GenerateGo,       "g", "Go",
    "Generate Go files for tables/structs" },
  { flatbuffers::GenerateJava,     "j", "Java",
    "Generate Java classes for tables/structs" },
};

const char *program_name = NULL;

static void Error(const char *err, const char *obj, bool usage) {
  printf("%s: %s", program_name, err);
  if (obj) printf(": %s", obj);
  printf("\n");
  if (usage) {
    printf("usage: %s [OPTION]... FILE... [-- FILE...]\n", program_name);
    for (size_t i = 0; i < sizeof(generators) / sizeof(generators[0]); ++i)
      printf("  -%s      %s.\n", generators[i].extension, generators[i].help);
    printf("  -o PATH Prefix PATH to all generated files.\n"
           "  -S      Strict JSON: add quotes to field names.\n"
           "FILEs may depend on declarations in earlier files.\n"
           "FILEs after the -- must be binary flatbuffer format files.\n"
           "Output files are named using the base file name of the input,"
           "and written to the current directory or the path given by -o.\n"
           "example: %s -c -b schema1.fbs schema2.fbs data.json\n",
           program_name);
  }
  exit(1);
}

int main(int argc, const char *argv[]) {
  program_name = argv[0];
  flatbuffers::Parser parser;
  flatbuffers::GeneratorOptions opts;
  std::string output_path;
  const size_t num_generators = sizeof(generators) / sizeof(generators[0]);
  bool generator_enabled[num_generators] = { false };
  bool any_generator = false;
  std::vector<std::string> filenames;
  size_t binary_files_from = std::numeric_limits<size_t>::max();
  for (int i = 1; i < argc; i++) {
    const char *arg = argv[i];
    if (arg[0] == '-') {
      if (filenames.size() && arg[1] != '-')
        Error("invalid option location", arg, true);
      if (strlen(arg) != 2)
        Error("invalid commandline argument", arg, true);
      switch (arg[1]) {
        case 'o':
          if (++i >= argc) Error("missing path following", arg, true);
          output_path = argv[i];
          if (!(output_path.back() == flatbuffers::kPathSeparator ||
                output_path.back() == flatbuffers::kPosixPathSeparator)) {
            output_path += flatbuffers::kPathSeparator;
          }
          break;
        case 'S':
          opts.strict_json = true;
          break;
        case '-':  // Separator between text and binary input files.
          binary_files_from = filenames.size();
          break;
        default:
          for (size_t i = 0; i < num_generators; ++i) {
            if(!strcmp(arg+1, generators[i].extension)) {
              generator_enabled[i] = true;
              any_generator = true;
              goto found;
            }
          }
          Error("unknown commandline argument", arg, true);
          found:
          break;
      }
    } else {
      filenames.push_back(argv[i]);
    }
  }

  if (!filenames.size()) Error("missing input files", nullptr, true);

  if (!any_generator)
    Error("no options: no output files generated.",
          "specify one of -c -g -j -t -b etc.", true);

  // Now process the files:
  for (auto file_it = filenames.begin();
            file_it != filenames.end();
          ++file_it) {
      std::string contents;
      if (!flatbuffers::LoadFile(file_it->c_str(), true, &contents))
        Error("unable to load file", file_it->c_str());

      bool is_binary = static_cast<size_t>(file_it - filenames.begin()) >=
                       binary_files_from;
      if (is_binary) {
        parser.builder_.Clear();
        parser.builder_.PushBytes(
          reinterpret_cast<const uint8_t *>(contents.c_str()),
          contents.length());
      } else {
        if (!parser.Parse(contents.c_str(), file_it->c_str()))
          Error(parser.error_.c_str());
      }

      std::string filebase = flatbuffers::StripPath(
                               flatbuffers::StripExtension(*file_it));

      for (size_t i = 0; i < num_generators; ++i) {
        if (generator_enabled[i]) {
          flatbuffers::EnsureDirExists(output_path);
          if (!generators[i].generate(parser, output_path, filebase, opts)) {
            Error((std::string("Unable to generate ") +
                   generators[i].name +
                   " for " +
                   filebase).c_str());
          }
        }
      }<commit_msg>flatc now outputs the filename with error messages.<commit_after>/*
 * Copyright 2014 Google Inc. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "flatbuffers/flatbuffers.h"
#include "flatbuffers/idl.h"
#include "flatbuffers/util.h"

static void Error(const char *err, const char *obj = nullptr,
                  bool usage = false);

namespace flatbuffers {

bool GenerateBinary(const Parser &parser,
                    const std::string &path,
                    const std::string &file_name,
                    const GeneratorOptions & /*opts*/) {
  auto ext = parser.file_extension_.length() ? parser.file_extension_ : "bin";
  return !parser.builder_.GetSize() ||
         flatbuffers::SaveFile(
           (path + file_name + "." + ext).c_str(),
           reinterpret_cast<char *>(parser.builder_.GetBufferPointer()),
           parser.builder_.GetSize(),
           true);
}

bool GenerateTextFile(const Parser &parser,
                      const std::string &path,
                      const std::string &file_name,
                      const GeneratorOptions &opts) {
  if (!parser.builder_.GetSize()) return true;
  if (!parser.root_struct_def) Error("root_type not set");
  std::string text;
  GenerateText(parser, parser.builder_.GetBufferPointer(), opts,
               &text);
  return flatbuffers::SaveFile((path + file_name + ".json").c_str(),
                               text,
                               false);

}

}

// This struct allows us to create a table of all possible output generators
// for the various programming languages and formats we support.
struct Generator {
  bool (*generate)(const flatbuffers::Parser &parser,
                   const std::string &path,
                   const std::string &file_name,
                   const flatbuffers::GeneratorOptions &opts);
  const char *extension;
  const char *name;
  const char *help;
};

const Generator generators[] = {
  { flatbuffers::GenerateBinary,   "b", "binary",
    "Generate wire format binaries for any data definitions" },
  { flatbuffers::GenerateTextFile, "t", "text",
    "Generate text output for any data definitions" },
  { flatbuffers::GenerateCPP,      "c", "C++",
    "Generate C++ headers for tables/structs" },
  { flatbuffers::GenerateGo,       "g", "Go",
    "Generate Go files for tables/structs" },
  { flatbuffers::GenerateJava,     "j", "Java",
    "Generate Java classes for tables/structs" },
};

const char *program_name = NULL;

static void Error(const char *err, const char *obj, bool usage) {
  printf("%s: %s", program_name, err);
  if (obj) printf(": %s", obj);
  printf("\n");
  if (usage) {
    printf("usage: %s [OPTION]... FILE... [-- FILE...]\n", program_name);
    for (size_t i = 0; i < sizeof(generators) / sizeof(generators[0]); ++i)
      printf("  -%s      %s.\n", generators[i].extension, generators[i].help);
    printf("  -o PATH Prefix PATH to all generated files.\n"
           "  -S      Strict JSON: add quotes to field names.\n"
           "FILEs may depend on declarations in earlier files.\n"
           "FILEs after the -- must be binary flatbuffer format files.\n"
           "Output files are named using the base file name of the input,"
           "and written to the current directory or the path given by -o.\n"
           "example: %s -c -b schema1.fbs schema2.fbs data.json\n",
           program_name);
  }
  exit(1);
}

int main(int argc, const char *argv[]) {
  program_name = argv[0];
  flatbuffers::Parser parser;
  flatbuffers::GeneratorOptions opts;
  std::string output_path;
  const size_t num_generators = sizeof(generators) / sizeof(generators[0]);
  bool generator_enabled[num_generators] = { false };
  bool any_generator = false;
  std::vector<std::string> filenames;
  size_t binary_files_from = std::numeric_limits<size_t>::max();
  for (int i = 1; i < argc; i++) {
    const char *arg = argv[i];
    if (arg[0] == '-') {
      if (filenames.size() && arg[1] != '-')
        Error("invalid option location", arg, true);
      if (strlen(arg) != 2)
        Error("invalid commandline argument", arg, true);
      switch (arg[1]) {
        case 'o':
          if (++i >= argc) Error("missing path following", arg, true);
          output_path = argv[i];
          if (!(output_path.back() == flatbuffers::kPathSeparator ||
                output_path.back() == flatbuffers::kPosixPathSeparator)) {
            output_path += flatbuffers::kPathSeparator;
          }
          break;
        case 'S':
          opts.strict_json = true;
          break;
        case '-':  // Separator between text and binary input files.
          binary_files_from = filenames.size();
          break;
        default:
          for (size_t i = 0; i < num_generators; ++i) {
            if(!strcmp(arg+1, generators[i].extension)) {
              generator_enabled[i] = true;
              any_generator = true;
              goto found;
            }
          }
          Error("unknown commandline argument", arg, true);
          found:
          break;
      }
    } else {
      filenames.push_back(argv[i]);
    }
  }

  if (!filenames.size()) Error("missing input files", nullptr, true);

  if (!any_generator)
    Error("no options: no output files generated.",
          "specify one of -c -g -j -t -b etc.", true);

  // Now process the files:
  for (auto file_it = filenames.begin();
            file_it != filenames.end();
          ++file_it) {
      std::string contents;
      if (!flatbuffers::LoadFile(file_it->c_str(), true, &contents))
        Error("unable to load file", file_it->c_str());

      bool is_binary = static_cast<size_t>(file_it - filenames.begin()) >=
                       binary_files_from;
      if (is_binary) {
        parser.builder_.Clear();
        parser.builder_.PushBytes(
          reinterpret_cast<const uint8_t *>(contents.c_str()),
          contents.length());
      } else {
        if (!parser.Parse(contents.c_str(), file_it->c_str()))
          Error((*file_it + ": " + parser.error_).c_str());
      }

      std::string filebase = flatbuffers::StripPath(
                               flatbuffers::StripExtension(*file_it));

      for (size_t i = 0; i < num_generators; ++i) {
        if (generator_enabled[i]) {
          flatbuffers::EnsureDirExists(output_path);
          if (!generators[i].generate(parser, output_path, filebase, opts)) {
            Error((std::string("Unable to generate ") +
                   generators[i].name +
                   " for " +
                   filebase).c_str());
          }
        }
      }
{'fraction_non_alphanumeric': 0.1292531413242976, 'fraction_numerical': 0.0033883947479881405, 'mean_word_length': 2.381145584725537, 'pattern_counts': {'":': 3, '<': 21, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3773171', 'n_tokens_mistral': 4128, 'n_tokens_neox': 4062, 'n_words': 1180}