<commit_before>#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <algorithm>

#include "llvm/Demangle/Demangle.h"

const char kDemumbleVersion[] = "1.2.0.git";

static void print_help(FILE* out) {
  fprintf(out,
"usage: demumble [options] [symbols...]\n"
"\n"
"if symbols are unspecified, reads from stdin.\n"
"\n"
"options:\n"
"  -b         print both demangled and mangled name\n"
"  -m         only print mangled names that were demangled, omit other output\n"
"  -u         use unbuffered output\n"
"  --version  print demumble version (\"%s\")\n", kDemumbleVersion);
}

static void print_demangled(const char* format, const char* s) {
  if (char* itanium = llvm::itaniumDemangle(s, NULL, NULL, NULL)) {
    printf(format, itanium, s);
    free(itanium);
  } else if (char* ms = llvm::microsoftDemangle(s, NULL, NULL, NULL)) {
    printf(format, ms, s);
    free(ms);
  } else {
    printf("%s", s);
  }
}

static bool is_mangle_char_posix(char c) {
  return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
         (c >= '0' && c <= '9') || c == '_';
}

static bool is_mangle_char_win(char c) {
  return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
         (c >= '0' && c <= '9') || strchr("?_@$", c);
}

static bool is_plausible_itanium_prefix(char* s) {
  // Itanium symbols start with 1-4 underscores followed by Z.
  // strnstr() is BSD, so use a small local buffer and strstr().
  const int N = 5;  // == strlen("____Z")
  char prefix[N + 1];
  strncpy(prefix, s, N); prefix[N] = '\0';
  return strstr(prefix, "_Z");
}

static char buf[8192];
int main(int argc, char* argv[]) {
  enum { kPrintAll, kPrintMatching } print_mode = kPrintAll;
  const char* print_format = "%s";
  while (argc > 1 && argv[1][0] == '-') {
    if (strcmp(argv[1], "-h") == 0 || strcmp(argv[1], "--help") == 0) {
      print_help(stdout);
      return 0;
    } else if (strcmp(argv[1], "-b") == 0) {
      print_format = "\"%s\" (%s)";
    } else if (strcmp(argv[1], "-m") == 0) {
      print_mode = kPrintMatching;
    } else if (strcmp(argv[1], "-u") == 0) {
      setbuf(stdout, NULL);
    } else if (strcmp(argv[1], "--version") == 0) {
      printf("%s\n", kDemumbleVersion);
      return 0;
    } else if (strcmp(argv[1], "--") == 0) {
      --argc;
      ++argv;
      break;
    } else {
      fprintf(stderr, "demumble: unrecognized option `%s'\n", argv[1]);
      print_help(stderr);
      return 1;
    }
    --argc;
    ++argv;
  }
  for (int i = 1; i < argc; ++i) {
    print_demangled(print_format, argv[i]);
    printf("\n");
  }
  if (argc == 1) {  // Read stdin instead.
    // By default, don't demangle types.  Mangled function names are unlikely
    // to appear in text for since they start with _Z (or ___Z) or ?? / ?$ / ?@.
    // But type manglings can be regular words ("Pi" is "int*").
    // (For command-line args, do try to demangle types though.)
    while (fgets(buf, sizeof(buf), stdin)) {
      bool need_separator = false;
      char* cur = buf;
      char* end = cur + strlen(cur);

      while (cur != end) {<commit_msg>minor simplification, no behavior change<commit_after>#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <algorithm>

#include "llvm/Demangle/Demangle.h"

const char kDemumbleVersion[] = "1.2.0.git";

static int print_help(FILE* out) {
  fprintf(out,
"usage: demumble [options] [symbols...]\n"
"\n"
"if symbols are unspecified, reads from stdin.\n"
"\n"
"options:\n"
"  -b         print both demangled and mangled name\n"
"  -m         only print mangled names that were demangled, omit other output\n"
"  -u         use unbuffered output\n"
"  --version  print demumble version (\"%s\")\n", kDemumbleVersion);
  return out == stdout ? 0 : 1;
}

static void print_demangled(const char* format, const char* s) {
  if (char* itanium = llvm::itaniumDemangle(s, NULL, NULL, NULL)) {
    printf(format, itanium, s);
    free(itanium);
  } else if (char* ms = llvm::microsoftDemangle(s, NULL, NULL, NULL)) {
    printf(format, ms, s);
    free(ms);
  } else {
    printf("%s", s);
  }
}

static bool is_mangle_char_posix(char c) {
  return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
         (c >= '0' && c <= '9') || c == '_';
}

static bool is_mangle_char_win(char c) {
  return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
         (c >= '0' && c <= '9') || strchr("?_@$", c);
}

static bool is_plausible_itanium_prefix(char* s) {
  // Itanium symbols start with 1-4 underscores followed by Z.
  // strnstr() is BSD, so use a small local buffer and strstr().
  const int N = 5;  // == strlen("____Z")
  char prefix[N + 1];
  strncpy(prefix, s, N); prefix[N] = '\0';
  return strstr(prefix, "_Z");
}

static char buf[8192];
int main(int argc, char* argv[]) {
  enum { kPrintAll, kPrintMatching } print_mode = kPrintAll;
  const char* print_format = "%s";
  while (argc > 1 && argv[1][0] == '-') {
    if (strcmp(argv[1], "-h") == 0 || strcmp(argv[1], "--help") == 0) {
      return print_help(stdout);
    } else if (strcmp(argv[1], "-b") == 0) {
      print_format = "\"%s\" (%s)";
    } else if (strcmp(argv[1], "-m") == 0) {
      print_mode = kPrintMatching;
    } else if (strcmp(argv[1], "-u") == 0) {
      setbuf(stdout, NULL);
    } else if (strcmp(argv[1], "--version") == 0) {
      printf("%s\n", kDemumbleVersion);
      return 0;
    } else if (strcmp(argv[1], "--") == 0) {
      --argc;
      ++argv;
      break;
    } else {
      fprintf(stderr, "demumble: unrecognized option `%s'\n", argv[1]);
      return print_help(stderr);
    }
    --argc;
    ++argv;
  }
  for (int i = 1; i < argc; ++i) {
    print_demangled(print_format, argv[i]);
    printf("\n");
  }
  if (argc == 1) {  // Read stdin instead.
    // By default, don't demangle types.  Mangled function names are unlikely
    // to appear in text for since they start with _Z (or ___Z) or ?? / ?$ / ?@.
    // But type manglings can be regular words ("Pi" is "int*").
    // (For command-line args, do try to demangle types though.)
    while (fgets(buf, sizeof(buf), stdin)) {
      bool need_separator = false;
      char* cur = buf;
      char* end = cur + strlen(cur);

      while (cur != end) {
{'fraction_non_alphanumeric': 0.19382835101253615, 'fraction_numerical': 0.01253616200578592, 'mean_word_length': 2.8342575477510783, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6131332', 'n_tokens_mistral': 2332, 'n_tokens_neox': 2290, 'n_words': 667}