<commit_before>#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "token.h"

// FIXME: from newparser.c
int prsr_token(char *buf, int (*fp)(token *));

// reads stdin into buf, reallocating as nessecary. returns strlen(buf) or < 0 for error.
int read_stdin(char **buf) {
  int pos = 0;
  int size = 1024;
  *buf = malloc(size);

  for (;;) {
    if (pos >= size - 1) {
      size *= 2;
      *buf = realloc(*buf, size);
    }
    if (!fgets(*buf + pos, size - pos, stdin)) {
      break;
    }
    pos += strlen(*buf + pos);
  }
  if (ferror(stdin)) {
    return -1;
  } 

  return pos;
}

int render(token *out) {
  if (out->len == 1 && *out->p == '\n') {
    // don't display, javascript is dumb
  } else if (out->type == TOKEN_ASI) {
    printf(";%4d: \n", out->line_no);
  } else {
    printf("%c%4d: %.*s\n", out->whitespace_after ? '.' : ' ', out->line_no, out->len, out->p);
  }
  return 0;
}

int main() {
  char *buf;
  if (read_stdin(&buf) < 0) {
    return -1;
  }

  int rem = prsr_token(buf, render);
  if (rem > 0) {
    printf("can't parse reminder:\n%s\n", buf + rem);
    return -2;
  }

  return 0;
}<commit_msg>print token type<commit_after>#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "token.h"

// FIXME: from newparser.c
int prsr_token(char *buf, int (*fp)(token *));

// reads stdin into buf, reallocating as nessecary. returns strlen(buf) or < 0 for error.
int read_stdin(char **buf) {
  int pos = 0;
  int size = 1024;
  *buf = malloc(size);

  for (;;) {
    if (pos >= size - 1) {
      size *= 2;
      *buf = realloc(*buf, size);
    }
    if (!fgets(*buf + pos, size - pos, stdin)) {
      break;
    }
    pos += strlen(*buf + pos);
  }
  if (ferror(stdin)) {
    return -1;
  } 

  return pos;
}

int render(token *out) {
  if (out->len == 1 && *out->p == '\n') {
    // don't display, javascript is dumb
  } else if (out->type == TOKEN_ASI) {
    printf(";%4d: \n", out->line_no);
  } else {
    printf("%c%4d: %.*s #%d\n", out->whitespace_after ? '.' : ' ', out->line_no, out->len, out->p, out->type);
  }
  return 0;
}

int main() {
  char *buf;
  if (read_stdin(&buf) < 0) {
    return -1;
  }

  int rem = prsr_token(buf, render);
  if (rem > 0) {
    printf("can't parse reminder:\n%s\n", buf + rem);
    return -2;
  }

  return 0;
}
{'fraction_non_alphanumeric': 0.20198961937716264, 'fraction_numerical': 0.015570934256055362, 'mean_word_length': 2.5694444444444446, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1800537', 'n_tokens_mistral': 984, 'n_tokens_neox': 932, 'n_words': 244}