<commit_before>#define _POSIX_C_SOURCE 200809L
#include <ctype.h>
#include <err.h>
#include <inttypes.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

#define MIN(X,Y) ((X) < (Y)? (X) : (Y))
typedef uint8_t u8;

//-- Options ----------------
struct offset_range { off_t start, end; };

size_t option_columns      = 16;
size_t option_groupsize    = 8;
bool option_use_formatting = true;
struct offset_range option_range  = { 0, -1 };

const char *formatting_zero      = "38;5;238";
const char *formatting_all       = "38;5;167";
const char *formatting_low       = "38;5;150";
const char *formatting_high      = "38;5;141";
const char *formatting_printable = "";

//-- Hexdump impl -----------
const char *format_of(int v) {
  return v == 0x00?  formatting_zero
       : v == 0xFF?  formatting_all
       : v <  0x20?  formatting_low
       : v >= 0x7F?  formatting_high
       :             formatting_printable;
}

const char *CHAR_AREA_HIGH_LUT[] = {
  "€", ".", "‚", "ƒ", "„", "…", "†", "‡", "ˆ", "‰", "Š", "‹", "Œ", ".", "Ž", ".",
  ".", "‘", "’", "“", "”", "•", "–", "—", "˜", "™", "š", "›", "œ", ".", "ž", "Ÿ",
  ".", "¡", "¢", "£", "¤", "¥", "¦", "§", "¨", "©", "ª", "«", "¬", ".", "®", "¯",
  "°", "±", "²", "³", "´", "µ", "¶", "·", "¸", "¹", "º", "»", "¼", "½", "¾", "¿",
  "À", "Á", "Â", "Ã", "Ä", "Å", "Æ", "Ç", "È", "É", "Ê", "Ë", "Ì", "Í", "Î", "Ï",
  "Ð", "Ñ", "Ò", "Ó", "Ô", "Õ", "Ö", "×", "Ø", "Ù", "Ú", "Û", "Ü", "Ý", "Þ", "ß",
  "à", "á", "â", "ã", "ä", "å", "æ", "ç", "è", "é", "ê", "ë", "ì", "í", "î", "ï",
  "ð", "ñ", "ò", "ó", "ô", "õ", "ö", "÷", "ø", "ù", "ú", "û", "ü", "ý", "þ", "ÿ",
};

void hexdump(FILE *f, const char *filename) {
  u8 buf[BUFSIZ];
  u8 prev_line[option_columns];

  bool first_line = true, printed_asterisk = false;

  // Seek to start; fall back to a consuming loop for non-seekable files
  if (fseeko(f, option_range.start, SEEK_SET) < 0) {
    off_t remaining = option_range.start;
    while (remaining != 0 && !feof(f) && !ferror(f)) {
      size_t n = fread(buf, 1, MIN(remaining, BUFSIZ), f);
      remaining -= n;
      if (n == 0) break;
    }
    if (ferror(f)) err(1, "(while seeking) %s", filename);
  }

  off_t count = option_range.end - option_range.start;
  off_t read = 0;
  while (read != count && !feof(f) && !ferror(f)) {
    size_t to_read = option_range.end == -1? BUFSIZ : MIN(count - read, BUFSIZ);
    size_t n = fread(buf, 1, to_read, f);

    for (size_t i = 0; i < n; i += option_columns) {
      u8 *p = &buf[i];

      // Contract repeated identical lines
      if (!first_line && memcmp(p, prev_line, option_columns) == 0) {
        if (!printed_asterisk) {
          printf("%8s\n", "*");
          printed_asterisk = true;
        }
        continue;
      }
      printed_asterisk = false;

      // Offset
      intmax_t offset = option_range.start + read + i;
      printf("%5jx%03jx", offset >> 12, offset & 0xFFF);

      // Print hex area
      const char *prev_fmt = NULL;
      for (size_t j = 0; j < option_columns; j++) {
        if (option_groupsize != 0 && j % option_groupsize == 0) printf(" ");
        if (i + j < n) {
          const char *fmt = format_of(p[j]);
          if (prev_fmt != fmt && option_use_formatting) printf("\x1B[%sm", fmt);
          printf(" %02x", p[j]);
          prev_fmt = fmt;
        } else {
          printf("   ");
        }
      }
      putchar(' ');

      // Print char area
      for (size_t j = 0; j < option_columns; j++) {
        if (option_groupsize != 0 && j % option_groupsize == 0) printf(" ");
        if (i + j < n) {
          const char *fmt = format_of(p[j]);
          if (prev_fmt != fmt && option_use_formatting) printf("\x1B[%sm", fmt);
          if (p[j] >= 0x80) printf("%s", CHAR_AREA_HIGH_LUT[p[j] - 0x80]);
          else putchar(isprint(p[j])? p[j] : '.');
          prev_fmt = fmt;
        } else {
          putchar(' ');
        }
      }
      printf("%s\n", option_use_formatting? "\x1B[m" : "");

      memcpy(prev_line, p, MIN(n - i, option_columns));
      first_line = false;
    }

    read += n;
  }<commit_msg>fix: simplify hexdump loop, solving column misalignment problem<commit_after>#define _POSIX_C_SOURCE 200809L
#include <ctype.h>
#include <err.h>
#include <inttypes.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

#define MIN(X,Y) ((X) < (Y)? (X) : (Y))
typedef uint8_t u8;

//-- Options ----------------
struct offset_range { off_t start, end; };

size_t option_columns      = 16;
size_t option_groupsize    = 8;
bool option_use_formatting = true;
struct offset_range option_range  = { 0, -1 };

const char *formatting_zero      = "38;5;238";
const char *formatting_all       = "38;5;167";
const char *formatting_low       = "38;5;150";
const char *formatting_high      = "38;5;141";
const char *formatting_printable = "";

//-- Hexdump impl -----------
const char *format_of(int v) {
  return v == 0x00?  formatting_zero
       : v == 0xFF?  formatting_all
       : v <  0x20?  formatting_low
       : v >= 0x7F?  formatting_high
       :             formatting_printable;
}

const char *CHAR_AREA_HIGH_LUT[] = {
  "€", ".", "‚", "ƒ", "„", "…", "†", "‡", "ˆ", "‰", "Š", "‹", "Œ", ".", "Ž", ".",
  ".", "‘", "’", "“", "”", "•", "–", "—", "˜", "™", "š", "›", "œ", ".", "ž", "Ÿ",
  ".", "¡", "¢", "£", "¤", "¥", "¦", "§", "¨", "©", "ª", "«", "¬", ".", "®", "¯",
  "°", "±", "²", "³", "´", "µ", "¶", "·", "¸", "¹", "º", "»", "¼", "½", "¾", "¿",
  "À", "Á", "Â", "Ã", "Ä", "Å", "Æ", "Ç", "È", "É", "Ê", "Ë", "Ì", "Í", "Î", "Ï",
  "Ð", "Ñ", "Ò", "Ó", "Ô", "Õ", "Ö", "×", "Ø", "Ù", "Ú", "Û", "Ü", "Ý", "Þ", "ß",
  "à", "á", "â", "ã", "ä", "å", "æ", "ç", "è", "é", "ê", "ë", "ì", "í", "î", "ï",
  "ð", "ñ", "ò", "ó", "ô", "õ", "ö", "÷", "ø", "ù", "ú", "û", "ü", "ý", "þ", "ÿ",
};

void hexdump(FILE *f, const char *filename) {
  u8 line[option_columns];
  u8 prev_line[option_columns];

  bool first_line = true, printed_asterisk = false;

  // Seek to start; fall back to a consuming loop for non-seekable files
  if (fseeko(f, option_range.start, SEEK_SET) < 0) {
    off_t remaining = option_range.start;
    while (remaining != 0 && fgetc(f) != EOF) remaining--;
    if (ferror(f)) err(1, "(while seeking) %s", filename);
  }

  for (off_t offset = option_range.start; offset < option_range.end || option_range.end == -1; offset += option_columns) {
    off_t read = offset - option_range.start;
    size_t n = fread(line, 1, option_columns, f);
    if (n == 0) break;

    // Contract repeated identical lines
    if (!first_line && memcmp(line, prev_line, option_columns) == 0 && n == option_columns) {
      if (!printed_asterisk) {
        printf("%8s\n", "*");
        printed_asterisk = true;
      }
      continue;
    }
    printed_asterisk = false;

    // Offset
    intmax_t offset = option_range.start + read;
    printf("%5jx%03jx", offset >> 12, offset & 0xFFF);

    // Print hex area
    const char *prev_fmt = NULL;
    for (size_t j = 0; j < option_columns; j++) {
      if (option_groupsize != 0 && j % option_groupsize == 0) printf(" ");
      if (j < n) {
        const char *fmt = format_of(line[j]);
        if (prev_fmt != fmt && option_use_formatting) printf("\x1B[%sm", fmt);
        printf(" %02x", line[j]);
        prev_fmt = fmt;
      } else {
        printf("   ");
      }
    }
    putchar(' ');

    // Print char area
    for (size_t j = 0; j < option_columns; j++) {
      if (option_groupsize != 0 && j % option_groupsize == 0) printf(" ");
      if (j < n) {
        const char *fmt = format_of(line[j]);
        if (prev_fmt != fmt && option_use_formatting) printf("\x1B[%sm", fmt);
        if (line[j] >= 0x80) printf("%s", CHAR_AREA_HIGH_LUT[line[j] - 0x80]);
        else putchar(isprint(line[j])? line[j] : '.');
        prev_fmt = fmt;
      } else {
        putchar(' ');
      }
    }
    printf("%s\n", option_use_formatting? "\x1B[m" : "");

    memcpy(prev_line, line, n);
    first_line = false;
    if (n < option_columns) break;
  }
{'fraction_non_alphanumeric': 0.24167283493708364, 'fraction_numerical': 0.01998519615099926, 'mean_word_length': 2.6733121884911646, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3879815', 'n_tokens_mistral': 3366, 'n_tokens_neox': 3295, 'n_words': 845}