<commit_before>#include "gap.h"

#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>

#include "util.h"

#define GAPSIZE 1024

gapbuf_t *gb_create(void) {
  gapbuf_t *gb = malloc(sizeof(*gb));
  if (!gb) {
    return NULL;
  }

  gb->bufstart = malloc(1 + GAPSIZE);
  if (!gb->bufstart) {
    free(gb);
    return NULL;
  }

  gb->gapstart = gb->bufstart;
  gb->gapend = gb->gapstart + GAPSIZE;
  gb->bufend = gb->bufstart + GAPSIZE + 1;
  gb->bufend[-1] = '\n';

  gb->lines = intbuf_create(10);
  intbuf_add(gb->lines, 0);
  return gb;
}

gapbuf_t *gb_load(FILE *fp) {
  gapbuf_t *gb = malloc(sizeof(*gb));
  if (!gb) {
    return NULL;
  }

  struct stat info;
  fstat(fileno(fp), &info);
  int filesize = info.st_size;
  int bufsize = filesize + GAPSIZE;

  gb->bufstart = malloc(bufsize);
  if (!gb->bufstart) {
    free(gb);
    return NULL;
  }

  gb->gapstart = gb->bufstart;
  gb->gapend = gb->bufstart + GAPSIZE;
  gb->bufend = gb->bufstart + bufsize;

  fread(gb->gapend, 1, filesize, fp);

  gb->lines = intbuf_create(10);
  int last = -1;
  for (int i = 0; i < filesize; ++i) {
    if (gb->gapend[i] == '\n') {
      intbuf_add(gb->lines, i - last - 1);
      last = i;
    }
  }

  return gb;
}

void gb_free(gapbuf_t *gb) {
  free(gb->bufstart);
  intbuf_free(gb->lines);
}

int gb_size(gapbuf_t *gb) {
  return (gb->gapstart - gb->bufstart) + (gb->bufend - gb->gapend);
}

int gb_nlines(gapbuf_t *gb) {
  return gb->lines->len;
}

void gb_save(gapbuf_t *gb, FILE *fp) {
  fwrite(gb->bufstart, 1, gb->gapstart - gb->bufstart, fp);
  fwrite(gb->gapend, 1, gb->bufend - gb->gapend, fp);
}

// Returns the real index of the logical offset pos.
static int gb_index(gapbuf_t *gb, int pos) {
  int gapsize = gb->gapend - gb->gapstart;
  return gb->bufstart + pos < gb->gapstart ? pos : pos + gapsize;
}

char gb_getchar(gapbuf_t *gb, int pos) {
  return gb->bufstart[gb_index(gb, pos)];
}

// Moves the gap so that gb->bufstart + pos == gb->gapstart.
static void gb_mvgap(gapbuf_t *gb, int pos) {
  char *point = gb->bufstart + gb_index(gb, pos);
  if (gb->gapend <= point) {
    int n = point - gb->gapend;
    memcpy(gb->gapstart, gb->gapend, n);
    gb->gapstart += n;
    gb->gapend += n;
  } else if (point < gb->gapstart) {
    int n = gb->gapstart - point;
    memcpy(gb->gapend - n, point, n);
    gb->gapstart -= n;
    gb->gapend -= n;
  }
}

// Ensure the gap fits at least n new characters.
static void gb_growgap(gapbuf_t *gb, int n) {
  int gapsize = gb->gapend - gb->gapstart;
  if (n <= gapsize) {
    return;
  }

  int newgapsize = 0;
  while (newgapsize < 2*n) {
    newgapsize += GAPSIZE;
  }
  // Pointers will be obsoleted so remember offsets...
  int leftsize = gb->gapstart - gb->bufstart;
  int rightsize = gb->bufend - gb->gapend;

  int newsize = leftsize + rightsize + newgapsize;

  gb->bufstart = realloc(gb->bufstart, newsize);
  gb->gapstart = gb->bufstart + leftsize;
  gb->gapend = gb->gapstart + newgapsize;
  // Move the bit after the gap right by gapsize positions...
  memcpy(gb->gapend, gb->gapstart + gapsize, rightsize);
  gb->bufend = gb->bufstart + newsize;
}

void gb_putchar(gapbuf_t *gb, char c, int pos) {
  gb_putstring(gb, &c, 1, pos);
}

void gb_putstring(gapbuf_t *gb, char *buf, int n, int pos) {
  gb_growgap(gb, n);
  gb_mvgap(gb, pos);
  memcpy(gb->gapstart, buf, n);

  int line, col;
  gb_pos_to_linecol(gb, pos, &line, &col);

  // TODO(isbadawi): Revisit this with n > 1...
  for (int i = 0; i < n; ++i) {
    if (gb->gapstart[i] == '\n') {
      int oldlen = gb->lines->buf[line];
      gb->lines->buf[line] = col + i;
      intbuf_insert(gb->lines, oldlen - col, ++line);
      col = 0;
    } else {
      gb->lines->buf[line]++;
    }
  }

  gb->gapstart += n;
}

void gb_del(gapbuf_t *gb, int n, int pos) {
  gb_mvgap(gb, pos);

  int line, col;
  gb_pos_to_linecol(gb, pos, &line, &col);

  for (int i = 0; i < n; ++i) {
    if (gb->gapstart[-i - 1] == '\n') {
      gb->lines->buf[line - 1] += gb->lines->buf[line];
      intbuf_remove(gb->lines, line);
      line--;
    } else {
      gb->lines->buf[line]--;
    }
  }
<commit_msg>Correctly track line lengths across inserts.<commit_after>#include "gap.h"

#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>

#include "util.h"

#define GAPSIZE 1024

gapbuf_t *gb_create(void) {
  gapbuf_t *gb = malloc(sizeof(*gb));
  if (!gb) {
    return NULL;
  }

  gb->bufstart = malloc(1 + GAPSIZE);
  if (!gb->bufstart) {
    free(gb);
    return NULL;
  }

  gb->gapstart = gb->bufstart;
  gb->gapend = gb->gapstart + GAPSIZE;
  gb->bufend = gb->bufstart + GAPSIZE + 1;
  gb->bufend[-1] = '\n';

  gb->lines = intbuf_create(10);
  intbuf_add(gb->lines, 0);
  return gb;
}

gapbuf_t *gb_load(FILE *fp) {
  gapbuf_t *gb = malloc(sizeof(*gb));
  if (!gb) {
    return NULL;
  }

  struct stat info;
  fstat(fileno(fp), &info);
  int filesize = info.st_size;
  int bufsize = filesize + GAPSIZE;

  gb->bufstart = malloc(bufsize);
  if (!gb->bufstart) {
    free(gb);
    return NULL;
  }

  gb->gapstart = gb->bufstart;
  gb->gapend = gb->bufstart + GAPSIZE;
  gb->bufend = gb->bufstart + bufsize;

  fread(gb->gapend, 1, filesize, fp);

  gb->lines = intbuf_create(10);
  int last = -1;
  for (int i = 0; i < filesize; ++i) {
    if (gb->gapend[i] == '\n') {
      intbuf_add(gb->lines, i - last - 1);
      last = i;
    }
  }

  return gb;
}

void gb_free(gapbuf_t *gb) {
  free(gb->bufstart);
  intbuf_free(gb->lines);
}

int gb_size(gapbuf_t *gb) {
  return (gb->gapstart - gb->bufstart) + (gb->bufend - gb->gapend);
}

int gb_nlines(gapbuf_t *gb) {
  return gb->lines->len;
}

void gb_save(gapbuf_t *gb, FILE *fp) {
  fwrite(gb->bufstart, 1, gb->gapstart - gb->bufstart, fp);
  fwrite(gb->gapend, 1, gb->bufend - gb->gapend, fp);
}

// Returns the real index of the logical offset pos.
static int gb_index(gapbuf_t *gb, int pos) {
  int gapsize = gb->gapend - gb->gapstart;
  return gb->bufstart + pos < gb->gapstart ? pos : pos + gapsize;
}

char gb_getchar(gapbuf_t *gb, int pos) {
  return gb->bufstart[gb_index(gb, pos)];
}

// Moves the gap so that gb->bufstart + pos == gb->gapstart.
static void gb_mvgap(gapbuf_t *gb, int pos) {
  char *point = gb->bufstart + gb_index(gb, pos);
  if (gb->gapend <= point) {
    int n = point - gb->gapend;
    memcpy(gb->gapstart, gb->gapend, n);
    gb->gapstart += n;
    gb->gapend += n;
  } else if (point < gb->gapstart) {
    int n = gb->gapstart - point;
    memcpy(gb->gapend - n, point, n);
    gb->gapstart -= n;
    gb->gapend -= n;
  }
}

// Ensure the gap fits at least n new characters.
static void gb_growgap(gapbuf_t *gb, int n) {
  int gapsize = gb->gapend - gb->gapstart;
  if (n <= gapsize) {
    return;
  }

  int newgapsize = 0;
  while (newgapsize < 2*n) {
    newgapsize += GAPSIZE;
  }
  // Pointers will be obsoleted so remember offsets...
  int leftsize = gb->gapstart - gb->bufstart;
  int rightsize = gb->bufend - gb->gapend;

  int newsize = leftsize + rightsize + newgapsize;

  gb->bufstart = realloc(gb->bufstart, newsize);
  gb->gapstart = gb->bufstart + leftsize;
  gb->gapend = gb->gapstart + newgapsize;
  // Move the bit after the gap right by gapsize positions...
  memcpy(gb->gapend, gb->gapstart + gapsize, rightsize);
  gb->bufend = gb->bufstart + newsize;
}

void gb_putchar(gapbuf_t *gb, char c, int pos) {
  gb_putstring(gb, &c, 1, pos);
}

void gb_putstring(gapbuf_t *gb, char *buf, int n, int pos) {
  gb_growgap(gb, n);
  gb_mvgap(gb, pos);
  memcpy(gb->gapstart, buf, n);

  int line, col;
  gb_pos_to_linecol(gb, pos, &line, &col);

  // Adjust the line lengths.

  // Where we started inserting on this line
  int start = col;
  // Characters since last newline
  int last = 0;
  for (int i = 0; i < n; ++i) {
    if (gb->gapstart[i] == '\n') {
      int oldlen = gb->lines->buf[line];
      gb->lines->buf[line] = start + last;
      intbuf_insert(gb->lines, oldlen - (start + last), ++line);
      start = last = 0;
    } else {
      gb->lines->buf[line]++;
      last++;
    }
  }
  gb->gapstart += n;
}

void gb_del(gapbuf_t *gb, int n, int pos) {
  gb_mvgap(gb, pos);

  int line, col;
  gb_pos_to_linecol(gb, pos, &line, &col);

  for (int i = 0; i < n; ++i) {
    if (gb->gapstart[-i - 1] == '\n') {
      gb->lines->buf[line - 1] += gb->lines->buf[line];
      intbuf_remove(gb->lines, line);
      line--;
    } else {
      gb->lines->buf[line]--;
    }
  }

{'fraction_non_alphanumeric': 0.17783597190810618, 'fraction_numerical': 0.006427806213546007, 'mean_word_length': 2.8862164662349676, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 190, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5634817', 'n_tokens_mistral': 3751, 'n_tokens_neox': 3476, 'n_words': 833}