<commit_before>#define _GNU_SOURCE

#include <stdarg.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>

#include <errno.h>
#include <string.h>
#include <wchar.h>

#include "user.h"
#include "data.h"

Result result_new(bool success, void *data, const wchar_t *fmt, ...) {
  va_list args;
  Result ret;

  ret.success = success;
  ret.data = data;

  va_start(args, fmt);
  vswprintf(ret.msg, ERR_MAX_LEN, fmt, args);
  va_end(args);

  return ret;
}

Result entry_new(int length) {
  Entry *new;
  
  if (!(new = malloc(sizeof(Entry))))
    return result_new(false, NULL, L"Couldn't allocate Entry");
  bzero(new, sizeof(Entry));

  new->text = calloc(length + 1, sizeof(wchar_t));
  if (!new->text) {
    free(new);
    return result_new(false, NULL, L"Couldn't allocate Entry text buffer");
  }
  bzero(new->text, (length + 1) * sizeof(wchar_t));
  new->length = length;
  new->size = length + 1;

  return result_new(true, new, L"Allocated new Entry with %d text buffer", length);
}

Result data_load(FILE *input) {
  Result ret, res;
  Entry *new, *r, *c;
  wchar_t *line;
  int line_nr, current_level;

  if (!(line = calloc(LINE_MAX_LEN, sizeof(wchar_t))))
    return result_new(false, NULL, L"Couldn't allocate line buffer");
  new = r = c = NULL;
  current_level = 0;
  line_nr = 1;
  errno = 0;

  while (fgetws(line, LINE_MAX_LEN, input)) {
    int length = wcslen(line);
    if (length <= 1) continue;  // ignore empty lines
    line[length-1] = L'\0';     // kill newline char
    --length;

    int level = 0;
    while ((line[level] == L'\t') && (level < length))
      level++;
    if (level >= (length - 2)) {
      ret = result_new(false, NULL, L"Malformed input at line %d", line_nr);
      goto error;
    }
    if (!((line[level] == L'-') && (line[level+1] == L' '))) {
      ret = result_new(false, NULL, L"Malformed input at line %d", line_nr);
      goto error;
    }

    res = entry_new(length);
    if (!res.success) {
      ret = res;
      goto error;
    }

    if (!c)
      r = c = (Entry *)res.data;
    else {
      new = (Entry *)res.data;

      if (current_level < level) {
        if (level - current_level != 1) {
          ret = result_new(false, NULL, L"Ambigous indentation at line %d", line_nr);
          goto error;
        }
        new->parent = c;
        c->child = new;
        current_level = level;
      } else {
        while (c->parent && (current_level != level)) {
          c = c->parent;
          --current_level;
        }
        if (current_level != level) {
          ret = result_new(false, NULL, L"Couldn't find parent at line %d", line_nr);
          goto error;
        }
        new->parent = c->parent;
        new->prev = c;
        c->next = new;
      }
      c = new;
    }

    if (length - level - 2 >= 4) {
      if ((line[level+2] == L'~') && (line[level+3] == L'~') &&
          (line[length-1] == L'~') && (line[length-2] == L'~')) {
        c->crossed = true;
        line[length-2] = L'\0';
      }
    }

    c->length = length - level - (c->crossed * 4) - 2;
    wcscpy(c->text, line+level+(2*c->crossed)+2);

    ++line_nr;
  }
  free(line);

  if (errno)
    ret = result_new(false, NULL, L"File access error at line %d", line_nr);
  else
    return result_new(true, r, L"Parsed %d lines", line_nr);

error:
  free(line);
  if (new)  free(new);
  if (r)    data_unload(r);
  return ret;
}

void data_unload(Entry *e) {
  if (e->child)
    data_unload(e->child);
  if (e->next)
    data_unload(e->next);

  free(e->text);<commit_msg>Fix error handling<commit_after>#define _GNU_SOURCE

#include <stdarg.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>

#include <errno.h>
#include <string.h>
#include <wchar.h>

#include "user.h"
#include "data.h"

Result result_new(bool success, void *data, const wchar_t *fmt, ...) {
  va_list args;
  Result ret;

  ret.success = success;
  ret.data = data;

  va_start(args, fmt);
  vswprintf(ret.msg, ERR_MAX_LEN, fmt, args);
  va_end(args);

  return ret;
}

Result entry_new(int length) {
  Entry *new;
  
  if (!(new = malloc(sizeof(Entry))))
    return result_new(false, NULL, L"Couldn't allocate Entry");
  bzero(new, sizeof(Entry));

  new->text = calloc(length + 1, sizeof(wchar_t));
  if (!new->text) {
    free(new);
    return result_new(false, NULL, L"Couldn't allocate Entry text buffer");
  }
  bzero(new->text, (length + 1) * sizeof(wchar_t));
  new->length = length;
  new->size = length + 1;

  return result_new(true, new, L"Allocated new Entry with %d text buffer", length);
}

Result data_load(FILE *input) {
  Result ret, res;
  Entry *new, *r, *c;
  wchar_t *line;
  int line_nr, current_level;

  if (!(line = calloc(LINE_MAX_LEN, sizeof(wchar_t))))
    return result_new(false, NULL, L"Couldn't allocate line buffer");
  new = r = c = NULL;
  current_level = 0;
  line_nr = 1;
  errno = 0;

  while (fgetws(line, LINE_MAX_LEN, input)) {
    new = NULL;
    int length = wcslen(line);
    if (length <= 1) continue;  // ignore empty lines
    line[length-1] = L'\0';     // kill newline char
    --length;

    int level = 0;
    while ((line[level] == L'\t') && (level < length))
      level++;
    if (level >= (length - 2)) {
      ret = result_new(false, NULL, L"Malformed input at line %d", line_nr);
      goto error;
    }
    if (!((line[level] == L'-') && (line[level+1] == L' '))) {
      ret = result_new(false, NULL, L"Malformed input at line %d", line_nr);
      goto error;
    }

    res = entry_new(length);
    if (!res.success) {
      ret = res;
      goto error;
    }

    if (!c)
      r = c = (Entry *)res.data;
    else {
      new = (Entry *)res.data;

      if (current_level < level) {
        if (level - current_level != 1) {
          ret = result_new(false, NULL, L"Ambigous indentation at line %d", line_nr);
          goto error;
        }
        new->parent = c;
        c->child = new;
        current_level = level;
      } else {
        while (c->parent && (current_level != level)) {
          c = c->parent;
          --current_level;
        }
        if (current_level != level) {
          ret = result_new(false, NULL, L"Couldn't find parent at line %d", line_nr);
          goto error;
        }
        new->parent = c->parent;
        new->prev = c;
        c->next = new;
      }
      c = new;
    }

    if (length - level - 2 >= 4) {
      if ((line[level+2] == L'~') && (line[level+3] == L'~') &&
          (line[length-1] == L'~') && (line[length-2] == L'~')) {
        c->crossed = true;
        line[length-2] = L'\0';
      }
    }

    c->length = length - level - (c->crossed * 4) - 2;
    wcscpy(c->text, line+level+(2*c->crossed)+2);

    ++line_nr;
  }
  free(line);

  if (errno)
    ret = result_new(false, NULL, L"File access error at line %d", line_nr);
  else
    return result_new(true, r, L"Parsed %d lines", line_nr);

error:
  free(line);
  if (new) free(new);
  if (r) data_unload(r);
  return ret;
}

void data_unload(Entry *e) {
  if (e->child)
    data_unload(e->child);
  if (e->next)
    data_unload(e->next);

  free(e->text);
{'fraction_non_alphanumeric': 0.17102473498233214, 'fraction_numerical': 0.007067137809187279, 'mean_word_length': 2.509920634920635, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 67, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5230409', 'n_tokens_mistral': 2695, 'n_tokens_neox': 2608, 'n_words': 669}