<commit_before>/* $Id$ */

#include <stdlib.h> /*calloc,free*/
#include <fcntl.h>  /*open,close*/
#include UNISTD_H   /*open,read,close*/
#include <stdio.h>  /*fprintf,stderr*/
#include <string.h> /*strerror,strncpy,strrchr*/
#include <errno.h>
#include EXPAT_H
#include "xmlc.h" /*xmlc_white_space*/
#include "rn.h"
#include "rnc.h"
#include "rnd.h"
#include "rnx.h"
#include "drv.h"
#include "rx.h"
#include "ll.h"

#define LEN_T RNV_LEN_T
#define LIM_T RNV_LIM_T

#define RNVER_ELEM 0
#define RNVER_AKEY 1
#define RNVER_AVAL 2
#define RNVER_EMIS 3
#define RNVER_AMIS 4
#define RNVER_UFIN 5
#define RNVER_TEXT 6
#define RNVER_MIXT 7

static char *xml;
static XML_Parser expat;
static int start,current,previous;
static int mixed=0;
static int errors;
static int explain=1;

/* Expat does not normalize strings on input unless the whole file is loaded into the buffer */
static char *text; static int len_t;
static int n_t;

static char *suri=NULL,*sname; static int len_suri=-1; /* qname() splits, handlers use */

static void init(void) {
  rn_init();
  rnc_init();
  rnd_init();
  rnx_init();
  drv_init();
  text=(char*)calloc(len_t=LEN_T,sizeof(char));
  text[n_t=0]='\0';
}

static int load_rnc(char *fn) {
  struct rnc_source *sp=rnc_alloc();
  if(rnc_open(sp,fn)!=-1) start=rnc_parse(sp); rnc_close(sp); 
  {int errors=rnc_errors(sp); rnc_free(sp); if(errors) return 0;}
  
  rnd_deref(start); if(rnd_errors()) return 0;
  rnd_restrictions(); if(rnd_errors()) return 0;
  rnd_traits();
  start=rnd_release(); 

  start=rn_compress_last(start);

  return 1;
}

#define err(msg) vfprintf(stderr,msg,ap);

static void error(int erno,...) {
  char *s;
  va_list ap;
  ++errors;
  fprintf(stderr,"invalid document (%s,%i,%i): ",xml,XML_GetCurrentLineNumber(expat),XML_GetCurrentColumnNumber(expat));
  va_start(ap,erno);
  switch(erno) {
  case RNVER_ELEM: err("element '%s^%s' not allowed"); break;
  case RNVER_AKEY: err("attribute '%s^%s' not allowed"); break;
  case RNVER_AVAL: err("attribute '%s^%s with invalid value \"%s\"'"); break;
  case RNVER_EMIS: err("incomplete content"); break;
  case RNVER_AMIS: err("missing attributes of '%s^%s'"); break;
  case RNVER_UFIN: err("unfinished content"); break;
  case RNVER_TEXT: err("invalid text or data"); break;
  case RNVER_MIXT: err("text not allowed"); break;
  default: assert(0);
  }                
  va_end(ap);
  if(explain) {
    rnx_expected(previous);
    if(rnx_n_exp!=0) {
      int i;
      fprintf(stderr,"\nexpected:");
      for(i=0;i!=rnx_n_exp;++i) {
	fprintf(stderr,"\n\t%s",s=rnx_p2str(rnx_exp[i]));
	free(s);
      }
    }
  }<commit_msg>*** empty log message ***<commit_after>/* $Id$ */

#include <stdlib.h> /*calloc,free*/
#include <fcntl.h>  /*open,close*/
#include UNISTD_H   /*open,read,close*/
#include <stdio.h>  /*fprintf,stderr*/
#include <string.h> /*strerror,strncpy,strrchr*/
#include <errno.h>
#include EXPAT_H
#include "xmlc.h" /*xmlc_white_space*/
#include "rn.h"
#include "rnc.h"
#include "rnd.h"
#include "rnx.h"
#include "drv.h"
#include "rx.h"
#include "ll.h"

#define LEN_T RNV_LEN_T
#define LIM_T RNV_LIM_T

#define RNVER_ELEM 0
#define RNVER_AKEY 1
#define RNVER_AVAL 2
#define RNVER_EMIS 3
#define RNVER_AMIS 4
#define RNVER_UFIN 5
#define RNVER_TEXT 6
#define RNVER_MIXT 7

static char *xml;
static XML_Parser expat;
static int start,current,previous;
static int mixed=0;
static int errors;
static int explain=1;

/* Expat does not normalize strings on input unless the whole file is loaded into the buffer */
static char *text; static int len_t;
static int n_t;

static char *suri=NULL,*sname; static int len_suri=-1; /* qname() splits, handlers use */

static void init(void) {
  rn_init();
  rnc_init();
  rnd_init();
  rnx_init();
  drv_init();
  text=(char*)calloc(len_t=LEN_T,sizeof(char));
  text[n_t=0]='\0';
}

static int load_rnc(char *fn) {
  struct rnc_source *sp=rnc_alloc();
  if(rnc_open(sp,fn)!=-1) start=rnc_parse(sp); rnc_close(sp); 
  {int errors=rnc_errors(sp); rnc_free(sp); if(errors) return 0;}
  
  rnd_deref(start); if(rnd_errors()) return 0;
  rnd_restrictions(); if(rnd_errors()) return 0;
  rnd_traits();
  start=rnd_release(); 

  start=rn_compress_last(start);

  return 1;
}

#define err(msg) vfprintf(stderr,msg,ap);

static void error(int erno,...) {
  char *s;
  va_list ap;
  ++errors;
  fprintf(stderr,"invalid document (%s,%i,%i): ",xml,XML_GetCurrentLineNumber(expat),XML_GetCurrentColumnNumber(expat));
  va_start(ap,erno);
  switch(erno) {
  case RNVER_ELEM: err("element %s^%s not allowed"); break;
  case RNVER_AKEY: err("attribute %s^%s not allowed"); break;
  case RNVER_AVAL: err("attribute %s^%s with invalid value \"%s\""); break;
  case RNVER_EMIS: err("incomplete content"); break;
  case RNVER_AMIS: err("missing attributes of %s^%s"); break;
  case RNVER_UFIN: err("unfinished content"); break;
  case RNVER_TEXT: err("invalid text or data"); break;
  case RNVER_MIXT: err("text not allowed"); break;
  default: assert(0);
  }                
  va_end(ap);
  if(explain) {
    rnx_expected(previous);
    if(rnx_n_exp!=0) {
      int i;
      fprintf(stderr,"\nexpected:");
      for(i=0;i!=rnx_n_exp;++i) {
	fprintf(stderr,"\n\t%s",s=rnx_p2str(rnx_exp[i]));
	free(s);
      }
    }
  }
{'fraction_non_alphanumeric': 0.1636779855017169, 'fraction_numerical': 0.008393742846241892, 'mean_word_length': 4.832035595105673, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6547647', 'n_tokens_mistral': 2181, 'n_tokens_neox': 2091, 'n_words': 490}