<commit_before>#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <sys/mman.h>
#include <stdint.h>
#include <string.h>
#include "bsdconv.h"

struct m_data_s{
	unsigned char *data;
	size_t len;
	struct data_s *next;

	uintptr_t p;
	unsigned char *dp;
	struct m_data_s *n;
};

struct m_state_s{
	char status;
	struct state_s *sub[257];

	uintptr_t data;

	struct m_state_s *psub[257];
	uintptr_t p;
	struct m_state_s *n;
	int child;
};

struct list{
	struct m_state_s *p;
	int u,l;
	struct list *n;
};

unsigned char table[256]={};

uintptr_t offset=0;

int main(int argc, char *argv[]){
	int i, j, k, l, c=0, cu, cl;
	FILE *fp;
	unsigned char inbuf[1024], *f, *t, dat[256], *tmp, *of, *ot;
	struct m_data_s *data_r=NULL, *data_p=NULL, *data_t=NULL;
	struct m_state_s *state_r, *state_t, holder;
	struct list *todo=NULL, *newtodo, *newtodo_tail, *state_p;
	struct state_s dstate;
	struct data_s ddata;
	uintptr_t callback=0;
	void *tofree;

	printf("Making table %s\n", argv[1]);

	table['0']=0;
	table['1']=1;
	table['2']=2;
	table['3']=3;
	table['4']=4;
	table['5']=5;
	table['6']=6;
	table['7']=7;
	table['8']=8;
	table['9']=9;
	table['a']=10;
	table['A']=10;
	table['b']=11;
	table['B']=11;
	table['c']=12;
	table['C']=12;
	table['d']=13;
	table['D']=13;
	table['e']=14;
	table['E']=14;
	table['f']=15;
	table['F']=15;
	fp=fopen(argv[1], "r");

	newtodo=malloc(sizeof(struct list));
	newtodo->n=NULL;
	newtodo->l=3;
	newtodo->u=2;
	newtodo_tail=newtodo;

	state_t=state_r=(struct m_state_s *)malloc(sizeof(struct m_state_s));
	state_t->status=DEADEND;
	state_t->data=0;
	state_t->p=offset;
	state_t->n=NULL;
	offset+=sizeof(struct state_s);
	for(i=0;i<257;i++){
		state_r->sub[i]=0;
		state_r->psub[i]=NULL;
	}

	holder.psub[0]=state_r;

	while(fgets((char *)inbuf, 1024, fp)){
		if(inbuf[0]=='#') continue;
		tmp=inbuf;
		f=of=(unsigned char *)strsep((char **)&tmp, "\t ");
		while(index("\t ",*tmp)){
			tmp++;
		}
		t=ot=(unsigned char *)strsep((char **)&tmp, "\t\r\n# ");

		todo=malloc(sizeof(struct list));
		todo->n=NULL;
		todo->p=&holder;
		todo->u=0;
		todo->l=0;

		while(*f){
			if(*f=='*'){
				cl=0;
				cu=255;
			}else if(*f==','){
				cu=cl=256;
			}else{
				cl=table[*f];
				++f;
				cl*=16;
				cl+=table[*f];
				cu=cl;
			}
			state_p=todo;
			while(todo){
				state_p=todo;
				for(c=state_p->l;c<=state_p->u;c++){
					if(state_p->p->psub[c]){<commit_msg>support case insensitive; allow empty data.<commit_after>#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <sys/mman.h>
#include <stdint.h>
#include <string.h>
#include "bsdconv.h"

struct m_data_s{
	unsigned char *data;
	size_t len;
	struct data_s *next;

	uintptr_t p;
	unsigned char *dp;
	struct m_data_s *n;
};

struct m_state_s{
	char status;
	struct state_s *sub[257];

	uintptr_t data;

	struct m_state_s *psub[257];
	uintptr_t p;
	struct m_state_s *n;
	int child;
};

struct list{
	struct m_state_s *p;
	int u,l;
	struct list *n;
};

unsigned char table[256]={};
unsigned char ci_table[256]={0};

uintptr_t offset=0;

int main(int argc, char *argv[]){
	int i, j, k, l, c=0, cu, cl, ci;
	FILE *fp;
	unsigned char inbuf[1024], *f, *t, dat[256], *tmp, *of, *ot;
	struct m_data_s *data_r=NULL, *data_p=NULL, *data_t=NULL;
	struct m_state_s *state_r, *state_t, holder;
	struct list *todo=NULL, *newtodo, *newtodo_tail, *state_p;
	struct state_s dstate;
	struct data_s ddata;
	uintptr_t callback=0;
	void *tofree;

	table['0']=0;
	table['1']=1;
	table['2']=2;
	table['3']=3;
	table['4']=4;
	table['5']=5;
	table['6']=6;
	table['7']=7;
	table['8']=8;
	table['9']=9;
	table['a']=10;
	table['A']=10;
	table['b']=11;
	table['B']=11;
	table['c']=12;
	table['C']=12;
	table['d']=13;
	table['D']=13;
	table['e']=14;
	table['E']=14;
	table['f']=15;
	table['F']=15;
	ci_table['A']='a';
	ci_table['B']='b';
	ci_table['C']='c';
	ci_table['D']='d';
	ci_table['E']='e';
	ci_table['F']='f';
	ci_table['G']='g';
	ci_table['H']='h';
	ci_table['I']='i';
	ci_table['J']='j';
	ci_table['K']='k';
	ci_table['L']='l';
	ci_table['M']='m';
	ci_table['N']='n';
	ci_table['O']='o';
	ci_table['P']='p';
	ci_table['Q']='q';
	ci_table['R']='r';
	ci_table['S']='s';
	ci_table['T']='t';
	ci_table['U']='u';
	ci_table['V']='v';
	ci_table['W']='w';
	ci_table['X']='x';
	ci_table['Y']='y';
	ci_table['Z']='z';
	ci_table['a']='A';
	ci_table['b']='B';
	ci_table['c']='C';
	ci_table['d']='D';
	ci_table['e']='E';
	ci_table['f']='F';
	ci_table['g']='G';
	ci_table['h']='H';
	ci_table['i']='I';
	ci_table['j']='J';
	ci_table['k']='K';
	ci_table['l']='L';
	ci_table['m']='M';
	ci_table['n']='N';
	ci_table['o']='O';
	ci_table['p']='P';
	ci_table['q']='Q';
	ci_table['r']='R';
	ci_table['s']='S';
	ci_table['t']='T';
	ci_table['u']='U';
	ci_table['v']='V';
	ci_table['w']='W';
	ci_table['x']='X';
	ci_table['y']='Y';
	ci_table['z']='Z';

	printf("Making table %s\n", argv[1]);

	fp=fopen(argv[1], "r");

	newtodo=malloc(sizeof(struct list));
	newtodo->n=NULL;
	newtodo->l=3;
	newtodo->u=2;
	newtodo_tail=newtodo;

	state_t=state_r=(struct m_state_s *)malloc(sizeof(struct m_state_s));
	state_t->status=DEADEND;
	state_t->data=0;
	state_t->p=offset;
	state_t->n=NULL;
	offset+=sizeof(struct state_s);
	for(i=0;i<257;i++){
		state_r->sub[i]=0;
		state_r->psub[i]=NULL;
	}

	holder.psub[0]=state_r;

	while(fgets((char *)inbuf, 1024, fp)){
		if(inbuf[0]=='#') continue;
		tmp=inbuf;
		f=of=(unsigned char *)strsep((char **)&tmp, "\t ");
		while(index("\t ",*tmp)){
			tmp++;
		}
		if(*tmp){
			t=ot=(unsigned char *)strsep((char **)&tmp, "\t\r\n# ");
		}else{
			t=ot=NULL;
		}

		todo=malloc(sizeof(struct list));
		todo->n=NULL;
		todo->p=&holder;
		todo->u=0;
		todo->l=0;

		if(*f=='!'){
			ci=1;
			++f;
		}else{
			ci=0;
		}
		while(*f){
			if(*f=='*'){
				cl=0;
				cu=255;
			}else if(*f==','){
				cu=cl=256;
			}else{
				cl=table[*f];
				++f;
				cl*=16;
				cl+=table[*f];
				cu=cl;
			}
			state_p=todo;
			while(todo){
				state_p=todo;
				for(c=0;c<256;c++){
					if(!((c>=state_p->l && c<=state_p->u) || (ci && ci_table[c] && (ci_table[c]>=state_p->l && ci_table[c]<=state_p->u)))){
						continue;
					}
					if(state_p->p->psub[c]){
{'fraction_non_alphanumeric': 0.25866751877296695, 'fraction_numerical': 0.030356286946796613, 'mean_word_length': 4.961904761904762, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1267466', 'n_tokens_mistral': 3331, 'n_tokens_neox': 3060, 'n_words': 503}