<commit_before>/*
 * Some code and table come from http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
 * Copyright (c) 2009-2011 <NAME> <<EMAIL>>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF MIND, USE, DATA OR PROFITS, WHETHER
 * IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
 * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

#include "../../src/bsdconv.h"
#define HASHKEY "WHITESPACE"

struct my_s{
	struct data_rt *queue;
	struct data_rt **last;
	struct data_rt **dlast;
	struct bsdconv_phase *rerail;
	size_t offsetA;
	size_t offsetB;
};

int cbcreate(struct bsdconv_instance *ins, struct bsdconv_hash_entry *arg){
	struct my_s *t;
	if(bsdconv_hash_has(ins, HASHKEY)){
		t=bsdconv_hash_get(ins, HASHKEY);
	}else{
		t=malloc(sizeof(struct my_s));
		bsdconv_hash_set(ins, HASHKEY, t);
	}
	t->queue=NULL;
	CURRENT_CODEC(ins)->priv=t;
	return 0;
}

void cbinit(struct bsdconv_instance *ins){
	struct my_s *t=CURRENT_CODEC(ins)->priv;
	t->offsetA=0;
	t->offsetB=0;
	t->last=&t->queue;
	t->dlast=NULL;
	struct data_rt *q;
	while(t->queue){
		DATA_FREE((struct data_rt *)t->queue->data);
		q=t->queue;
		t->queue=t->queue->next;
		DATA_FREE(q);
	}
}

void cbdestroy(struct bsdconv_instance *ins){
	struct my_s *t=CURRENT_CODEC(ins)->priv;
	struct data_rt *q;
	if(bsdconv_hash_has(ins, HASHKEY)){
		while(t->queue){
			DATA_FREE((struct data_rt *)t->queue->data);
			q=t->queue;
			t->queue=t->queue->next;
			DATA_FREE(q);
		}
		free(t);
		bsdconv_hash_del(ins, HASHKEY);
	}
}

void cbconv(struct bsdconv_instance *ins){
	unsigned char *data;
	struct bsdconv_phase *this_phase=CURRENT_PHASE(ins);
	struct my_s *t=CURRENT_CODEC(ins)->priv;
	struct data_rt *q;
	data=this_phase->curr->data;
	int i;
	int ucs=0;

	this_phase->state.status=NEXTPHASE;

	if(this_phase->curr->len>0 && data[0]==0x1){
		for(i=1;i<this_phase->curr->len;++i){
			ucs<<=8;
			ucs|=data[i];
		}
		if(ucs==0x09||ucs==0x0A||ucs==0x0D||ucs==0x20){
			DATA_MALLOC(q);
			*(t->last)=q;
			q->next=NULL;
			q->flags=0;
			t->last=&q->next;
			DATA_MALLOC(q->data);
			if(t->dlast!=NULL)
				*(t->dlast)=q->data;
			t->dlast=&(((struct data_rt *)q->data)->next);
			*((struct data_rt *)q->data)=*(this_phase->curr);
			((struct data_rt *)q->data)->next=NULL;
			this_phase->curr->flags &= ~F_FREE;
			q->len=t->offsetA;

			t->rerail->flags |= (F_MATCH | F_PENDING);
			t->rerail->match_data=t->queue->data;

			return;
		}
	}
	t->offsetA+=1;

	DATA_MALLOC(this_phase->data_tail->next);
	this_phase->data_tail=this_phase->data_tail->next;<commit_msg>fix memory issue happen in exceptional case in inter/WHITESPACE-DERAIL<commit_after>/*
 * Some code and table come from http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
 * Copyright (c) 2009-2011 <NAME> <<EMAIL>>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF MIND, USE, DATA OR PROFITS, WHETHER
 * IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
 * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

#include "../../src/bsdconv.h"
#define HASHKEY "WHITESPACE"

struct my_s{
	struct data_rt *queue;
	struct data_rt **last;
	struct data_rt **dlast;
	struct bsdconv_phase *rerail;
	size_t offsetA;
	size_t offsetB;
};

int cbcreate(struct bsdconv_instance *ins, struct bsdconv_hash_entry *arg){
	struct my_s *t;
	if(bsdconv_hash_has(ins, HASHKEY)){
		t=bsdconv_hash_get(ins, HASHKEY);
	}else{
		t=malloc(sizeof(struct my_s));
		bsdconv_hash_set(ins, HASHKEY, t);
	}
	t->queue=NULL;
	t->rerail=NULL;
	CURRENT_CODEC(ins)->priv=t;
	return 0;
}

void cbinit(struct bsdconv_instance *ins){
	struct my_s *t=CURRENT_CODEC(ins)->priv;
	t->offsetA=0;
	t->offsetB=0;
	t->last=&t->queue;
	t->dlast=NULL;
	struct data_rt *q;
	while(t->queue){
		DATA_FREE((struct data_rt *)t->queue->data);
		q=t->queue;
		t->queue=t->queue->next;
		DATA_FREE(q);
	}
}

void cbdestroy(struct bsdconv_instance *ins){
	struct my_s *t=CURRENT_CODEC(ins)->priv;
	struct data_rt *q;
	if(bsdconv_hash_has(ins, HASHKEY)){
		while(t->queue){
			DATA_FREE((struct data_rt *)t->queue->data);
			q=t->queue;
			t->queue=t->queue->next;
			DATA_FREE(q);
		}
		free(t);
		bsdconv_hash_del(ins, HASHKEY);
	}
}

void cbconv(struct bsdconv_instance *ins){
	unsigned char *data;
	struct bsdconv_phase *this_phase=CURRENT_PHASE(ins);
	struct my_s *t=CURRENT_CODEC(ins)->priv;
	struct data_rt *q;
	data=this_phase->curr->data;
	int i;
	int ucs=0;

	this_phase->state.status=NEXTPHASE;

	if(this_phase->curr->len>0 && data[0]==0x1){
		for(i=1;i<this_phase->curr->len;++i){
			ucs<<=8;
			ucs|=data[i];
		}
		if(ucs==0x09||ucs==0x0A||ucs==0x0D||ucs==0x20){
			DATA_MALLOC(q);
			*(t->last)=q;
			q->next=NULL;
			q->flags=0;
			t->last=&q->next;
			DATA_MALLOC(q->data);
			if(t->dlast!=NULL)
				*(t->dlast)=q->data;
			t->dlast=&(((struct data_rt *)q->data)->next);
			*((struct data_rt *)q->data)=*(this_phase->curr);
			((struct data_rt *)q->data)->next=NULL;
			this_phase->curr->flags &= ~F_FREE;
			q->len=t->offsetA;

			if(t->rerail){
				t->rerail->flags |= (F_MATCH | F_PENDING);
				t->rerail->match_data=t->queue->data;
			}

			return;
		}
	}
	t->offsetA+=1;

	DATA_MALLOC(this_phase->data_tail->next);
	this_phase->data_tail=this_phase->data_tail->next;
{'fraction_non_alphanumeric': 0.1586129394766415, 'fraction_numerical': 0.010274522395248033, 'mean_word_length': 4.547640249332146, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 139, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7368935', 'n_tokens_mistral': 2631, 'n_tokens_neox': 2404, 'n_words': 551}