<commit_before>/* ************************************************************************ */
/*																			*/
/*  Neko PCRE Bindings Library												*/
/*  Copyright (c)2005 Motion-Twin											*/
/*																			*/
/* This library is free software; you can redistribute it and/or			*/
/* modify it under the terms of the GNU Lesser General Public				*/
/* License as published by the Free Software Foundation; either				*/
/* version 2.1 of the License, or (at your option) any later version.		*/
/*																			*/
/* This library is distributed in the hope that it will be useful,			*/
/* but WITHOUT ANY WARRANTY; without even the implied warranty of			*/
/* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU		*/
/* Lesser General Public License or the LICENSE file for more details.		*/
/*																			*/
/* ************************************************************************ */
#include <neko.h>
#include <string.h>
#define PCRE_STATIC
#include <pcre.h>

#define PCRE(o)		((pcredata*)val_data(o))
#define NMATCHS		(10 * 3)

typedef struct {
	value str;
	pcre *r;
	int nmatchs;
	int matchs[NMATCHS];
} pcredata;

DEFINE_KIND(k_regexp);

static field id_pos;
static field id_len;

/**
	<doc>
	<h1>Regexp</h1>
	<p>
	Regular expressions using PCRE engine.
	</p>
	</doc>
**/

static void free_regexp( value p ) {	
	pcre_free( PCRE(p)->r );
}

/**
	regexp_new_options : reg:string -> options:string -> 'regexp
	<doc>Build a new regexpr with the following options :
	<ul>
		<li>i : case insensitive matching</li>
		<li>s : . match anything including newlines</li>
		<li>m : treat the input as a multiline string</li>
		<li>u : run in utf8 mode</li>
		<li>g : turn off greedy behavior</li>
	</ul>
	</doc>
**/
static value regexp_new_options( value s, value opt ) {
	val_check(s,string);
	val_check(opt,string);
	{
		value v;
		const char *error;
		int err_offset;
		pcre *p;
		pcredata *pdata;
		char *o = val_string(opt);
		int options = 0;
		while( *o ) {
			switch( *o++ ) {
			case 'i':
				options |= PCRE_CASELESS;
				break;
			case 's':
				options |= PCRE_DOTALL;
				break;
			case 'm':
				options |= PCRE_MULTILINE;
				break;
			case 'u':
				options |= PCRE_UTF8;
				break;
			case 'g':
				options |= PCRE_UNGREEDY;
				break;
			default:
				neko_error();
				break;
			}
		}
		p = pcre_compile(val_string(s),options,&error,&err_offset,NULL);
		if( p == NULL ) {
			buffer b = alloc_buffer("Regexp compilation error : ");
			buffer_append(b,error);
			buffer_append(b," in ");
			val_buffer(b,s);
			bfailure(b);
		}
		v = alloc_abstract(k_regexp,alloc(sizeof(pcredata)));
		pdata = PCRE(v);
		pdata->r = p;
		pdata->str = val_null;
		pdata->nmatchs = 0;
		pcre_fullinfo(p,NULL,PCRE_INFO_CAPTURECOUNT,&pdata->nmatchs);
		val_gc(v,free_regexp);
		return v;
	}	
}

/**
	regexp_new : string -> 'regexp
	<doc>Build a new regexp</doc>
**/
static value regexp_new( value s ) {
	return regexp_new_options(s,alloc_string(""));
}

/**
	regexp_match : 'regexp -> string -> pos:int -> len:int -> bool
	<doc>Match [len] chars of a string starting at [pos] using the regexp.
	Return true if match found</doc>
**/
static value regexp_match( value o, value s, value p, value len ) {
	pcredata *d;
	int pp,ll;
	val_check_kind(o,k_regexp);
	val_check(s,string);
	val_check(p,int);
	val_check(len,int);
	pp = val_int(p);
	ll = val_int(len);
	if( pp < 0 || ll < 0 || pp > val_strlen(s) || pp + ll > val_strlen(s) )
		neko_error();
	d = PCRE(o);
	if( pcre_exec(d->r,NULL,val_string(s)+pp,ll,0,0,d->matchs,NMATCHS) >= 0 ) {
		if( pp > 0 ) {
			int i;
			for(i=0;i<NMATCHS;i+=2)
				d->matchs[i] += pp;
		}
		d->str = s;
		return val_true;
	} else {
		d->str = val_null;
		return val_false;
	}
}

static value do_replace( value o, value s, value s2, bool all ) {	
	val_check_kind(o,k_regexp);	
	val_check(s,string);
	val_check(s2,string);	
	{
		pcredata *d = PCRE(o);
		buffer b = alloc_buffer(NULL);
		int pos = 0;
		int len = val_strlen(s);
		const char *str = val_string(s);<commit_msg>fixed invalid matched length when not index 0.<commit_after>/* ************************************************************************ */
/*																			*/
/*  Neko PCRE Bindings Library												*/
/*  Copyright (c)2005 Motion-Twin											*/
/*																			*/
/* This library is free software; you can redistribute it and/or			*/
/* modify it under the terms of the GNU Lesser General Public				*/
/* License as published by the Free Software Foundation; either				*/
/* version 2.1 of the License, or (at your option) any later version.		*/
/*																			*/
/* This library is distributed in the hope that it will be useful,			*/
/* but WITHOUT ANY WARRANTY; without even the implied warranty of			*/
/* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU		*/
/* Lesser General Public License or the LICENSE file for more details.		*/
/*																			*/
/* ************************************************************************ */
#include <neko.h>
#include <string.h>
#define PCRE_STATIC
#include <pcre.h>

#define PCRE(o)		((pcredata*)val_data(o))
#define NMATCHS		(10 * 3)

typedef struct {
	value str;
	pcre *r;
	int nmatchs;
	int matchs[NMATCHS];
} pcredata;

DEFINE_KIND(k_regexp);

static field id_pos;
static field id_len;

/**
	<doc>
	<h1>Regexp</h1>
	<p>
	Regular expressions using PCRE engine.
	</p>
	</doc>
**/

static void free_regexp( value p ) {	
	pcre_free( PCRE(p)->r );
}

/**
	regexp_new_options : reg:string -> options:string -> 'regexp
	<doc>Build a new regexpr with the following options :
	<ul>
		<li>i : case insensitive matching</li>
		<li>s : . match anything including newlines</li>
		<li>m : treat the input as a multiline string</li>
		<li>u : run in utf8 mode</li>
		<li>g : turn off greedy behavior</li>
	</ul>
	</doc>
**/
static value regexp_new_options( value s, value opt ) {
	val_check(s,string);
	val_check(opt,string);
	{
		value v;
		const char *error;
		int err_offset;
		pcre *p;
		pcredata *pdata;
		char *o = val_string(opt);
		int options = 0;
		while( *o ) {
			switch( *o++ ) {
			case 'i':
				options |= PCRE_CASELESS;
				break;
			case 's':
				options |= PCRE_DOTALL;
				break;
			case 'm':
				options |= PCRE_MULTILINE;
				break;
			case 'u':
				options |= PCRE_UTF8;
				break;
			case 'g':
				options |= PCRE_UNGREEDY;
				break;
			default:
				neko_error();
				break;
			}
		}
		p = pcre_compile(val_string(s),options,&error,&err_offset,NULL);
		if( p == NULL ) {
			buffer b = alloc_buffer("Regexp compilation error : ");
			buffer_append(b,error);
			buffer_append(b," in ");
			val_buffer(b,s);
			bfailure(b);
		}
		v = alloc_abstract(k_regexp,alloc(sizeof(pcredata)));
		pdata = PCRE(v);
		pdata->r = p;
		pdata->str = val_null;
		pdata->nmatchs = 0;
		pcre_fullinfo(p,NULL,PCRE_INFO_CAPTURECOUNT,&pdata->nmatchs);
		val_gc(v,free_regexp);
		return v;
	}	
}

/**
	regexp_new : string -> 'regexp
	<doc>Build a new regexp</doc>
**/
static value regexp_new( value s ) {
	return regexp_new_options(s,alloc_string(""));
}

/**
	regexp_match : 'regexp -> string -> pos:int -> len:int -> bool
	<doc>Match [len] chars of a string starting at [pos] using the regexp.
	Return true if match found</doc>
**/
static value regexp_match( value o, value s, value p, value len ) {
	pcredata *d;
	int pp,ll;
	val_check_kind(o,k_regexp);
	val_check(s,string);
	val_check(p,int);
	val_check(len,int);
	pp = val_int(p);
	ll = val_int(len);
	if( pp < 0 || ll < 0 || pp > val_strlen(s) || pp + ll > val_strlen(s) )
		neko_error();
	d = PCRE(o);
	if( pcre_exec(d->r,NULL,val_string(s)+pp,ll,0,0,d->matchs,NMATCHS) >= 0 ) {
		if( pp > 0 ) {
			int i;
			for(i=0;i<NMATCHS;i++)
				d->matchs[i] += pp;
		}
		d->str = s;
		return val_true;
	} else {
		d->str = val_null;
		return val_false;
	}
}

static value do_replace( value o, value s, value s2, bool all ) {	
	val_check_kind(o,k_regexp);	
	val_check(s,string);
	val_check(s2,string);	
	{
		pcredata *d = PCRE(o);
		buffer b = alloc_buffer(NULL);
		int pos = 0;
		int len = val_strlen(s);
		const char *str = val_string(s);
{'fraction_non_alphanumeric': 0.17996541501976285, 'fraction_numerical': 0.006422924901185771, 'mean_word_length': 3.2281984334203657, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 0, '>': 99, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2089507', 'n_tokens_mistral': 3483, 'n_tokens_neox': 3059, 'n_words': 796}