<commit_before>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "header.h"

#define CREATE_SIZE 1

extern symbol * create_s(void) {
    symbol * p;
    void * mem = malloc(HEAD + (CREATE_SIZE + 1) * sizeof(symbol));
    if (mem == NULL) return NULL;
    p = (symbol *) (HEAD + (char *) mem);
    CAPACITY(p) = CREATE_SIZE;
    SET_SIZE(p, 0);
    return p;
}

extern void lose_s(symbol * p) {
    if (p == NULL) return;
    free((char *) p - HEAD);
}

/*
   new_p = skip_utf8(p, c, lb, l, n); skips n characters forwards from p + c
   if n +ve, or n characters backwards from p + c - 1 if n -ve. new_p is the new
   position, or 0 on failure.

   -- used to implement hop and next in the utf8 case.
*/

extern int skip_utf8(const symbol * p, int c, int lb, int l, int n) {
    int b;
    if (n >= 0) {
        for (; n > 0; n--) {
            if (c >= l) return -1;
            b = p[c++];
            if (b >= 0xC0) {   /* 1100 0000 */
                while (c < l) {
                    b = p[c];
                    if (b >= 0xC0 || b < 0x80) break;
                    /* break unless b is 10------ */
                    c++;
                }
            }
        }
    } else {
        for (; n < 0; n++) {
            if (c <= lb) return -1;
            b = p[--c];
            if (b >= 0x80) {   /* 1000 0000 */
                while (c > lb) {
                    b = p[c];
                    if (b >= 0xC0) break; /* 1100 0000 */
                    c--;
                }
            }
        }
    }
    return c;
}

/* Code for character groupings: utf8 cases */

static int get_utf8(const symbol * p, int c, int l, int * slot) {
    int b0, b1;
    if (c >= l) return 0;
    b0 = p[c++];
    if (b0 < 0xC0 || c == l) {   /* 1100 0000 */
        * slot = b0; return 1;
    }
    b1 = p[c++];
    if (b0 < 0xE0 || c == l) {   /* 1110 0000 */
        * slot = (b0 & 0x1F) << 6 | (b1 & 0x3F); return 2;
    }
    * slot = (b0 & 0xF) << 12 | (b1 & 0x3F) << 6 | (p[c] & 0x3F); return 3;
}

static int get_b_utf8(const symbol * p, int c, int lb, int * slot) {
    int b0, b1;
    if (c <= lb) return 0;
    b0 = p[--c];
    if (b0 < 0x80 || c == lb) {   /* 1000 0000 */
        * slot = b0; return 1;
    }
    b1 = p[--c];
    if (b1 >= 0xC0 || c == lb) {   /* 1100 0000 */
        * slot = (b1 & 0x1F) << 6 | (b0 & 0x3F); return 2;
    }
    * slot = (p[c] & 0xF) << 12 | (b1 & 0x3F) << 6 | (b0 & 0x3F); return 3;
}

extern int in_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
    do {
	int ch;
	int w = get_utf8(z->p, z->c, z->l, & ch);
	if (!w) return -1;
	if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
	    return w;
	z->c += w;
    } while (repeat);
    return 0;
}

extern int in_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
    do {
	int ch;
	int w = get_b_utf8(z->p, z->c, z->lb, & ch);<commit_msg>Fix get_b_utf8() handling for 3 byte sequences<commit_after>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "header.h"

#define CREATE_SIZE 1

extern symbol * create_s(void) {
    symbol * p;
    void * mem = malloc(HEAD + (CREATE_SIZE + 1) * sizeof(symbol));
    if (mem == NULL) return NULL;
    p = (symbol *) (HEAD + (char *) mem);
    CAPACITY(p) = CREATE_SIZE;
    SET_SIZE(p, 0);
    return p;
}

extern void lose_s(symbol * p) {
    if (p == NULL) return;
    free((char *) p - HEAD);
}

/*
   new_p = skip_utf8(p, c, lb, l, n); skips n characters forwards from p + c
   if n +ve, or n characters backwards from p + c - 1 if n -ve. new_p is the new
   position, or 0 on failure.

   -- used to implement hop and next in the utf8 case.
*/

extern int skip_utf8(const symbol * p, int c, int lb, int l, int n) {
    int b;
    if (n >= 0) {
        for (; n > 0; n--) {
            if (c >= l) return -1;
            b = p[c++];
            if (b >= 0xC0) {   /* 1100 0000 */
                while (c < l) {
                    b = p[c];
                    if (b >= 0xC0 || b < 0x80) break;
                    /* break unless b is 10------ */
                    c++;
                }
            }
        }
    } else {
        for (; n < 0; n++) {
            if (c <= lb) return -1;
            b = p[--c];
            if (b >= 0x80) {   /* 1000 0000 */
                while (c > lb) {
                    b = p[c];
                    if (b >= 0xC0) break; /* 1100 0000 */
                    c--;
                }
            }
        }
    }
    return c;
}

/* Code for character groupings: utf8 cases */

static int get_utf8(const symbol * p, int c, int l, int * slot) {
    int b0, b1;
    if (c >= l) return 0;
    b0 = p[c++];
    if (b0 < 0xC0 || c == l) {   /* 1100 0000 */
        * slot = b0; return 1;
    }
    b1 = p[c++];
    if (b0 < 0xE0 || c == l) {   /* 1110 0000 */
        * slot = (b0 & 0x1F) << 6 | (b1 & 0x3F); return 2;
    }
    * slot = (b0 & 0xF) << 12 | (b1 & 0x3F) << 6 | (p[c] & 0x3F); return 3;
}

static int get_b_utf8(const symbol * p, int c, int lb, int * slot) {
    int b0, b1;
    if (c <= lb) return 0;
    b0 = p[--c];
    if (b0 < 0x80 || c == lb) {   /* 1000 0000 */
        * slot = b0; return 1;
    }
    b1 = p[--c];
    if (b1 >= 0xC0 || c == lb) {   /* 1100 0000 */
        * slot = (b1 & 0x1F) << 6 | (b0 & 0x3F); return 2;
    }
    * slot = (p[--c] & 0xF) << 12 | (b1 & 0x3F) << 6 | (b0 & 0x3F); return 3;
}

extern int in_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
    do {
	int ch;
	int w = get_utf8(z->p, z->c, z->l, & ch);
	if (!w) return -1;
	if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
	    return w;
	z->c += w;
    } while (repeat);
    return 0;
}

extern int in_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
    do {
	int ch;
	int w = get_b_utf8(z->p, z->c, z->lb, & ch);
{'fraction_non_alphanumeric': 0.17634191485781592, 'fraction_numerical': 0.05485445061416793, 'mean_word_length': 1.5369184805804523, 'pattern_counts': {'":': 0, '<': 55, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3699480', 'n_tokens_mistral': 2585, 'n_tokens_neox': 2313, 'n_words': 757}