<commit_before>#include <ctype.h>
#include <string.h>
#include <stdio.h>
#include <xlocale.h>
#include <stdarg.h>
#include <strings.h>
#include <stdlib.h>
#include "util.h"

/* Blatantly stolen from darwin source code and modified for my own purposes
   TODO: take a look at boyer-moore string searching
 */
char* ag_strnstr(const char *s, const char *find, size_t slen)
{
    char c, sc;
    size_t len;

    if ((c = *find++) != '\0') {
        len = strlen(find);
        do {
            do {
                if (slen-- < 1 || (sc = *s++) == '\0')
                    return(NULL);
            } while (sc != c);
            if (len > slen)
                return(NULL);
        } while (strncmp(s, find, len) != 0);
        s--;
    }
    return((char *)s);
}

char* ag_strncasestr(const char *s, const char *find, size_t slen)
{
    char c, sc;
    size_t len;

    if ((c = *find++) != '\0') {
        len = strlen(find);
        do {
            do {
                if (slen-- < 1 || (sc = *s++) == '\0')
                    return(NULL);
            } while (tolower(sc) != tolower(c));
            if (len > slen)
                return(NULL);
        } while (strncasecmp(s, find, len) != 0);
        s--;
    }
    return((char *)s);
}

void generate_skip_lookup(const char *find, size_t f_len, size_t skip_lookup[]) {
    int i = 0;
    for (i = 0; i < 256; i++) {
        skip_lookup[i] = f_len - 1;
    }

    for (i = f_len - 1; i >= 0; i--) {
        skip_lookup[(unsigned char)find[i]] = f_len - i;
    }
}

/* Boyer-Moore-Horspool strstr */
char* ag_boyer_moore_strnstr(const char *s, const char *find, size_t s_len, size_t f_len, size_t skip_lookup[]) {
    int i;
    int pos = 0;
    int skip_chars;
    if (f_len > s_len) {
        return(NULL);
    }

    for (i = 0; i < 256; i++) {
        skip_lookup[i] = f_len - 1;
    }

    /* TODO: move the lookup generation outside this function */
    for (i = f_len - 1; i >= 0; i--) {
        skip_lookup[(int)find[i]] = f_len - i;
/*        printf("skip_lookup[%c] = %i\n", find[i], (int)(f_len - i)); */
    }

    while (pos < (int)(s_len - f_len)) {
        for (i = 0; s[pos] == find[i]; pos++) {
            if (i == (int)f_len - 1) {
/*                printf("match found at position %i, returning\n", pos); */
                return((char *)(&(s[pos-i])));
            }
            i++;
        }
        pos += i;

        skip_chars = skip_lookup[(int)(unsigned char)s[pos]];
        pos += skip_chars;
    }
/*    printf("nothing found, returning\n"); */
    return(NULL);
}

int is_binary(const void* buf, const int buf_len) {
    int suspicious_bytes = 0;
    int total_bytes = buf_len > 1024 ? 1024 : buf_len;
    const unsigned char *buf_c = buf;
    int i;

    for (i = 0; i < buf_len && i < 1024; i++) {
        if (buf_c[i] == '\0') {
            /* NULL char. It's binary */
            return(1);
        }
        else if (buf_c[i] < 32 || buf_c[i] > 127) {
            suspicious_bytes++;
        }
    }

    /* If > 10% of bytes are suspicious, assume it's binary */
    if ((suspicious_bytes * 100) / total_bytes > 10) {
        return(1);
    }

    return(0);
}

/*
 * These functions are taken from Linux. Renamed so there's no
 * possible function name conflicts.
 */
size_t ag_strlcat(char *dest, const char *src, size_t count)
{
    size_t dsize = strlen(dest);
    size_t len = strlen(src);
    size_t res = dsize + len;

    dest += dsize;
    count -= dsize;

    if (len >= count)
        len = count - 1;

    memcpy(dest, src, len);

    dest[len] = 0;

    return res;
}

size_t ag_strlcpy(char *dest, const char *src, size_t size)
{
    size_t ret = strlen(src);

    if (size)
    {
        size_t len = (ret >= size) ? size - 1 : ret;
        memcpy(dest, src, len);
        dest[len] = '\0';
    }

    return ret;
}
<commit_msg>Remove duplicated code. Actually return the right position.<commit_after>#include <ctype.h>
#include <string.h>
#include <stdio.h>
#include <xlocale.h>
#include <stdarg.h>
#include <strings.h>
#include <stdlib.h>
#include "util.h"

/* Blatantly stolen from darwin source code and modified for my own purposes
   TODO: take a look at boyer-moore string searching
 */
char* ag_strnstr(const char *s, const char *find, size_t slen)
{
    char c, sc;
    size_t len;

    if ((c = *find++) != '\0') {
        len = strlen(find);
        do {
            do {
                if (slen-- < 1 || (sc = *s++) == '\0')
                    return(NULL);
            } while (sc != c);
            if (len > slen)
                return(NULL);
        } while (strncmp(s, find, len) != 0);
        s--;
    }
    return((char *)s);
}

char* ag_strncasestr(const char *s, const char *find, size_t slen)
{
    char c, sc;
    size_t len;

    if ((c = *find++) != '\0') {
        len = strlen(find);
        do {
            do {
                if (slen-- < 1 || (sc = *s++) == '\0')
                    return(NULL);
            } while (tolower(sc) != tolower(c));
            if (len > slen)
                return(NULL);
        } while (strncasecmp(s, find, len) != 0);
        s--;
    }
    return((char *)s);
}

void generate_skip_lookup(const char *find, size_t f_len, size_t skip_lookup[]) {
    int i = 0;
    for (i = 0; i < 256; i++) {
        skip_lookup[i] = f_len - 1;
    }

    for (i = f_len - 1; i >= 0; i--) {
        skip_lookup[(unsigned char)find[i]] = f_len - i;
    }
}

/* Boyer-Moore-Horspool strstr */
char* ag_boyer_moore_strnstr(const char *s, const char *find, size_t s_len, size_t f_len, size_t skip_lookup[]) {
    int i;
    int pos = 0;
    int skip_chars;
    if (f_len > s_len) {
        return(NULL);
    }

    while (pos < (int)(s_len - f_len)) {
        for (i = f_len - 1; s[pos + i] == find[i]; i--) {
            if (i == 0) {
/*                printf("match found at position %i, returning\n", pos); */
                return((char *)(&(s[pos])));
            }
        }
        pos += i;

        skip_chars = skip_lookup[(int)(unsigned char)s[pos]];
        pos += skip_chars;
    }
/*    printf("nothing found, returning\n"); */
    return(NULL);
}

int is_binary(const void* buf, const int buf_len) {
    int suspicious_bytes = 0;
    int total_bytes = buf_len > 1024 ? 1024 : buf_len;
    const unsigned char *buf_c = buf;
    int i;

    for (i = 0; i < buf_len && i < 1024; i++) {
        if (buf_c[i] == '\0') {
            /* NULL char. It's binary */
            return(1);
        }
        else if (buf_c[i] < 32 || buf_c[i] > 127) {
            suspicious_bytes++;
        }
    }

    /* If > 10% of bytes are suspicious, assume it's binary */
    if ((suspicious_bytes * 100) / total_bytes > 10) {
        return(1);
    }

    return(0);
}

/*
 * These functions are taken from Linux. Renamed so there's no
 * possible function name conflicts.
 */
size_t ag_strlcat(char *dest, const char *src, size_t count)
{
    size_t dsize = strlen(dest);
    size_t len = strlen(src);
    size_t res = dsize + len;

    dest += dsize;
    count -= dsize;

    if (len >= count)
        len = count - 1;

    memcpy(dest, src, len);

    dest[len] = 0;

    return res;
}

size_t ag_strlcpy(char *dest, const char *src, size_t size)
{
    size_t ret = strlen(src);

    if (size)
    {
        size_t len = (ret >= size) ? size - 1 : ret;
        memcpy(dest, src, len);
        dest[len] = '\0';
    }

    return ret;
}

{'fraction_non_alphanumeric': 0.16070222822417285, 'fraction_numerical': 0.015259959486833221, 'mean_word_length': 1.8429942418426104, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4383817', 'n_tokens_mistral': 2785, 'n_tokens_neox': 2632, 'n_words': 747}