<commit_before>/*
	This is part of pyahocorasick Python module.
	
	Helpers functions.
	This file is included directly.

	Author    : <NAME>, <EMAIL>
    WWW       : http://0x80.pl
	License   : public domain
*/


void* memory_alloc(ssize_t size) {
    void* res = memalloc(size);

    //printf("allocated %p %d\n", res, size);
    return res;
}

void memory_free(void* ptr) {
    //printf("freeing %p\n", ptr);
    memfree(ptr);
}


void xfree(void* ptr) {
	if (ptr != NULL) {
		memory_free(ptr);
	}
}


#if !defined(PY3K) || !defined(AHOCORASICK_UNICODE)
//  define when pymod_get_string makes a copy of string
#   define INPUT_KEEPS_COPY
#endif


/* returns bytes or unicode internal buffer */
static PyObject*
pymod_get_string(PyObject* obj, TRIE_LETTER_TYPE** word, ssize_t* wordlen) {

#ifdef INPUT_KEEPS_COPY
	ssize_t i;
	char* bytes;
#endif

#ifdef PY3K
#   ifdef AHOCORASICK_UNICODE
        if (PyUnicode_Check(obj)) {
            *word = (TRIE_LETTER_TYPE*)(PyUnicode_AS_UNICODE(obj));
            *wordlen = PyUnicode_GET_SIZE(obj);
            Py_INCREF(obj);
            return obj;
        }
        else {
            PyErr_SetString(PyExc_TypeError, "string expected");
            return NULL;
        }
#   else
#       ifndef INPUT_KEEPS_COPY
#           error "defines inconsistency"
#       endif
        if (PyBytes_Check(obj)) {
            *wordlen = PyBytes_GET_SIZE(obj);
			*word    = (TRIE_LETTER_TYPE*)memory_alloc(*wordlen * TRIE_LETTER_SIZE);
            if (*word == NULL) {
                PyErr_NoMemory();
                return NULL;
            }

			bytes = PyBytes_AS_STRING(obj);
			for (i=0; i < *wordlen; i++) {
				(*word)[i] = bytes[i];
			}
			// Note: there is no INCREF
            return obj;
        }
        else {
            PyErr_SetString(PyExc_TypeError, "bytes expected");
            return NULL;
        }
#   endif
#else // PY_MAJOR_VERSION == 3
#       ifndef INPUT_KEEPS_COPY
#           error "defines inconsistency"
#       endif
	if (PyString_Check(obj)) {
        *wordlen = PyString_GET_SIZE(obj);
		*word    = (TRIE_LETTER_TYPE*)memory_alloc(*wordlen * TRIE_LETTER_SIZE);
        if (*word == NULL) {
		    PyErr_NoMemory();
            return NULL;
        }


		bytes = PyString_AS_STRING(obj);
		for (i=0; i < *wordlen; i++) {
			(*word)[i] = bytes[i];
		};

        Py_INCREF(obj);
		return obj;
    } else {
		PyErr_SetString(PyExc_TypeError, "string required");
		return NULL;
	}
#endif
}


static bool
__read_sequence__from_tuple(PyObject* obj, TRIE_LETTER_TYPE** word, ssize_t* wordlen) {
	Py_ssize_t i;
	Py_ssize_t size = PyTuple_Size(obj);

	*wordlen = size;
	*word = (TRIE_LETTER_TYPE*)memory_alloc(size * TRIE_LETTER_SIZE);
	if (*word == NULL) {
		PyErr_NoMemory();
		return false;
	}

	for (i=0; i < size; i++) {
		Py_ssize_t value = PyNumber_AsSsize_t(PyTuple_GetItem(obj, i), PyExc_ValueError);
		if (value == -1 && PyErr_Occurred()) {
			PyErr_Format(PyExc_ValueError, "item #%zd is not a number", i);
			memory_free(*word);
			return false;
		}


        // TODO: both min and max values should be configured
		if (value < 0 || value > 65535) {
			PyErr_Format(PyExc_ValueError, "item #%zd: value %zd outside range [%zd..%zd]", i, value, 0, 65535);
			memory_free(*word);
			return false;
		}

		(*word)[i] = (TRIE_LETTER_TYPE)value;
	}

	return true;
}


static bool
pymod_get_sequence(PyObject* obj, TRIE_LETTER_TYPE** word, ssize_t* wordlen) {
	if (PyTuple_Check(obj)) {
		return __read_sequence__from_tuple(obj, word, wordlen);
	} else {
		PyErr_Format(PyExc_TypeError, "argument is not a supported sequence type");
		return false;
	}
}

<commit_msg>Remove remaining compilation warnings (issue #72)<commit_after>/*
	This is part of pyahocorasick Python module.
	
	Helpers functions.
	This file is included directly.

	Author    : <NAME>, <EMAIL>
    WWW       : http://0x80.pl
	License   : public domain
*/


void* memory_alloc(ssize_t size) {
    void* res = memalloc(size);

    //printf("allocated %p %d\n", res, size);
    return res;
}

void memory_free(void* ptr) {
    //printf("freeing %p\n", ptr);
    memfree(ptr);
}


void xfree(void* ptr) {
	if (ptr != NULL) {
		memory_free(ptr);
	}
}


#if !defined(PY3K) || !defined(AHOCORASICK_UNICODE)
//  define when pymod_get_string makes a copy of string
#   define INPUT_KEEPS_COPY
#endif


/* returns bytes or unicode internal buffer */
static PyObject*
pymod_get_string(PyObject* obj, TRIE_LETTER_TYPE** word, ssize_t* wordlen) {

#ifdef INPUT_KEEPS_COPY
	ssize_t i;
	char* bytes;
#endif

#ifdef PY3K
#   ifdef AHOCORASICK_UNICODE
        if (PyUnicode_Check(obj)) {
            *word = (TRIE_LETTER_TYPE*)(PyUnicode_AS_UNICODE(obj));
            *wordlen = PyUnicode_GET_SIZE(obj);
            Py_INCREF(obj);
            return obj;
        }
        else {
            PyErr_SetString(PyExc_TypeError, "string expected");
            return NULL;
        }
#   else
#       ifndef INPUT_KEEPS_COPY
#           error "defines inconsistency"
#       endif
        if (PyBytes_Check(obj)) {
            *wordlen = PyBytes_GET_SIZE(obj);
			*word    = (TRIE_LETTER_TYPE*)memory_alloc(*wordlen * TRIE_LETTER_SIZE);
            if (*word == NULL) {
                PyErr_NoMemory();
                return NULL;
            }

			bytes = PyBytes_AS_STRING(obj);
			for (i=0; i < *wordlen; i++) {
				(*word)[i] = bytes[i];
			}
			// Note: there is no INCREF
            return obj;
        }
        else {
            PyErr_SetString(PyExc_TypeError, "bytes expected");
            return NULL;
        }
#   endif
#else // PY_MAJOR_VERSION == 3
#       ifndef INPUT_KEEPS_COPY
#           error "defines inconsistency"
#       endif
	if (PyString_Check(obj)) {
        *wordlen = PyString_GET_SIZE(obj);
		*word    = (TRIE_LETTER_TYPE*)memory_alloc(*wordlen * TRIE_LETTER_SIZE);
        if (*word == NULL) {
		    PyErr_NoMemory();
            return NULL;
        }


		bytes = PyString_AS_STRING(obj);
		for (i=0; i < *wordlen; i++) {
			(*word)[i] = bytes[i];
		};

        Py_INCREF(obj);
		return obj;
    } else {
		PyErr_SetString(PyExc_TypeError, "string required");
		return NULL;
	}
#endif
}


static bool
__read_sequence__from_tuple(PyObject* obj, TRIE_LETTER_TYPE** word, ssize_t* wordlen) {
	Py_ssize_t i;
	Py_ssize_t size = PyTuple_Size(obj);

	*wordlen = size;
	*word = (TRIE_LETTER_TYPE*)memory_alloc(size * TRIE_LETTER_SIZE);
	if (*word == NULL) {
		PyErr_NoMemory();
		return false;
	}

	for (i=0; i < size; i++) {
		Py_ssize_t value = PyNumber_AsSsize_t(PyTuple_GetItem(obj, i), PyExc_ValueError);
		if (value == -1 && PyErr_Occurred()) {
			PyErr_Format(PyExc_ValueError, "item #%zd is not a number", i);
			memory_free(*word);
			return false;
		}


        // TODO: both min and max values should be configured
		if (value < 0 || value > 65535) {
			PyErr_Format(PyExc_ValueError, "item #%zd: value %zd outside range [%d..%d]", i, value, 0, 65535);
			memory_free(*word);
			return false;
		}

		(*word)[i] = (TRIE_LETTER_TYPE)value;
	}

	return true;
}


static bool
pymod_get_sequence(PyObject* obj, TRIE_LETTER_TYPE** word, ssize_t* wordlen) {
	if (PyTuple_Check(obj)) {
		return __read_sequence__from_tuple(obj, word, wordlen);
	} else {
		PyErr_Format(PyExc_TypeError, "argument is not a supported sequence type");
		return false;
	}
}


{'fraction_non_alphanumeric': 0.1262999452654625, 'fraction_numerical': 0.006294471811713191, 'mean_word_length': 2.8427970557308098, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '542645', 'n_tokens_mistral': 2864, 'n_tokens_neox': 2700, 'n_words': 595}