<commit_before>/*
 * Copyright (c) 2009-2014 <NAME>, <NAME>, <NAME>,
 *                         <NAME>
 *
 * The software is licensed under either the MIT License (below) or the Perl
 * license.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#include <assert.h>
#include <stddef.h>
#include <string.h>
#ifdef __SSE4_2__
#ifdef _MSC_VER
#include <nmmintrin.h>
#else
#include <x86intrin.h>
#endif
#endif
#include "picohttpparser.h"

/* $Id: 8e21379070e9a13462ee692b40548e5fd59a547c $ */

#if __GNUC__ >= 3
#define likely(x) __builtin_expect(!!(x), 1)
#define unlikely(x) __builtin_expect(!!(x), 0)
#else
#define likely(x) (x)
#define unlikely(x) (x)
#endif

#ifdef _MSC_VER
#define ALIGNED(n) _declspec(align(n))
#else
#define ALIGNED(n) __attribute__((aligned(n)))
#endif

static inline const void *check_eof(const void *buf, const void *end, int *ret)
{
	if (likely((buf != end)))
		return buf;
	*ret = -2;
	return NULL;
}

static inline const void *expect_char(char ch, const void *buf, const void *end,
				      int *ret)
{
	const char *ptr = buf;

	if (!check_eof(buf, end, ret))
		return NULL;
	if (*ptr++ != ch) {
		*ret = -1;
		return NULL;
	}
	return ptr;
}

#define IS_PRINTABLE_ASCII(c) ((unsigned char)(c)-040u < 0137u)

static const char *token_char_map =
	"\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"
	"\0\1\1\1\1\1\1\1\0\0\1\1\0\1\1\0\1\1\1\1\1\1\1\1\1\1\0\0\0\0\0\0"
	"\0\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\0\0\0\1\1"
	"\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\0\1\0\1\0"
	"\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"
	"\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"
	"\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"
	"\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0";

static const char *findchar_fast(const char *buf, const char *buf_end, const char *ranges, size_t ranges_size, int *found)
{
    *found = 0;
#ifdef __SSE4_2__
    if (likely(buf_end - buf >= 16)) {
        __m128i ranges16 = _mm_loadu_si128((const __m128i *)ranges);

        size_t left = (buf_end - buf) & ~15;
        do {
            __m128i b16 = _mm_loadu_si128((void *)buf);
            int r = _mm_cmpestri(ranges16, ranges_size, b16, 16, _SIDD_LEAST_SIGNIFICANT | _SIDD_CMP_RANGES | _SIDD_UBYTE_OPS);
            if (unlikely(r != 16)) {
                buf += r;
                *found = 1;
                break;
            }
            buf += 16;
            left -= 16;
        } while (likely(left != 0));
    }
#else
    /* suppress unused parameter warning */
    (void)buf_end;
    (void)ranges;
    (void)ranges_size;
#endif
    return buf;
}

static inline const void *advance_token(const char **tok, size_t *toklen,
					const char *buf, const void *end,
					int *ret)
{
	const char *tok_start = buf;
	static const char ALIGNED(16) ranges2[] = "\000\040\177\177";
	int found;

	buf = findchar_fast(buf, end, ranges2, sizeof(ranges2) - 1, &found);
	if (!found) {
		if (!check_eof(buf, end, ret))
			return NULL;
	}
	while (1) {
		if (*buf == ' ') {
			break;
		} else if (unlikely(!IS_PRINTABLE_ASCII(*buf))) {
			if ((unsigned char)*buf < '\040' || *buf == '\177') {
				*ret = -1;
				return NULL;
			}
		}
		++buf;
		if (!check_eof(buf, end, ret))
			return NULL;
	}
	*tok = tok_start;
	*toklen = buf - tok_start;
	return buf;
}


static const char *get_token_to_eol(const char *buf, const char *buf_end,
				    const char **token, size_t *token_len,
				    int *ret)
{
	const char *token_start = buf;

#ifdef __SSE4_2__
	static const char ranges1[] = "\0\010"
		/* allow HT */
		"\012\037"
		/* allow SP and up to but not including DEL */
		"\177\177"
		/* allow chars w. MSB set */
		;
	int found;
	buf = findchar_fast(buf, buf_end, ranges1, sizeof(ranges1) - 1, &found);
	if (found)
		goto FOUND_CTL;
#else
	/*
	 * find non-printable char within the next 8 bytes, this is the
	 * hottest code; manually inlined
	 */
	while (likely(buf_end - buf >= 8)) {
#define DOIT()								\
		do {							\
			if (unlikely(!IS_PRINTABLE_ASCII(*buf)))	\
				goto NonPrintable;			\
			++buf;						\
		} while (0)
		DOIT();
		DOIT();
		DOIT();
		DOIT();
		DOIT();
		DOIT();
		DOIT();
		DOIT();
#undef DOIT
		continue;
	NonPrintable:
		if ((likely((unsigned char)*buf < '\040') &&
		     likely(*buf != '\011')) || unlikely(*buf == '\177')) {
			goto FOUND_CTL;
		}
		++buf;
	}
#endif
	for (;; ++buf) {
		if (!check_eof(buf, buf_end, ret))
			return NULL;
		if (unlikely(!IS_PRINTABLE_ASCII(*buf))) {
			if ((likely((unsigned char)*buf < '\040') &&
			     likely(*buf != '\011')) ||
			    unlikely(*buf == '\177')) {
				goto FOUND_CTL;
			}
		}
	}
FOUND_CTL:
	if (likely(*buf == '\015')) {
		++buf;
		if (!expect_char('\012', buf, buf_end, ret))
			return NULL;
		*token_len = buf - 2 - token_start;
	} else if (*buf == '\012') {
		*token_len = buf - token_start;
		++buf;
	} else {
		*ret = -1;
		return NULL;
	}
	*token = token_start;
	return buf;
}

static const char *is_complete(const char *buf, const char *buf_end,
			       size_t last_len, int *ret)
{
	int ret_cnt = 0;
	buf = last_len < 3 ? buf : buf + last_len - 3;

	while (1) {
		if (!check_eof(buf, buf_end, ret))
			return NULL;
		if (*buf == '\015') {
			++buf;
			if (!check_eof(buf, buf_end, ret))
				return NULL;
			if (!expect_char('\012', buf, buf_end, ret))
				return NULL;
			++ret_cnt;
		} else if (*buf == '\012') {
			++buf;
			++ret_cnt;
		} else {
			++buf;
			ret_cnt = 0;
		}
		if (ret_cnt == 2) {
			return buf;
		}
	}
	*ret = -2;
	return NULL;
}

/* *_buf is always within [buf, buf_end) upon success */
static const char *parse_int(const char *buf, const char *buf_end,
			     int *value, int *ret)
{
	int v;
	if (!check_eof(buf, buf_end, ret))
		return NULL;
	if (!('0' <= *buf && *buf <= '9')) {
		*ret = -1;
		return NULL;
	}
	v = 0;
	for (;; ++buf) {
		if (!check_eof(buf, buf_end, ret))
			return NULL;
		if ('0' <= *buf && *buf <= '9') {
			v = v * 10 + *buf - '0';
		} else {
			break;
		}
	}

	*value = v;
	return buf;
}

/* returned pointer is always within [buf, buf_end), or null */
static const char *parse_http_version(const char *buf, const char *buf_end, int *minor_version, int *ret)
{
	char str[] = { 'H', 'T', 'T', 'P', '/', '1', '.', };
	int i;

	for (i = 0; i < sizeof(str); i++)
		if (!expect_char(str[i], buf, buf_end, ret))
			return NULL;
    return parse_int(buf, buf_end, minor_version, ret);
}

static const char *parse_headers(const char *buf, const char *buf_end, struct phr_header *headers, size_t *num_headers,
                                 size_t max_headers, int *ret)
{
    for (;; ++*num_headers) {
 if (!check_eof(buf, buf_end, ret))
	 return NULL;
        if (*buf == '\015') {
            ++buf;<commit_msg>fix parse_http_version() for indentation<commit_after>/*
 * Copyright (c) 2009-2014 <NAME>, <NAME>, <NAME>,
 *                         <NAME>
 *
 * The software is licensed under either the MIT License (below) or the Perl
 * license.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#include <assert.h>
#include <stddef.h>
#include <string.h>
#ifdef __SSE4_2__
#ifdef _MSC_VER
#include <nmmintrin.h>
#else
#include <x86intrin.h>
#endif
#endif
#include "picohttpparser.h"

/* $Id: 8e21379070e9a13462ee692b40548e5fd59a547c $ */

#if __GNUC__ >= 3
#define likely(x) __builtin_expect(!!(x), 1)
#define unlikely(x) __builtin_expect(!!(x), 0)
#else
#define likely(x) (x)
#define unlikely(x) (x)
#endif

#ifdef _MSC_VER
#define ALIGNED(n) _declspec(align(n))
#else
#define ALIGNED(n) __attribute__((aligned(n)))
#endif

static inline const void *check_eof(const void *buf, const void *end, int *ret)
{
	if (likely((buf != end)))
		return buf;
	*ret = -2;
	return NULL;
}

static inline const void *expect_char(char ch, const void *buf, const void *end,
				      int *ret)
{
	const char *ptr = buf;

	if (!check_eof(buf, end, ret))
		return NULL;
	if (*ptr++ != ch) {
		*ret = -1;
		return NULL;
	}
	return ptr;
}

#define IS_PRINTABLE_ASCII(c) ((unsigned char)(c)-040u < 0137u)

static const char *token_char_map =
	"\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"
	"\0\1\1\1\1\1\1\1\0\0\1\1\0\1\1\0\1\1\1\1\1\1\1\1\1\1\0\0\0\0\0\0"
	"\0\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\0\0\0\1\1"
	"\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\0\1\0\1\0"
	"\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"
	"\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"
	"\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"
	"\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0";

static const char *findchar_fast(const char *buf, const char *buf_end, const char *ranges, size_t ranges_size, int *found)
{
    *found = 0;
#ifdef __SSE4_2__
    if (likely(buf_end - buf >= 16)) {
        __m128i ranges16 = _mm_loadu_si128((const __m128i *)ranges);

        size_t left = (buf_end - buf) & ~15;
        do {
            __m128i b16 = _mm_loadu_si128((void *)buf);
            int r = _mm_cmpestri(ranges16, ranges_size, b16, 16, _SIDD_LEAST_SIGNIFICANT | _SIDD_CMP_RANGES | _SIDD_UBYTE_OPS);
            if (unlikely(r != 16)) {
                buf += r;
                *found = 1;
                break;
            }
            buf += 16;
            left -= 16;
        } while (likely(left != 0));
    }
#else
    /* suppress unused parameter warning */
    (void)buf_end;
    (void)ranges;
    (void)ranges_size;
#endif
    return buf;
}

static inline const void *advance_token(const char **tok, size_t *toklen,
					const char *buf, const void *end,
					int *ret)
{
	const char *tok_start = buf;
	static const char ALIGNED(16) ranges2[] = "\000\040\177\177";
	int found;

	buf = findchar_fast(buf, end, ranges2, sizeof(ranges2) - 1, &found);
	if (!found) {
		if (!check_eof(buf, end, ret))
			return NULL;
	}
	while (1) {
		if (*buf == ' ') {
			break;
		} else if (unlikely(!IS_PRINTABLE_ASCII(*buf))) {
			if ((unsigned char)*buf < '\040' || *buf == '\177') {
				*ret = -1;
				return NULL;
			}
		}
		++buf;
		if (!check_eof(buf, end, ret))
			return NULL;
	}
	*tok = tok_start;
	*toklen = buf - tok_start;
	return buf;
}


static const char *get_token_to_eol(const char *buf, const char *buf_end,
				    const char **token, size_t *token_len,
				    int *ret)
{
	const char *token_start = buf;

#ifdef __SSE4_2__
	static const char ranges1[] = "\0\010"
		/* allow HT */
		"\012\037"
		/* allow SP and up to but not including DEL */
		"\177\177"
		/* allow chars w. MSB set */
		;
	int found;
	buf = findchar_fast(buf, buf_end, ranges1, sizeof(ranges1) - 1, &found);
	if (found)
		goto FOUND_CTL;
#else
	/*
	 * find non-printable char within the next 8 bytes, this is the
	 * hottest code; manually inlined
	 */
	while (likely(buf_end - buf >= 8)) {
#define DOIT()								\
		do {							\
			if (unlikely(!IS_PRINTABLE_ASCII(*buf)))	\
				goto NonPrintable;			\
			++buf;						\
		} while (0)
		DOIT();
		DOIT();
		DOIT();
		DOIT();
		DOIT();
		DOIT();
		DOIT();
		DOIT();
#undef DOIT
		continue;
	NonPrintable:
		if ((likely((unsigned char)*buf < '\040') &&
		     likely(*buf != '\011')) || unlikely(*buf == '\177')) {
			goto FOUND_CTL;
		}
		++buf;
	}
#endif
	for (;; ++buf) {
		if (!check_eof(buf, buf_end, ret))
			return NULL;
		if (unlikely(!IS_PRINTABLE_ASCII(*buf))) {
			if ((likely((unsigned char)*buf < '\040') &&
			     likely(*buf != '\011')) ||
			    unlikely(*buf == '\177')) {
				goto FOUND_CTL;
			}
		}
	}
FOUND_CTL:
	if (likely(*buf == '\015')) {
		++buf;
		if (!expect_char('\012', buf, buf_end, ret))
			return NULL;
		*token_len = buf - 2 - token_start;
	} else if (*buf == '\012') {
		*token_len = buf - token_start;
		++buf;
	} else {
		*ret = -1;
		return NULL;
	}
	*token = token_start;
	return buf;
}

static const char *is_complete(const char *buf, const char *buf_end,
			       size_t last_len, int *ret)
{
	int ret_cnt = 0;
	buf = last_len < 3 ? buf : buf + last_len - 3;

	while (1) {
		if (!check_eof(buf, buf_end, ret))
			return NULL;
		if (*buf == '\015') {
			++buf;
			if (!check_eof(buf, buf_end, ret))
				return NULL;
			if (!expect_char('\012', buf, buf_end, ret))
				return NULL;
			++ret_cnt;
		} else if (*buf == '\012') {
			++buf;
			++ret_cnt;
		} else {
			++buf;
			ret_cnt = 0;
		}
		if (ret_cnt == 2) {
			return buf;
		}
	}
	*ret = -2;
	return NULL;
}

/* *_buf is always within [buf, buf_end) upon success */
static const char *parse_int(const char *buf, const char *buf_end,
			     int *value, int *ret)
{
	int v;
	if (!check_eof(buf, buf_end, ret))
		return NULL;
	if (!('0' <= *buf && *buf <= '9')) {
		*ret = -1;
		return NULL;
	}
	v = 0;
	for (;; ++buf) {
		if (!check_eof(buf, buf_end, ret))
			return NULL;
		if ('0' <= *buf && *buf <= '9') {
			v = v * 10 + *buf - '0';
		} else {
			break;
		}
	}

	*value = v;
	return buf;
}

/* returned pointer is always within [buf, buf_end), or null */
static const char *parse_http_version(const char *buf, const char *buf_end,
				      int *minor_version, int *ret)
{
	char str[] = { 'H', 'T', 'T', 'P', '/', '1', '.', };
	int i;

	for (i = 0; i < sizeof(str); i++)
		if (!expect_char(str[i], buf, buf_end, ret))
			return NULL;
	return parse_int(buf, buf_end, minor_version, ret);
}

static const char *parse_headers(const char *buf, const char *buf_end, struct phr_header *headers, size_t *num_headers,
                                 size_t max_headers, int *ret)
{
    for (;; ++*num_headers) {
 if (!check_eof(buf, buf_end, ret))
	 return NULL;
        if (*buf == '\015') {
            ++buf;
{'fraction_non_alphanumeric': 0.17470881863560733, 'fraction_numerical': 0.05874824011263279, 'mean_word_length': 2.7322665392882732, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2184925', 'n_tokens_mistral': 7048, 'n_tokens_neox': 6362, 'n_words': 1664}