<commit_before>/*
 * $Id$
 */

#include <assert.h>
#include <stdio.h>
#include <ctype.h>
#include <inttypes.h>
#include <unistd.h>
#include <string.h>
#include <stdlib.h>
#include <fcntl.h>
#include <pthread.h>
#include <sys/queue.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netdb.h>
#include <sbuf.h>
#include <event.h>

#include "libvarnish.h"
#include "shmlog.h"
#include "cache.h"

/*
 * Chunked encoding is a hack.  We prefer to have a single or a few
 * large storage objects, and a terribly long list of small ones.
 * If our stevedore can trim, we alloc big chunks and trim the last one
 * at the end know the result.
 *
 * Good testcase: http://www.washingtonpost.com/
 */
#define CHUNK_PREALLOC		(128 * 1024)

/*--------------------------------------------------------------------*/
static int
fetch_straight(struct worker *w, struct sess *sp, int fd, struct http *hp, char *b)
{
	int i;
	char *e;
	unsigned char *p;
	off_t	cl;
	struct storage *st;

	cl = strtoumax(b, NULL, 0);

	st = stevedore->alloc(stevedore, cl);
	TAILQ_INSERT_TAIL(&sp->obj->store, st, list);
	st->len = cl;
	sp->obj->len = cl;
	p = st->ptr;

	i = fcntl(fd, F_GETFL);		/* XXX ? */
	i &= ~O_NONBLOCK;
	i = fcntl(fd, F_SETFL, i);

	if (http_GetTail(hp, cl, &b, &e)) {
		i = e - b;
		memcpy(p, b, i);
		p += i;
		cl -= i;
	}

	while (cl != 0) {
		i = read(fd, p, cl);
		assert(i > 0);
		p += i;
		cl -= i;
	}
	return (0);
}

/*--------------------------------------------------------------------*/
/* XXX: Cleanup.  It must be possible somehow :-( */

static int
fetch_chunked(struct worker *w, struct sess *sp, int fd, struct http *hp)
{
	int i;
	char *b, *q, *e;
	unsigned char *p;
	struct storage *st;
	unsigned u, v;
	char buf[20];
	char *bp, *be;

	i = fcntl(fd, F_GETFL);		/* XXX ? */
	i &= ~O_NONBLOCK;
	i = fcntl(fd, F_SETFL, i);

	be = buf + sizeof buf;
	bp = buf;
	st = NULL;
	while (1) {
		if (http_GetTail(hp, be - bp, &b, &e)) {
			memcpy(bp, b, e - b);
			bp += e - b;
			*bp = '\0';
		} else {
			i = read(fd, bp, be - bp);
			assert(i >= 0);
			bp += i;
			*bp = '\0';
		}
		u = strtoul(buf, &q, 16);
		if (q == NULL || q == buf)
			continue;
		assert(isspace(*q));
		while (*q == '\t' || *q == ' ')
			q++;
		if (*q == '\r')
			q++;
		assert(*q == '\n');
		q++;
		if (u == 0)
			break;
		sp->obj->len += u;

		while (u > 0) {
			if (st != NULL && st->len < st->space) {
				p = st->ptr + st->len;
			} else {
				st = stevedore->alloc(stevedore,
				    stevedore->trim == NULL ? u : CHUNK_PREALLOC);
				TAILQ_INSERT_TAIL(&sp->obj->store, st, list);
				p = st->ptr;
			}
			v = st->space - st->len;
			if (v > u)
				v = u;

			i = bp - q;
			if (i == 0) {
			} else if (v > i) {
				assert(i > 0);
				memcpy(p, q, i);
				p += i;
				st->len += i;
				u -= i;
				v -= i;
				q = bp = buf;
			} else if (i >= v) {
				memcpy(p, q, v);
				p += v;
				st->len += i;
				q += v;
				u -= v;
				v -= v;
				if (u == 0 && bp > q) {
					memcpy(buf, q, bp - q);
					q = bp = buf + (bp - q);
				}
			}
			if (u == 0)
				break;
			if (v == 0)
				continue;
			if (http_GetTail(hp, v, &b, &e)) {
				memcpy(p, b, e - b);
				p += e - b;
				st->len += e - b;
				v -= e - b;
				u -= e - b;
				sp->obj->len += e - b;
			}
			while (v > 0) {
				i = read(fd, p, v);
				assert(i > 0);
				st->len += i;
				v -= i;
				u -= i;
				p += i;
				sp->obj->len += i;
			}
		}
	}

	if (st != NULL && stevedore->trim != NULL)
		stevedore->trim(st, st->len);
	return (0);
}


/*--------------------------------------------------------------------*/

#include <errno.h>

static int
fetch_eof(struct worker *w, struct sess *sp, int fd, struct http *hp)
{
	int i;
	char *b, *e;
	unsigned char *p;
	struct storage *st;
	unsigned v;

	i = fcntl(fd, F_GETFL);		/* XXX ? */
	i &= ~O_NONBLOCK;<commit_msg>Fix object length double accounting in chunked fetch<commit_after>/*
 * $Id$
 */

#include <assert.h>
#include <stdio.h>
#include <ctype.h>
#include <inttypes.h>
#include <unistd.h>
#include <string.h>
#include <stdlib.h>
#include <fcntl.h>
#include <pthread.h>
#include <sys/queue.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netdb.h>
#include <sbuf.h>
#include <event.h>

#include "libvarnish.h"
#include "shmlog.h"
#include "cache.h"

/*
 * Chunked encoding is a hack.  We prefer to have a single or a few
 * large storage objects, and a terribly long list of small ones.
 * If our stevedore can trim, we alloc big chunks and trim the last one
 * at the end know the result.
 *
 * Good testcase: http://www.washingtonpost.com/
 */
#define CHUNK_PREALLOC		(128 * 1024)

/*--------------------------------------------------------------------*/
static int
fetch_straight(struct worker *w, struct sess *sp, int fd, struct http *hp, char *b)
{
	int i;
	char *e;
	unsigned char *p;
	off_t	cl;
	struct storage *st;

	cl = strtoumax(b, NULL, 0);

	st = stevedore->alloc(stevedore, cl);
	TAILQ_INSERT_TAIL(&sp->obj->store, st, list);
	st->len = cl;
	sp->obj->len = cl;
	p = st->ptr;

	i = fcntl(fd, F_GETFL);		/* XXX ? */
	i &= ~O_NONBLOCK;
	i = fcntl(fd, F_SETFL, i);

	if (http_GetTail(hp, cl, &b, &e)) {
		i = e - b;
		memcpy(p, b, i);
		p += i;
		cl -= i;
	}

	while (cl != 0) {
		i = read(fd, p, cl);
		assert(i > 0);
		p += i;
		cl -= i;
	}
	return (0);
}

/*--------------------------------------------------------------------*/
/* XXX: Cleanup.  It must be possible somehow :-( */

static int
fetch_chunked(struct worker *w, struct sess *sp, int fd, struct http *hp)
{
	int i;
	char *b, *q, *e;
	unsigned char *p;
	struct storage *st;
	unsigned u, v;
	char buf[20];
	char *bp, *be;

	i = fcntl(fd, F_GETFL);		/* XXX ? */
	i &= ~O_NONBLOCK;
	i = fcntl(fd, F_SETFL, i);

	be = buf + sizeof buf;
	bp = buf;
	st = NULL;
	while (1) {
		if (http_GetTail(hp, be - bp, &b, &e)) {
			memcpy(bp, b, e - b);
			bp += e - b;
			*bp = '\0';
		} else {
			i = read(fd, bp, be - bp);
			assert(i >= 0);
			bp += i;
			*bp = '\0';
		}
		u = strtoul(buf, &q, 16);
		if (q == NULL || q == buf)
			continue;
		assert(isspace(*q));
		while (*q == '\t' || *q == ' ')
			q++;
		if (*q == '\r')
			q++;
		assert(*q == '\n');
		q++;
		if (u == 0)
			break;
		sp->obj->len += u;

		while (u > 0) {
			if (st != NULL && st->len < st->space) {
				p = st->ptr + st->len;
			} else {
				st = stevedore->alloc(stevedore,
				    stevedore->trim == NULL ? u : CHUNK_PREALLOC);
				TAILQ_INSERT_TAIL(&sp->obj->store, st, list);
				p = st->ptr;
			}
			v = st->space - st->len;
			if (v > u)
				v = u;

			i = bp - q;
			if (i == 0) {
			} else if (v > i) {
				assert(i > 0);
				memcpy(p, q, i);
				p += i;
				st->len += i;
				u -= i;
				v -= i;
				q = bp = buf;
			} else if (i >= v) {
				memcpy(p, q, v);
				p += v;
				st->len += i;
				q += v;
				u -= v;
				v -= v;
				if (u == 0 && bp > q) {
					memcpy(buf, q, bp - q);
					q = bp = buf + (bp - q);
				}
			}
			if (u == 0)
				break;
			if (v == 0)
				continue;
			if (http_GetTail(hp, v, &b, &e)) {
				memcpy(p, b, e - b);
				p += e - b;
				st->len += e - b;
				v -= e - b;
				u -= e - b;
			}
			while (v > 0) {
				i = read(fd, p, v);
				assert(i > 0);
				st->len += i;
				v -= i;
				u -= i;
				p += i;
			}
		}
	}

	if (st != NULL && stevedore->trim != NULL)
		stevedore->trim(st, st->len);
	return (0);
}


/*--------------------------------------------------------------------*/

#include <errno.h>

static int
fetch_eof(struct worker *w, struct sess *sp, int fd, struct http *hp)
{
	int i;
	char *b, *e;
	unsigned char *p;
	struct storage *st;
	unsigned v;

	i = fcntl(fd, F_GETFL);		/* XXX ? */
	i &= ~O_NONBLOCK;
{'fraction_non_alphanumeric': 0.24904517318582906, 'fraction_numerical': 0.007638614513367575, 'mean_word_length': 1.888550779764169, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 113, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5274789', 'n_tokens_mistral': 3691, 'n_tokens_neox': 3211, 'n_words': 943}