<commit_before>/*
	Onion HTTP server library
	Copyright (C) 2010 <NAME>

	This program is free software: you can redistribute it and/or modify
	it under the terms of the GNU Affero General Public License as
	published by the Free Software Foundation, either version 3 of the
	License, or (at your option) any later version.

	This program is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
	GNU Affero General Public License for more details.

	You should have received a copy of the GNU Affero General Public License
	along with this program.  If not, see <http://www.gnu.org/licenses/>.
	*/

#include <malloc.h>
#include <string.h>
#include <stdlib.h>
#include <libgen.h>
#include <ctype.h>
#include <fcntl.h>
#include <unistd.h>

#include "server.h"
#include "dict.h"
#include "request.h"
#include "response.h"
#include "handler.h"
#include "server.h"
#include "types_internal.h"
#include "codecs.h"
#include "log.h"

/// Known token types. This is merged with onion_connection_status as return value at token readers.
typedef enum{
	STRING=1001, // I start on 1000, to prevent collission with error codes
	KEY=1002,
	LINE=1003,
	NEW_LINE=1004,
	URLENCODE=1005,
	MULTIPART_BOUNDARY=1006,
	MULTIPART_END=1007,
	MULTIPART_NEXT=1008,
	STRING_NEW_LINE=1009,
}onion_token_token;

typedef struct onion_token_s{
	char str[256];
	off_t pos;
	
	char *extra; // Only used when need some previous data, like at header value, i need the key
	size_t extra_size;
}onion_token;

typedef struct onion_buffer_s{
	const char *data;
	size_t size;
	off_t pos;
}onion_buffer;

/**
 * @short This struct is mapped at token->extra. The token, data_start and data are maped to token->extra area too.
 * 
 * This is a bit juggling with pointers, but needed to keep the deallocation simple, and the token object minimal in most cases.
 * 
 * token->pos is the pointer to the free area and token->extra_size is the real size.
 */
typedef struct onion_multipart_buffer_s{
	char *boundary;
	size_t size;
	off_t pos;
	char *data;// When need a new buffered data (always from stream), get it from here. This advances as used.
	char *name; // point to a in data position
	char *filename; // point to a in data position
	size_t file_total_size;
	size_t post_total_size;
	int fd; // If file, the file descriptor.
}onion_multipart_buffer;

static onion_connection_status onion_request_process(onion_request *req);
static void onion_request_parse_query_to_dict(onion_dict *dict, char *p);
static int onion_request_parse_query(onion_request *req);
static onion_connection_status prepare_POST(onion_request *req);

/// Reads a string until a non-string char. Returns an onion_token
int token_read_STRING(onion_token *token, onion_buffer *data){
	if (data->pos>=data->size)
		return OCS_NEED_MORE_DATA;
	
	char c=data->data[data->pos++];
	while (!isspace(c)){
		token->str[token->pos++]=c;
		if (data->pos>=data->size)
			return OCS_NEED_MORE_DATA;
		if (token->pos>=(sizeof(token->str)-1)){
			ONION_ERROR("Token too long to parse it. Part read is %s (%d bytes)",token->str,token->pos);
			return OCS_INTERNAL_ERROR;
		}
		c=data->data[data->pos++];
	}
	int ret;
	if (c=='\n')
		ret=STRING_NEW_LINE;
	else
		ret=STRING;
	
	token->str[token->pos]='\0';
	token->pos=0;
	//ONION_DEBUG0("Found STRING token %s",token->str);
	return ret;
}

/**
 * @short Reads a string until a delimiter is found. Returns an onion_token. Also detects an empty line.
 * 
 * @returns error code | STRING | STRING_NEW_LINE | NEW_LINE
 * 
 * STRING is a delimited string, string new line is that the string finished witha  new line, new line is that I fuond only a new line char.
 */
///
int token_read_until(onion_token *token, onion_buffer *data, char delimiter){
	if (data->pos>=data->size)
		return OCS_NEED_MORE_DATA;
	
	//ONION_DEBUG0("Read data %d bytes, at token pos %d",data->size-data->pos, token->pos);
	
	char c=data->data[data->pos++];
	while (c!=delimiter && c!='\n'){
		if (c!='\r') // Just ignore it here
			token->str[token->pos++]=c;
		if (data->pos>=data->size){
			return OCS_NEED_MORE_DATA;
		}
		if (token->pos>=(sizeof(token->str)-1)){
			ONION_ERROR("Token too long to parse it. Part read is %s (%d bytes)",token->str,token->pos);
			return OCS_INTERNAL_ERROR;
		}
		c=data->data[data->pos++];
	}
	int ret=STRING;
	token->str[token->pos]='\0';
	if (c!=delimiter){
		if ( token->pos==0 && c=='\n' ){
			ret=NEW_LINE;
		}
		else{ // only option left.
			ret=STRING_NEW_LINE;
		}
	}
	token->pos=0;
	
	//ONION_DEBUG0("Found KEY token %s",token->str);
	return ret;
}

/// Reads a key, that is a string ended with ':'.
int token_read_KEY(onion_token *token, onion_buffer *data){
	int res=token_read_until(token, data, ':');
	if (res==STRING)
		return KEY;
	if (res==STRING_NEW_LINE){
		ONION_ERROR("When parsing header, found a non valid delimited string token: '%s'",token->str);
		return OCS_INTERNAL_ERROR;
	}
	return res;
}

/// Reads a string until a '\n|\r\n' is found. Returns an onion_token.
int token_read_LINE(onion_token *token, onion_buffer *data){
	if (data->pos>=data->size)
		return OCS_NEED_MORE_DATA;
	
	char c=data->data[data->pos++];
	while (c!='\n'){
		token->str[token->pos++]=c;
		if (data->pos>=data->size)
			return OCS_NEED_MORE_DATA;
		if (token->pos>=(sizeof(token->str)-1)){
			ONION_ERROR("Token too long to parse it. Part read is %s (%d bytes)",token->str,token->pos);
			return OCS_INTERNAL_ERROR;
		}
		c=data->data[data->pos++];
	}
	if (token->str[token->pos-1]=='\r')
		token->str[token->pos-1]='\0';
	else
		token->str[token->pos]='\0';
	token->pos=0;
	
	//ONION_DEBUG0("Found LINE token %s",token->str);
	return LINE;
}

/// Reads a string until a '\n|\r\n' is found. Returns an onion_token.
int token_read_URLENCODE(onion_token *token, onion_buffer *data){
	int l=data->size-data->pos;
	if (l > (token->extra_size-token->pos))
		l=token->extra_size-token->pos;
	ONION_DEBUG("Feed %d bytes",l);
	
	memcpy(&token->extra[token->pos], &data->data[data->pos], l);
	token->pos+=l;
	data->size-=l;
	
	if (token->extra_size==token->pos){
		token->extra[token->pos]='\0';
		return URLENCODE;
	}
	return OCS_NEED_MORE_DATA;
}

/// Reads as much as possible from the boundary. 
int token_read_MULTIPART_BOUNDARY(onion_token *token, onion_buffer *data){
	onion_multipart_buffer *multipart=(onion_multipart_buffer*)token->extra;
	while (multipart->boundary[multipart->pos] && multipart->boundary[multipart->pos]==data->data[data->pos]){
		multipart->pos++;
		data->pos++;
		if (data->pos>=data->size)
			return OCS_NEED_MORE_DATA;
	}
	if (multipart->boundary[multipart->pos]){
		ONION_ERROR("Expecting multipart boundary, but not here (pos %d) (%c!=%c)", multipart->pos, data->data[data->pos], multipart->boundary[multipart->pos]);
		return OCS_INTERNAL_ERROR;
	}
	return MULTIPART_BOUNDARY;
}

/// Reads as much as possible from the boundary. 
int token_read_MULTIPART_next(onion_token *token, onion_buffer *data){
	//onion_multipart_buffer *multipart=(onion_multipart_buffer*)token->extra;
	while (token->pos<2 && (data->size-data->pos)>0){
		token->str[token->pos++]=data->data[data->pos++];
		if (token->pos==1 && token->str[0]=='\n'){
			token->pos=0;
			return MULTIPART_NEXT;
		}
	}
	//ONION_DEBUG("pos %d, %d %d (%c %c)",token->pos, token->str[0], token->str[1], token->str[0], token->str[1]);
	if (token->pos==2){
		token->pos=0;
		if (token->str[0]=='-' && token->str[1]=='-')
			return MULTIPART_END;
		if (token->str[0]=='\r' && token->str[1]=='\n')
			return MULTIPART_NEXT;
		return OCS_INTERNAL_ERROR;
	}
	
	return OCS_NEED_MORE_DATA;
}

int token_read_NEW_LINE(onion_token *token, onion_buffer *data){
	while (token->pos<2 && (data->size-data->pos)>0){
		token->str[token->pos++]=data->data[data->pos++];
		if (token->pos==1 && token->str[0]=='\n'){
			token->pos=0;
			return NEW_LINE;
		}
	}
	//ONION_DEBUG("pos %d, %d %d (%c %c)",token->pos, token->str[0], token->str[1], token->str[0], token->str[1]);
	int res=OCS_NEED_MORE_DATA;
	if (token->pos==2){
		token->pos=0;
		if (token->str[0]=='\r' && token->str[1]=='\n')
			res=NEW_LINE;
		else
			res=OCS_INTERNAL_ERROR;
	}
	token->pos=0;
	
	return res;
}

static onion_connection_status parse_POST_multipart_next(onion_request *req, onion_buffer *data);


/**
 * Hard parser as I must set into the file as I read, until i found the boundary token (or start), try to parse, and if fail, 
 * write to the file.
 * 
 * The boundary may be in two or N parts.
 */
static onion_connection_status parse_POST_multipart_file(onion_request *req, onion_buffer *data){
	onion_token *token=req->parser_data;
	onion_multipart_buffer *multipart=(onion_multipart_buffer*)token->extra;
	const char *p=data->data+data->pos;
	for (;data->pos<data->size;data->pos++){
		//ONION_DEBUG("*p %d boundary %d (%s)",*p,multipart->boundary[multipart->pos],multipart->boundary);
		if (multipart->pos==0 && *p=='\n') // \r is optional.
			multipart->pos=1;
		if (*p==multipart->boundary[multipart->pos]){
			multipart->pos++;
			if (multipart->pos==multipart->size){
				multipart->pos=0;
				data->pos++; // Not sure why this is needed. FIXME.
				close(multipart->fd);
				req->parser=parse_POST_multipart_next;
				return OCS_NEED_MORE_DATA;
			}
		}
		else{
			if (multipart->file_total_size>req->server->max_file_size){
				ONION_ERROR("Files on this post too big. Aborting.");
				return OCS_INTERNAL_ERROR;
			}
			if (multipart->pos!=0){
				multipart->file_total_size+=multipart->pos;
				int w=write(multipart->fd, multipart->boundary, multipart->pos);
				if (w!=multipart->pos){
					ONION_ERROR("Error writing multipart data to file. Check permissions on temp directory, and availabe disk.");
					close(multipart->fd);
					return OCS_INTERNAL_ERROR;
				}
				multipart->pos=0;
			}
			int w=write(multipart->fd,p,1); // SLOW!! FIXME.
			if (w!=1){
				ONION_ERROR("Error writing multipart data to file. Check permissions on temp directory, and availabe disk.");
				close(multipart->fd);
				return OCS_INTERNAL_ERROR;
			}

		}
		multipart->file_total_size++;
		p++;
	}
	return OCS_NEED_MORE_DATA;
}

/**
 * Hard parser as I must set into the file as I read, until i found the boundary token (or start), try to parse, and if fail, 
 * write to the file.
 * 
 * The boundary may be in two or N parts.
 */
static onion_connection_status parse_POST_multipart_data(onion_request *req, onion_buffer *data){
	onion_token *token=req->parser_data;
	onion_multipart_buffer *multipart=(onion_multipart_buffer*)token->extra;
	const char *p=data->data+data->pos;
	char *d=&multipart->data[token->pos];
<commit_msg>Fixed bug at POST files where it was writing \r\n on single \n found.<commit_after>/*
	Onion HTTP server library
	Copyright (C) 2010 <NAME>

	This program is free software: you can redistribute it and/or modify
	it under the terms of the GNU Affero General Public License as
	published by the Free Software Foundation, either version 3 of the
	License, or (at your option) any later version.

	This program is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
	GNU Affero General Public License for more details.

	You should have received a copy of the GNU Affero General Public License
	along with this program.  If not, see <http://www.gnu.org/licenses/>.
	*/

#include <malloc.h>
#include <string.h>
#include <stdlib.h>
#include <libgen.h>
#include <ctype.h>
#include <fcntl.h>
#include <unistd.h>

#include "server.h"
#include "dict.h"
#include "request.h"
#include "response.h"
#include "handler.h"
#include "server.h"
#include "types_internal.h"
#include "codecs.h"
#include "log.h"

/// Known token types. This is merged with onion_connection_status as return value at token readers.
typedef enum{
	STRING=1001, // I start on 1000, to prevent collission with error codes
	KEY=1002,
	LINE=1003,
	NEW_LINE=1004,
	URLENCODE=1005,
	MULTIPART_BOUNDARY=1006,
	MULTIPART_END=1007,
	MULTIPART_NEXT=1008,
	STRING_NEW_LINE=1009,
}onion_token_token;

typedef struct onion_token_s{
	char str[256];
	off_t pos;
	
	char *extra; // Only used when need some previous data, like at header value, i need the key
	size_t extra_size;
}onion_token;

typedef struct onion_buffer_s{
	const char *data;
	size_t size;
	off_t pos;
}onion_buffer;

/**
 * @short This struct is mapped at token->extra. The token, data_start and data are maped to token->extra area too.
 * 
 * This is a bit juggling with pointers, but needed to keep the deallocation simple, and the token object minimal in most cases.
 * 
 * token->pos is the pointer to the free area and token->extra_size is the real size.
 */
typedef struct onion_multipart_buffer_s{
	char *boundary;
	size_t size;
	off_t pos;
	off_t startpos;
	char *data;// When need a new buffered data (always from stream), get it from here. This advances as used.
	char *name; // point to a in data position
	char *filename; // point to a in data position
	size_t file_total_size;
	size_t post_total_size;
	int fd; // If file, the file descriptor.
}onion_multipart_buffer;

static onion_connection_status onion_request_process(onion_request *req);
static void onion_request_parse_query_to_dict(onion_dict *dict, char *p);
static int onion_request_parse_query(onion_request *req);
static onion_connection_status prepare_POST(onion_request *req);

/// Reads a string until a non-string char. Returns an onion_token
int token_read_STRING(onion_token *token, onion_buffer *data){
	if (data->pos>=data->size)
		return OCS_NEED_MORE_DATA;
	
	char c=data->data[data->pos++];
	while (!isspace(c)){
		token->str[token->pos++]=c;
		if (data->pos>=data->size)
			return OCS_NEED_MORE_DATA;
		if (token->pos>=(sizeof(token->str)-1)){
			ONION_ERROR("Token too long to parse it. Part read is %s (%d bytes)",token->str,token->pos);
			return OCS_INTERNAL_ERROR;
		}
		c=data->data[data->pos++];
	}
	int ret;
	if (c=='\n')
		ret=STRING_NEW_LINE;
	else
		ret=STRING;
	
	token->str[token->pos]='\0';
	token->pos=0;
	//ONION_DEBUG0("Found STRING token %s",token->str);
	return ret;
}

/**
 * @short Reads a string until a delimiter is found. Returns an onion_token. Also detects an empty line.
 * 
 * @returns error code | STRING | STRING_NEW_LINE | NEW_LINE
 * 
 * STRING is a delimited string, string new line is that the string finished witha  new line, new line is that I fuond only a new line char.
 */
///
int token_read_until(onion_token *token, onion_buffer *data, char delimiter){
	if (data->pos>=data->size)
		return OCS_NEED_MORE_DATA;
	
	//ONION_DEBUG0("Read data %d bytes, at token pos %d",data->size-data->pos, token->pos);
	
	char c=data->data[data->pos++];
	while (c!=delimiter && c!='\n'){
		if (c!='\r') // Just ignore it here
			token->str[token->pos++]=c;
		if (data->pos>=data->size){
			return OCS_NEED_MORE_DATA;
		}
		if (token->pos>=(sizeof(token->str)-1)){
			ONION_ERROR("Token too long to parse it. Part read is %s (%d bytes)",token->str,token->pos);
			return OCS_INTERNAL_ERROR;
		}
		c=data->data[data->pos++];
	}
	int ret=STRING;
	token->str[token->pos]='\0';
	if (c!=delimiter){
		if ( token->pos==0 && c=='\n' ){
			ret=NEW_LINE;
		}
		else{ // only option left.
			ret=STRING_NEW_LINE;
		}
	}
	token->pos=0;
	
	//ONION_DEBUG0("Found KEY token %s",token->str);
	return ret;
}

/// Reads a key, that is a string ended with ':'.
int token_read_KEY(onion_token *token, onion_buffer *data){
	int res=token_read_until(token, data, ':');
	if (res==STRING)
		return KEY;
	if (res==STRING_NEW_LINE){
		ONION_ERROR("When parsing header, found a non valid delimited string token: '%s'",token->str);
		return OCS_INTERNAL_ERROR;
	}
	return res;
}

/// Reads a string until a '\n|\r\n' is found. Returns an onion_token.
int token_read_LINE(onion_token *token, onion_buffer *data){
	if (data->pos>=data->size)
		return OCS_NEED_MORE_DATA;
	
	char c=data->data[data->pos++];
	while (c!='\n'){
		token->str[token->pos++]=c;
		if (data->pos>=data->size)
			return OCS_NEED_MORE_DATA;
		if (token->pos>=(sizeof(token->str)-1)){
			ONION_ERROR("Token too long to parse it. Part read is %s (%d bytes)",token->str,token->pos);
			return OCS_INTERNAL_ERROR;
		}
		c=data->data[data->pos++];
	}
	if (token->str[token->pos-1]=='\r')
		token->str[token->pos-1]='\0';
	else
		token->str[token->pos]='\0';
	token->pos=0;
	
	//ONION_DEBUG0("Found LINE token %s",token->str);
	return LINE;
}

/// Reads a string until a '\n|\r\n' is found. Returns an onion_token.
int token_read_URLENCODE(onion_token *token, onion_buffer *data){
	int l=data->size-data->pos;
	if (l > (token->extra_size-token->pos))
		l=token->extra_size-token->pos;
	ONION_DEBUG("Feed %d bytes",l);
	
	memcpy(&token->extra[token->pos], &data->data[data->pos], l);
	token->pos+=l;
	data->size-=l;
	
	if (token->extra_size==token->pos){
		token->extra[token->pos]='\0';
		return URLENCODE;
	}
	return OCS_NEED_MORE_DATA;
}

/// Reads as much as possible from the boundary. 
int token_read_MULTIPART_BOUNDARY(onion_token *token, onion_buffer *data){
	onion_multipart_buffer *multipart=(onion_multipart_buffer*)token->extra;
	while (multipart->boundary[multipart->pos] && multipart->boundary[multipart->pos]==data->data[data->pos]){
		multipart->pos++;
		data->pos++;
		if (data->pos>=data->size)
			return OCS_NEED_MORE_DATA;
	}
	if (multipart->boundary[multipart->pos]){
		ONION_ERROR("Expecting multipart boundary, but not here (pos %d) (%c!=%c)", multipart->pos, data->data[data->pos], multipart->boundary[multipart->pos]);
		return OCS_INTERNAL_ERROR;
	}
	return MULTIPART_BOUNDARY;
}

/// Reads as much as possible from the boundary. 
int token_read_MULTIPART_next(onion_token *token, onion_buffer *data){
	//onion_multipart_buffer *multipart=(onion_multipart_buffer*)token->extra;
	while (token->pos<2 && (data->size-data->pos)>0){
		token->str[token->pos++]=data->data[data->pos++];
		if (token->pos==1 && token->str[0]=='\n'){
			token->pos=0;
			return MULTIPART_NEXT;
		}
	}
	//ONION_DEBUG("pos %d, %d %d (%c %c)",token->pos, token->str[0], token->str[1], token->str[0], token->str[1]);
	if (token->pos==2){
		token->pos=0;
		if (token->str[0]=='-' && token->str[1]=='-')
			return MULTIPART_END;
		if (token->str[0]=='\r' && token->str[1]=='\n')
			return MULTIPART_NEXT;
		return OCS_INTERNAL_ERROR;
	}
	
	return OCS_NEED_MORE_DATA;
}

int token_read_NEW_LINE(onion_token *token, onion_buffer *data){
	while (token->pos<2 && (data->size-data->pos)>0){
		token->str[token->pos++]=data->data[data->pos++];
		if (token->pos==1 && token->str[0]=='\n'){
			token->pos=0;
			return NEW_LINE;
		}
	}
	//ONION_DEBUG("pos %d, %d %d (%c %c)",token->pos, token->str[0], token->str[1], token->str[0], token->str[1]);
	int res=OCS_NEED_MORE_DATA;
	if (token->pos==2){
		token->pos=0;
		if (token->str[0]=='\r' && token->str[1]=='\n')
			res=NEW_LINE;
		else
			res=OCS_INTERNAL_ERROR;
	}
	token->pos=0;
	
	return res;
}

static onion_connection_status parse_POST_multipart_next(onion_request *req, onion_buffer *data);


/**
 * Hard parser as I must set into the file as I read, until i found the boundary token (or start), try to parse, and if fail, 
 * write to the file.
 * 
 * The boundary may be in two or N parts.
 */
static onion_connection_status parse_POST_multipart_file(onion_request *req, onion_buffer *data){
	onion_token *token=req->parser_data;
	onion_multipart_buffer *multipart=(onion_multipart_buffer*)token->extra;
	const char *p=data->data+data->pos;
	for (;data->pos<data->size;data->pos++){
		//ONION_DEBUG("*p %d boundary %d (%s)",*p,multipart->boundary[multipart->pos],multipart->boundary);
		if (multipart->pos==0 && *p=='\n') // \r is optional.
			multipart->startpos=multipart->pos=1;
		if (*p==multipart->boundary[multipart->pos]){
			multipart->pos++;
			if (multipart->pos==multipart->size){
				multipart->startpos=multipart->pos=0;
				data->pos++; // Not sure why this is needed. FIXME.
				close(multipart->fd);
				req->parser=parse_POST_multipart_next;
				return OCS_NEED_MORE_DATA;
			}
		}
		else{
			if (multipart->file_total_size>req->server->max_file_size){
				ONION_ERROR("Files on this post too big. Aborting.");
				return OCS_INTERNAL_ERROR;
			}
			if (multipart->pos!=0){
				multipart->file_total_size+=multipart->pos;
				int r=multipart->pos-multipart->startpos;
				ONION_DEBUG0("Write %d bytes",r);
				int w=write(multipart->fd, multipart->boundary+multipart->startpos, r);
				if (w!=r){
					ONION_ERROR("Error writing multipart data to file. Check permissions on temp directory, and availabe disk.");
					close(multipart->fd);
					return OCS_INTERNAL_ERROR;
				}
				multipart->pos=0;
				data->pos--; // Ignore read charater, try again. May be start of boundary.
				continue;
			}
			ONION_DEBUG0("Write 1 byte");
			int w=write(multipart->fd,p,1); // SLOW!! FIXME.
			if (w!=1){
				ONION_ERROR("Error writing multipart data to file. Check permissions on temp directory, and availabe disk.");
				close(multipart->fd);
				return OCS_INTERNAL_ERROR;
			}

		}
		multipart->file_total_size++;
		p++;
	}
	return OCS_NEED_MORE_DATA;
}

/**
 * Hard parser as I must set into the file as I read, until i found the boundary token (or start), try to parse, and if fail, 
 * write to the file.
 * 
 * The boundary may be in two or N parts.
 */
static onion_connection_status parse_POST_multipart_data(onion_request *req, onion_buffer *data){
	onion_token *token=req->parser_data;
	onion_multipart_buffer *multipart=(onion_multipart_buffer*)token->extra;
	const char *p=data->data+data->pos;
	char *d=&multipart->data[token->pos];

{'fraction_non_alphanumeric': 0.15472726432959216, 'fraction_numerical': 0.009560759318276293, 'mean_word_length': 4.434738955823293, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 423, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6784852', 'n_tokens_mistral': 8483, 'n_tokens_neox': 7798, 'n_words': 2045}