<commit_before>//
// Gigablast, Copyright March 2005
// Author: <NAME> <<EMAIL>>
//
// DMOZ RDF file parser into proprietary format
// See the "usage" note in the main function for usage and features.
// I apologize to anyone who must maintain or even simply read this code.
//

#include "gb-include.h"

#include <iostream>
#include <fstream>
#include "Url.h"
#include "HttpRequest.h"
#include "sort.h"

#undef malloc
#undef calloc
#undef realloc
bool closeAll ( void *state , void (* callback)(void *state) ) { return true; }
bool allExit ( ) { return true; };

bool sendPageSEO(TcpSocket *s, HttpRequest *hr) {return true;}

//int32_t g_qbufNeedSave = false;
//SafeBuf g_qbuf;

bool g_recoveryMode;

#define RDFBUFFER_SIZE    (1024*1024*10)
#define RDFSTRUCTURE_FILE "structure.rdf.u8"
#define RDFCONTENT_FILE   "content.rdf.u8"<commit_msg>fix dmozparse compiler error<commit_after>//
// Gigablast, Copyright March 2005
// Author: <NAME> <<EMAIL>>
//
// DMOZ RDF file parser into proprietary format
// See the "usage" note in the main function for usage and features.
// I apologize to anyone who must maintain or even simply read this code.
//

#include "gb-include.h"

#include <iostream>
#include <fstream>
#include "Url.h"
#include "HttpRequest.h"
#include "sort.h"

#undef malloc
#undef calloc
#undef realloc
bool closeAll ( void *state , void (* callback)(void *state) ) { return true; }
bool allExit ( ) { return true; };

bool sendPageSEO(TcpSocket *s, HttpRequest *hr) {return true;}

//int32_t g_qbufNeedSave = false;
//SafeBuf g_qbuf;

bool g_recoveryMode;

int g_inMemcpy;

#define RDFBUFFER_SIZE    (1024*1024*10)
#define RDFSTRUCTURE_FILE "structure.rdf.u8"
#define RDFCONTENT_FILE   "content.rdf.u8"
{'fraction_non_alphanumeric': 0.1252913752913753, 'fraction_numerical': 0.02097902097902098, 'mean_word_length': 4.450793650793651, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5060071', 'n_tokens_mistral': 625, 'n_tokens_neox': 594, 'n_words': 192}