<commit_before>/*************************************************************************
*    UrBackup - Client/Server backup system
*    Copyright (C) 2011-2018 <NAME>
*
*    This program is free software: you can redistribute it and/or modify
*    it under the terms of the GNU Affero General Public License as published by
*    the Free Software Foundation, either version 3 of the License, or
*    (at your option) any later version.
*
*    This program is distributed in the hope that it will be useful,
*    but WITHOUT ANY WARRANTY; without even the implied warranty of
*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
*    GNU Affero General Public License for more details.
*
*    You should have received a copy of the GNU Affero General Public License
*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
**************************************************************************/

#include <iostream>
#include <limits.h>
#include <fstream>
#include <stdint.h>
#include <string>

#include <map>
#include <vector>
#include <algorithm>
#include <memory>
#include <string.h>
#ifdef _WIN32
#include <fcntl.h>
#include <Windows.h>
#include <io.h>
#else
#include <sys/mman.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <fcntl.h>
#endif
#include <assert.h>
#ifdef BLOCKALIGN_USE_CRYPTOPP
#include "../cryptoplugin/cryptopp_inc.h"
#include "crc32c-adler.cpp"
#if (CRYPTOPP_VERSION < 564)
#include "crc.cpp"
#endif
#else
#include "crc32c-adler.h"
#include "crc.h"
#endif

#if defined(__FreeBSD__) || defined(__APPLE__)
#define lstat64 lstat
#define stat64 stat
#define open64 open
#endif

const unsigned int blocksize_min = 64;
const unsigned int blocksize_max = 1024;
const size_t c_buffer_reset = 20;
const unsigned int blocksize_avg = blocksize_min + (blocksize_max - blocksize_min) / 2 + sizeof(unsigned short);
const size_t hash_search_limit = 10000;
const size_t c_fit_off = sizeof(unsigned short);
const std::string magic = "BLOCKALIGN#1";
const unsigned int double_check_lim = 100 * 1024; //100kb
const size_t max_backlog_size = 10 * 1024 * 1024; //10MB

namespace
{
<commit_msg>Fix FreeBSD build<commit_after>/*************************************************************************
*    UrBackup - Client/Server backup system
*    Copyright (C) 2011-2018 <NAME>
*
*    This program is free software: you can redistribute it and/or modify
*    it under the terms of the GNU Affero General Public License as published by
*    the Free Software Foundation, either version 3 of the License, or
*    (at your option) any later version.
*
*    This program is distributed in the hope that it will be useful,
*    but WITHOUT ANY WARRANTY; without even the implied warranty of
*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
*    GNU Affero General Public License for more details.
*
*    You should have received a copy of the GNU Affero General Public License
*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
**************************************************************************/

#include <iostream>
#include <limits.h>
#include <fstream>
#include <stdint.h>
#include <string>

#include <map>
#include <vector>
#include <algorithm>
#include <memory>
#include <string.h>
#ifdef _WIN32
#include <fcntl.h>
#include <Windows.h>
#include <io.h>
#else
#include <sys/mman.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <fcntl.h>
#endif
#include <assert.h>
#ifdef BLOCKALIGN_USE_CRYPTOPP
#include "../cryptoplugin/cryptopp_inc.h"
#include "crc32c-adler.cpp"
#if (CRYPTOPP_VERSION < 564)
#include "crc.cpp"
#endif
#else
#include "crc32c-adler.h"
#include "crc.h"
#endif

#if defined(__FreeBSD__) || defined(__APPLE__)
#define lstat64 lstat
#define stat64 stat
#define fstat64 fstat
#define open64 open
#endif

const unsigned int blocksize_min = 64;
const unsigned int blocksize_max = 1024;
const size_t c_buffer_reset = 20;
const unsigned int blocksize_avg = blocksize_min + (blocksize_max - blocksize_min) / 2 + sizeof(unsigned short);
const size_t hash_search_limit = 10000;
const size_t c_fit_off = sizeof(unsigned short);
const std::string magic = "BLOCKALIGN#1";
const unsigned int double_check_lim = 100 * 1024; //100kb
const size_t max_backlog_size = 10 * 1024 * 1024; //10MB

namespace
{

{'fraction_non_alphanumeric': 0.16490340386384544, 'fraction_numerical': 0.028518859245630176, 'mean_word_length': 4.925068119891008, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2323096', 'n_tokens_mistral': 1466, 'n_tokens_neox': 1337, 'n_words': 477}