<commit_before>/******************************************************************************
 * This file is part of the MULA project
 * Copyright (c) 2011 <NAME> <<EMAIL>>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 */

#include "dictziplib.h"

#include <QtGlobal>

#include <QtCore/QDebug>>
#include <QtCore/QString>
#ifdef Q_OS_UNIX
#include <unistd.h>
#endif
#ifdef Q_OS_WIN32
#include <io.h>
#endif
#include <limits.h>
#include <fcntl.h>

#include <sys/stat.h>

#define USE_CACHE 1

#define BUFFERSIZE 10240

/*
 * Output buffer must be greater than or
 * equal to 110% of input buffer size, plus
 * 12 bytes. 
*/
#define OUT_BUFFER_SIZE 0xffffL

#define IN_BUFFER_SIZE ((unsigned long)((double)(OUT_BUFFER_SIZE - 12) * 0.89))

/* For gzip-compatible header, as defined in RFC 1952 */

/* Magic for GZIP (rfc1952)                */
#define GZ_MAGIC1     0x1f	/* First magic byte                        */
#define GZ_MAGIC2     0x8b	/* Second magic byte                       */

/* FLaGs (bitmapped), from rfc1952         */
enum Flags {
    GZ_FTEXT      = 0x01,	/* Set for ASCII text                      */
    GZ_FHCRC      = 0x02,	/* Header CRC16                            */
    GZ_FEXTRA     = 0x04,	/* Optional field (random access index)    */
    GZ_FNAME      = 0x08,	/* Original name                           */
    GZ_COMMENT    = 0x10,	/* Zero-terminated, human-readable comment */
};

enum ExtraFlags {
    GZ_MAX        =    2,	/* Maximum compression                     */
    GZ_FAST       =    4,	/* Fasted compression                      */
};

/* These are from rfc1952                  */
enum SystemType {
    GZ_OS_FAT        = 0,	    /* FAT filesystem (MS-DOS, OS/2, NT/Win32) */
    GZ_OS_AMIGA      = 1,	    /* Amiga                                   */
    GZ_OS_VMS        = 2,	    /* VMS (or OpenVMS)                        */
    GZ_OS_UNIX       = 3,       /* Unix                                    */
    GZ_OS_VMCMS      = 4,       /* VM/CMS                                  */
    GZ_OS_ATARI      = 5,       /* Atari TOS                               */
    GZ_OS_HPFS       = 6,       /* HPFS filesystem (OS/2, NT)              */
    GZ_OS_MAC        = 7,       /* Macintosh                               */
    GZ_OS_Z          = 8,       /* Z-System                                */
    GZ_OS_CPM        = 9,       /* CP/M                                    */
    GZ_OS_TOPS20     = 10,      /* TOPS-20                                 */
    GZ_OS_NTFS       = 11,      /* NTFS filesystem (NT)                    */
    GZ_OS_QDOS       = 12,      /* QDOS                                    */
    GZ_OS_ACORN      = 13,      /* Acorn RISCOS                            */
    GZ_OS_UNKNOWN    = 255,     /* unknown                                 */
};

#define GZ_RND_S1       'R'	    /* First magic for random access format    */
#define GZ_RND_S2       'A'	    /* Second magic for random access format   */

enum Fields {
    GZ_ID1           = 0,	/* GZ_MAGIC1                               */
    GZ_ID2           = 1,	/* GZ_MAGIC2                               */
    GZ_CM            = 2,	/* Compression Method (Z_DEFALTED)         */
    GZ_FLG	         = 3,	/* FLaGs (see above)                       */
    GZ_MTIME         = 4,	/* Modification TIME                       */
    GZ_XFL           = 8,	/* eXtra FLags (GZ_MAX or GZ_FAST)         */
    GZ_OS            = 9,	/* Operating System                        */
    GZ_XLEN          = 10,	/* eXtra LENgth (16bit)                    */
    GZ_FEXTRA_START  = 12,	/* Start of extra fields                   */
    GZ_SI1           = 12,	/* Subfield ID1                            */
    GZ_SI2           = 13,  /* Subfield ID2                            */
    GZ_SUBLEN        = 14,	/* Subfield length (16bit)                 */
    GZ_VERSION       = 16,  /* Version for subfield format             */
    GZ_CHUNKLEN      = 18,	/* Chunk length (16bit)                    */
    GZ_CHUNKCNT      = 20,	/* Number of chunks (16bit)                */
    GZ_RNDDATA       = 22,	/* Random access data (16bit)              */
};

enum DictionaryFormat {
    DICT_UNKNOWN    = 0,
    DICT_TEXT       = 1,
    DICT_GZIP       = 2,
    DICT_DZIP       = 3,<commit_msg>Add a TODO about the duplication gz_header_s and own types<commit_after>/******************************************************************************
 * This file is part of the MULA project
 * Copyright (c) 2011 <NAME> <<EMAIL>>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 */

#include "dictziplib.h"

#include <QtGlobal>

#include <QtCore/QDebug>>
#include <QtCore/QString>
#ifdef Q_OS_UNIX
#include <unistd.h>
#endif
#ifdef Q_OS_WIN32
#include <io.h>
#endif
#include <limits.h>
#include <fcntl.h>

#include <sys/stat.h>

#define USE_CACHE 1

#define BUFFERSIZE 10240

/*
 * Output buffer must be greater than or
 * equal to 110% of input buffer size, plus
 * 12 bytes. 
*/
#define OUT_BUFFER_SIZE 0xffffL

#define IN_BUFFER_SIZE ((unsigned long)((double)(OUT_BUFFER_SIZE - 12) * 0.89))

/* For gzip-compatible header, as defined in RFC 1952 */

/* Magic for GZIP (rfc1952)                */
#define GZ_MAGIC1     0x1f	/* First magic byte                        */
#define GZ_MAGIC2     0x8b	/* Second magic byte                       */

/* FLaGs (bitmapped), from rfc1952         */
enum Flags {
    GZ_FTEXT      = 0x01,	/* Set for ASCII text                      */
    GZ_FHCRC      = 0x02,	/* Header CRC16                            */
    GZ_FEXTRA     = 0x04,	/* Optional field (random access index)    */
    GZ_FNAME      = 0x08,	/* Original name                           */
    GZ_COMMENT    = 0x10,	/* Zero-terminated, human-readable comment */
};

enum ExtraFlags {
    GZ_MAX        =    2,	/* Maximum compression                     */
    GZ_FAST       =    4,	/* Fasted compression                      */
};

/* These are from rfc1952                  */
enum SystemType {
    GZ_OS_FAT        = 0,	    /* FAT filesystem (MS-DOS, OS/2, NT/Win32) */
    GZ_OS_AMIGA      = 1,	    /* Amiga                                   */
    GZ_OS_VMS        = 2,	    /* VMS (or OpenVMS)                        */
    GZ_OS_UNIX       = 3,       /* Unix                                    */
    GZ_OS_VMCMS      = 4,       /* VM/CMS                                  */
    GZ_OS_ATARI      = 5,       /* Atari TOS                               */
    GZ_OS_HPFS       = 6,       /* HPFS filesystem (OS/2, NT)              */
    GZ_OS_MAC        = 7,       /* Macintosh                               */
    GZ_OS_Z          = 8,       /* Z-System                                */
    GZ_OS_CPM        = 9,       /* CP/M                                    */
    GZ_OS_TOPS20     = 10,      /* TOPS-20                                 */
    GZ_OS_NTFS       = 11,      /* NTFS filesystem (NT)                    */
    GZ_OS_QDOS       = 12,      /* QDOS                                    */
    GZ_OS_ACORN      = 13,      /* Acorn RISCOS                            */
    GZ_OS_UNKNOWN    = 255,     /* unknown                                 */
};

#define GZ_RND_S1       'R'	    /* First magic for random access format    */
#define GZ_RND_S2       'A'	    /* Second magic for random access format   */

// TODO: gz_header_s usage from the zlib.h include
enum Fields {
    GZ_ID1           = 0,	/* GZ_MAGIC1                               */
    GZ_ID2           = 1,	/* GZ_MAGIC2                               */
    GZ_CM            = 2,	/* Compression Method (Z_DEFALTED)         */
    GZ_FLG	         = 3,	/* FLaGs (see above)                       */
    GZ_MTIME         = 4,	/* Modification TIME                       */
    GZ_XFL           = 8,	/* eXtra FLags (GZ_MAX or GZ_FAST)         */
    GZ_OS            = 9,	/* Operating System                        */
    GZ_XLEN          = 10,	/* eXtra LENgth (16bit)                    */
    GZ_FEXTRA_START  = 12,	/* Start of extra fields                   */
    GZ_SI1           = 12,	/* Subfield ID1                            */
    GZ_SI2           = 13,  /* Subfield ID2                            */
    GZ_SUBLEN        = 14,	/* Subfield length (16bit)                 */
    GZ_VERSION       = 16,  /* Version for subfield format             */
    GZ_CHUNKLEN      = 18,	/* Chunk length (16bit)                    */
    GZ_CHUNKCNT      = 20,	/* Number of chunks (16bit)                */
    GZ_RNDDATA       = 22,	/* Random access data (16bit)              */
};

enum DictionaryFormat {
    DICT_UNKNOWN    = 0,
    DICT_TEXT       = 1,
    DICT_GZIP       = 2,
    DICT_DZIP       = 3,
{'fraction_non_alphanumeric': 0.10640413683373111, 'fraction_numerical': 0.030827366746221163, 'mean_word_length': 1.238370799020699, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2618050', 'n_tokens_mistral': 3328, 'n_tokens_neox': 3020, 'n_words': 863}