<commit_before>/* shred.h -- types for shred similarity tools */

#include <sys/types.h>
#include <netinet/in.h>

#ifndef LARGEFILES
/*
 * 65,536 lines should be enough, but make it possible to compile with 32 bits.
 * The point of making it a short is that the data sets can get quite large,
 * so we want the hash_t and sorthash_t structures to be small.
 */
typedef u_int16_t	linenum_t;
#define TONET		htons
#define FROMNET		ntohs
#else
typedef u_int32_t	linenum_t;
#define TONET		htonl
#define FROMNET		ntohl
#endif
#define UNIQUE_FLAG	(linenum_t)-1	/* 2s-complement assumption */

/* use this to hold total line count of the entire source tree set */
typedef u_int32_t	linecount_t;

#define HASHSIZE	16
struct hash_t
{
    linenum_t   	start, end;
    unsigned char	hash[HASHSIZE];
};
#define HASHCMP(s, t)	memcmp((s)->hash.hash, (t)->hash.hash, HASHSIZE)

struct filehdr_t
{
    char	*name;
    linenum_t	length;
    struct filehdr_t *next;
};

struct sorthash_t
{
    struct hash_t	hash;
    struct filehdr_t	*file;
};

/* control bits, meant to be set at startup */
extern int remove_braces;
extern int remove_whitespace;
extern int remove_comments;<commit_msg>Remove a cohesion.<commit_after>/* shred.h -- types for shred similarity tools */

#include <sys/types.h>
#include <netinet/in.h>

#ifndef LARGEFILES
/*
 * 65,536 lines should be enough, but make it possible to compile with 32 bits.
 * The point of making it a short is that the data sets can get quite large,
 * so we want the hash_t and sorthash_t structures to be small.
 */
typedef u_int16_t	linenum_t;
#define TONET		htons
#define FROMNET		ntohs
#else
typedef u_int32_t	linenum_t;
#define TONET		htonl
#define FROMNET		ntohl
#endif
#define MAX_LINENUM	(linenum_t)-1	/* 2s-complement assumption */
#define UNIQUE_FLAG	MAX_LINENUM	/* 2s-complement assumption */

/* use this to hold total line count of the entire source tree set */
typedef u_int32_t	linecount_t;

#define HASHSIZE	16
struct hash_t
{
    linenum_t   	start, end;
    unsigned char	hash[HASHSIZE];
};
#define HASHCMP(s, t)	memcmp((s)->hash.hash, (t)->hash.hash, HASHSIZE)

struct filehdr_t
{
    char	*name;
    linenum_t	length;
    struct filehdr_t *next;
};

struct sorthash_t
{
    struct hash_t	hash;
    struct filehdr_t	*file;
};

/* control bits, meant to be set at startup */
extern int remove_braces;
extern int remove_whitespace;
extern int remove_comments;
{'fraction_non_alphanumeric': 0.08935993349958438, 'fraction_numerical': 0.014546965918536992, 'mean_word_length': 3.9731404958677685, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6548367', 'n_tokens_mistral': 923, 'n_tokens_neox': 867, 'n_words': 298}