<commit_before>/*********************************************************************
 * Copyright (c) 2011 <NAME>                                  *
 * All rights reserved.                                              *
 *                                                                   *
 * This software is licensed as described in the file COPYING, which *
 * you should have received as part of this distribution.            *
 *********************************************************************/

#include <errno.h>
#include <stdio.h>
#include <string.h>
#include <time.h>
#include <unistd.h>
#include <sys/time.h>
#include <sys/resource.h>
#include <Judy.h>
#include "buzhash.h"
#include "version.h"

#define BITMASK_HIGH63 0xfffffffffffffffeul

#define NGRAM_SIZE 5
#define DUPL_THRES 0.5
#define DOC_TAG "doc"
#define PAR_TAG "p"
#define TRIM_HASHES 64
#define MAX_STUB_LENGTH 20
#define BUFFER_SIZE 16777216

// options
int Ngram_size = NGRAM_SIZE;
float Dupl_thresh = DUPL_THRES;
char* Doc_tag = DOC_TAG;<commit_msg>Changed default n-gram size from 5 to 7.<commit_after>/*********************************************************************
 * Copyright (c) 2011 <NAME>                                  *
 * All rights reserved.                                              *
 *                                                                   *
 * This software is licensed as described in the file COPYING, which *
 * you should have received as part of this distribution.            *
 *********************************************************************/

#include <errno.h>
#include <stdio.h>
#include <string.h>
#include <time.h>
#include <unistd.h>
#include <sys/time.h>
#include <sys/resource.h>
#include <Judy.h>
#include "buzhash.h"
#include "version.h"

#define BITMASK_HIGH63 0xfffffffffffffffeul

#define NGRAM_SIZE 7
#define DUPL_THRES 0.5
#define DOC_TAG "doc"
#define PAR_TAG "p"
#define TRIM_HASHES 64
#define MAX_STUB_LENGTH 20
#define BUFFER_SIZE 16777216

// options
int Ngram_size = NGRAM_SIZE;
float Dupl_thresh = DUPL_THRES;
char* Doc_tag = DOC_TAG;
{'fraction_non_alphanumeric': 0.215414073719483, 'fraction_numerical': 0.02202010531354715, 'mean_word_length': 2.6666666666666665, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5577476', 'n_tokens_mistral': 646, 'n_tokens_neox': 574, 'n_words': 170}