<commit_before>/**
 * Copyright (c) 2016-present, <NAME>, <NAME>, Facebook, Inc.
 * All rights reserved.
 *
 * This source code is licensed under the BSD-style license found in the
 * LICENSE file in the root directory of this source tree. An additional grant
 * of patent rights can be found in the PATENTS file in the same directory.
 */

#ifndef UTIL_H_MODULE
#define UTIL_H_MODULE

#if defined (__cplusplus)
extern "C" {
#endif

/* **************************************
*  Compiler Options
****************************************/
#if defined(__INTEL_COMPILER)
#  pragma warning(disable : 177)    /* disable: message #177: function was declared but never referenced */
#endif
#if defined(_MSC_VER)
#  define _CRT_SECURE_NO_WARNINGS    /* Disable some Visual warning messages for fopen, strncpy */
#  define _CRT_SECURE_NO_DEPRECATE   /* VS2005 */
#  pragma warning(disable : 4127)    /* disable: C4127: conditional expression is constant */
#if _MSC_VER <= 1800                 /* (1800 = Visual Studio 2013) */
    #define snprintf sprintf_s       /* snprintf unsupported by Visual <= 2013 */
#endif
#endif


/* Unix Large Files support (>4GB) */
#if !defined(__LP64__)              /* No point defining Large file for 64 bit */
#   define _FILE_OFFSET_BITS 64     /* turn off_t into a 64-bit type for ftello, fseeko */
#   if defined(__sun__)             /* Sun Solaris 32-bits requires specific definitions */
#      define _LARGEFILE_SOURCE     /* fseeko, ftello */
#   else
#      define _LARGEFILE64_SOURCE   /* off64_t, fseeko64, ftello64 */
#   endif
#endif


/*-****************************************
*  Dependencies
******************************************/
#include <stdlib.h>     /* features.h with _POSIX_C_SOURCE, malloc */
#include <stdio.h>      /* fprintf */
#include <sys/types.h>  /* stat */
#include <sys/stat.h>   /* stat */
#include "mem.h"        /* U32, U64 */


/* *************************************
*  Constants
***************************************/
#define LIST_SIZE_INCREASE   (8*1024)


/*-****************************************<commit_msg>Don't redefine macro in util.h<commit_after>/**
 * Copyright (c) 2016-present, <NAME>, <NAME>, Facebook, Inc.
 * All rights reserved.
 *
 * This source code is licensed under the BSD-style license found in the
 * LICENSE file in the root directory of this source tree. An additional grant
 * of patent rights can be found in the PATENTS file in the same directory.
 */

#ifndef UTIL_H_MODULE
#define UTIL_H_MODULE

#if defined (__cplusplus)
extern "C" {
#endif

/* **************************************
*  Compiler Options
****************************************/
#if defined(__INTEL_COMPILER)
#  pragma warning(disable : 177)    /* disable: message #177: function was declared but never referenced */
#endif
#if defined(_MSC_VER)
#  define _CRT_SECURE_NO_WARNINGS    /* Disable some Visual warning messages for fopen, strncpy */
#  define _CRT_SECURE_NO_DEPRECATE   /* VS2005 */
#  pragma warning(disable : 4127)    /* disable: C4127: conditional expression is constant */
#if _MSC_VER <= 1800                 /* (1800 = Visual Studio 2013) */
    #define snprintf sprintf_s       /* snprintf unsupported by Visual <= 2013 */
#endif
#endif


/* Unix Large Files support (>4GB) */
#if !defined(__LP64__)                                  /* No point defining Large file for 64 bit */
#   define _FILE_OFFSET_BITS 64                         /* turn off_t into a 64-bit type for ftello, fseeko */
#   if defined(__sun__) && !defined(_LARGEFILE_SOURCE)  /* Sun Solaris 32-bits requires specific definitions */
#      define _LARGEFILE_SOURCE                         /* fseeko, ftello */
#   elif !defined(_LARGEFILE64_SOURCE)
#      define _LARGEFILE64_SOURCE                       /* off64_t, fseeko64, ftello64 */
#   endif
#endif


/*-****************************************
*  Dependencies
******************************************/
#include <stdlib.h>     /* features.h with _POSIX_C_SOURCE, malloc */
#include <stdio.h>      /* fprintf */
#include <sys/types.h>  /* stat */
#include <sys/stat.h>   /* stat */
#include "mem.h"        /* U32, U64 */


/* *************************************
*  Constants
***************************************/
#define LIST_SIZE_INCREASE   (8*1024)


/*-****************************************
{'fraction_non_alphanumeric': 0.22633649618143947, 'fraction_numerical': 0.031011339967600093, 'mean_word_length': 3.383367139959432, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7412414', 'n_tokens_mistral': 1380, 'n_tokens_neox': 1201, 'n_words': 384}