<commit_before>/*************************************************************************/
/* Copyright (c) 2004                                                    */
/* <NAME>, <NAME>, and <NAME>                    */
/* Copyright (c) 2009-2013 <NAME>                                 */
/* All rights reserved                                                   */
/*                                                                       */
/* Use of the link grammar parsing system is subject to the terms of the */
/* license set forth in the LICENSE file included with this software.    */
/* This license allows free redistribution and use in source and binary  */
/* forms, with or without modification, subject to certain conditions.   */
/*                                                                       */
/*************************************************************************/
#ifndef _LINK_GRAMMAR_UTILITIES_H_
#define _LINK_GRAMMAR_UTILITIES_H_

/* The _Win32 definitions are for native-Windows compilers.  This includes
 * MSVC (only versions >=14 are supported) and MINGW (under MSYS or Cygwin).
 * The _WIN32 definitions are not for Cygwin, which doesn't define _WIN32. */

#include <ctype.h>
#include <stdio.h>
#ifdef _WIN32
#define _CRT_RAND_S
#endif /* _WIN32 */
#include <stdlib.h>
#include <string.h>
#include <wchar.h>
#include <wctype.h>
#include <locale.h>
#ifdef HAVE_LOCALE_T_IN_XLOCALE_H
#include <xlocale.h>
#endif /* HAVE_LOCALE_T_IN_XLOCALE_H */

#include "error.h"
#include "lg_assert.h"

#ifdef HAVE_ALLOCA_H
# include <alloca.h>
#elif defined __GNUC__
#ifndef alloca
# define alloca __builtin_alloca
#endif /* !alloca */
#elif defined _AIX
# define alloca __alloca
#elif defined _MSC_VER
# include <malloc.h>
# define alloca _alloca
#else
# include <stddef.h>
# ifdef  __cplusplus
extern "C"
# endif
void *alloca (size_t);
#endif

#ifndef TLS
#ifdef _MSC_VER
#define TLS __declspec(thread)
#else
#define TLS
#endif /* _MSC_VER */
#endif /* !TLS */

#ifndef strdupa
/* In the following, the argument should not have side effects. */
#define strdupa(s) strcpy(alloca(strlen(s)+1), s)
#endif

/* Windows, POSIX and GNU have different ideas about thread-safe strerror(). */
#ifdef _WIN32
#define strerror_r(errno, buf, len) strerror_s(buf, len, errno)
#else
#ifdef _GNU_SOURCE
/* Emulate the POSIX version; assuming len>0 and a successful call. */
#define strerror_r(errno, buf, len) abs((strcpy(buf, strerror_r (errno, buf, len)), 0))
#endif /* _GNU_SOURCE */
#endif /* _WIN32 */

#ifdef _MSC_VER
/* These definitions are incorrect, as these functions are different(!)
 * (non-standard functionality).
 * See http://stackoverflow.com/questions/27754492 . Fortunately,
 * MSVC 14 supports C99 functions so these definitions are now unneeded.
 * (LG library compilation is unsupported for previous MSVC versions.)
 * (Left here for documentation.) */
#if 0
/* MS Visual C uses non-standard string function names */
#define snprintf _snprintf
#define vsnprintf _vsnprintf
#endif

/* Avoid plenty of: warning C4090: 'function': different 'const' qualifiers.
 * This happens, for example, when the argument is "const void **". */
#define free(x) free((void *)x)
#define realloc(x, s) realloc((void *)x, s)
#define memcpy(x, y, s) memcpy((void *)x, (void *)y, s)
#define qsort(x, y, z, w) qsort((void *)x, y, z, w)
#endif /* _MSC_VER */

#if defined(HAVE_LOCALE_T_IN_LOCALE_H) || defined(HAVE_LOCALE_T_IN_XLOCALE_H)
#define HAVE_LOCALE_T 1
#endif /* HAVE_LOCALE_T_IN_LOCALE_H || HAVE_LOCALE_T_IN_XLOCALE_H) */

#ifdef _WIN32
#include <windows.h>
#include <mbctype.h>

/* Compatibility definitions. */
#ifndef strncasecmp
#define strncasecmp(a,b,s) strnicmp((a),(b),(s))
#endif
/* Note that "#define _CRT_RAND_S" is needed before "#include <stdlib.h>" */
#define rand_r(seedp) rand_s(seedp)

/* No strtok_s in these versions and their strtok_r is incompatible. */
#if _WINVER != 0x501 /* XP */ && _WINVER != 0x502 /* Server 2003 */
#define strtok_r strtok_s
#define HAVE_STRTOK_R
#endif

#ifndef __MINGW32__
/* There is no ssize_t definition in native Windows. */
#include <BaseTsd.h>
typedef SSIZE_T ssize_t;
#endif

/* Native windows has locale_t, and hence HAVE_LOCALE_T is defined here.
 * However, MinGW currently doesn't have locale_t. If/when it has locale_t,<commit_msg>MinGW: Avoid a duplicate definition of HAVE_STRTOK_R<commit_after>/*************************************************************************/
/* Copyright (c) 2004                                                    */
/* <NAME>, <NAME>, and <NAME>                    */
/* Copyright (c) 2009-2013 <NAME>                                 */
/* All rights reserved                                                   */
/*                                                                       */
/* Use of the link grammar parsing system is subject to the terms of the */
/* license set forth in the LICENSE file included with this software.    */
/* This license allows free redistribution and use in source and binary  */
/* forms, with or without modification, subject to certain conditions.   */
/*                                                                       */
/*************************************************************************/
#ifndef _LINK_GRAMMAR_UTILITIES_H_
#define _LINK_GRAMMAR_UTILITIES_H_

/* The _Win32 definitions are for native-Windows compilers.  This includes
 * MSVC (only versions >=14 are supported) and MINGW (under MSYS or Cygwin).
 * The _WIN32 definitions are not for Cygwin, which doesn't define _WIN32. */

#include <ctype.h>
#include <stdio.h>
#ifdef _WIN32
#define _CRT_RAND_S
#endif /* _WIN32 */
#include <stdlib.h>
#include <string.h>
#include <wchar.h>
#include <wctype.h>
#include <locale.h>
#ifdef HAVE_LOCALE_T_IN_XLOCALE_H
#include <xlocale.h>
#endif /* HAVE_LOCALE_T_IN_XLOCALE_H */

#include "error.h"
#include "lg_assert.h"

#ifdef HAVE_ALLOCA_H
# include <alloca.h>
#elif defined __GNUC__
#ifndef alloca
# define alloca __builtin_alloca
#endif /* !alloca */
#elif defined _AIX
# define alloca __alloca
#elif defined _MSC_VER
# include <malloc.h>
# define alloca _alloca
#else
# include <stddef.h>
# ifdef  __cplusplus
extern "C"
# endif
void *alloca (size_t);
#endif

#ifndef TLS
#ifdef _MSC_VER
#define TLS __declspec(thread)
#else
#define TLS
#endif /* _MSC_VER */
#endif /* !TLS */

#ifndef strdupa
/* In the following, the argument should not have side effects. */
#define strdupa(s) strcpy(alloca(strlen(s)+1), s)
#endif

/* Windows, POSIX and GNU have different ideas about thread-safe strerror(). */
#ifdef _WIN32
#define strerror_r(errno, buf, len) strerror_s(buf, len, errno)
#else
#ifdef _GNU_SOURCE
/* Emulate the POSIX version; assuming len>0 and a successful call. */
#define strerror_r(errno, buf, len) abs((strcpy(buf, strerror_r (errno, buf, len)), 0))
#endif /* _GNU_SOURCE */
#endif /* _WIN32 */

#ifdef _MSC_VER
/* These definitions are incorrect, as these functions are different(!)
 * (non-standard functionality).
 * See http://stackoverflow.com/questions/27754492 . Fortunately,
 * MSVC 14 supports C99 functions so these definitions are now unneeded.
 * (LG library compilation is unsupported for previous MSVC versions.)
 * (Left here for documentation.) */
#if 0
/* MS Visual C uses non-standard string function names */
#define snprintf _snprintf
#define vsnprintf _vsnprintf
#endif

/* Avoid plenty of: warning C4090: 'function': different 'const' qualifiers.
 * This happens, for example, when the argument is "const void **". */
#define free(x) free((void *)x)
#define realloc(x, s) realloc((void *)x, s)
#define memcpy(x, y, s) memcpy((void *)x, (void *)y, s)
#define qsort(x, y, z, w) qsort((void *)x, y, z, w)
#endif /* _MSC_VER */

#if defined(HAVE_LOCALE_T_IN_LOCALE_H) || defined(HAVE_LOCALE_T_IN_XLOCALE_H)
#define HAVE_LOCALE_T 1
#endif /* HAVE_LOCALE_T_IN_LOCALE_H || HAVE_LOCALE_T_IN_XLOCALE_H) */

#ifdef _WIN32
#include <windows.h>
#include <mbctype.h>

/* Compatibility definitions. */
#ifndef strncasecmp
#define strncasecmp(a,b,s) strnicmp((a),(b),(s))
#endif
/* Note that "#define _CRT_RAND_S" is needed before "#include <stdlib.h>" */
#define rand_r(seedp) rand_s(seedp)

#ifndef __MINGW32__
/* No strtok_s in XP/2003 and their strtok_r is incompatible.
 * Hence HAVE_STRTOK_R will not be defined and our own one will be used. */
#if _WINVER != 0x501 /* XP */ && _WINVER != 0x502 /* Server 2003 */
#define strtok_r strtok_s
#define HAVE_STRTOK_R
#endif /* _WINVER != XP|2003 */
#endif

#ifndef __MINGW32__
/* There is no ssize_t definition in native Windows. */
#include <BaseTsd.h>
typedef SSIZE_T ssize_t;
#endif

/* Native windows has locale_t, and hence HAVE_LOCALE_T is defined here.
 * However, MinGW currently doesn't have locale_t. If/when it has locale_t,
{'fraction_non_alphanumeric': 0.14666666666666667, 'fraction_numerical': 0.015954415954415956, 'mean_word_length': 3.4256177508825014, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2849620', 'n_tokens_mistral': 2951, 'n_tokens_neox': 2750, 'n_words': 912}