<commit_before># specify the standard configuration here
# 1. set the variable you want to use and assign it a default value
#    make sure you specify the datatype and the keyword CACHE to
#    tell cmake that it needs to store the value in its cache
# 2. print out a message showing the status of your variable
# 3. add your configuration parameter to include/zorba/config.h.cmake
#    for example for string variables use
#    #define ZORBA_BATCHING_TYPE    ${ZORBA_BATCHING_TYPE}
#    ${VARIABLE} is replaced by cmake using the value
#    that is set for this variable
#    for booleans (e.g. SET (ZORBA_VARIABLE OFF CACHE BOOL "zorba variable")), use
#    #cmakedefine VARIABLE
# 4. if you want to change your variable value, execute
#    cmake -DVARIABLE=new_value
#    the variable will keep this value (i.e. store it in the cache file CMakeCache.txt)
#    until you set a new one or edit the CMakeCache.txt file

# how should batching in the runtime system be done
SET(ZORBA_BATCHING_TYPE 0 CACHE STRING 
    "0 = NO_BATCHING, 1 = SIMPLE_BATCHING, 2 = SUPER_BATCHING") 
MESSAGE( STATUS "ZORBA_BATCHING_TYPE:      " ${ZORBA_BATCHING_TYPE} )

# number of items to batch if ZORBA_BATCHING_TYPE is SIMPLE_BATCHING or SUPER_BATCHING
IF (ZORBA_BATCHING_TYPE GREATER 0)
    SET(ZORBA_BATCHING_BATCHSIZE 100 CACHE STRING 
        "the batchsize used if batching is used")
    MESSAGE( STATUS "ZORBA_BATCHING_BATCHSIZE: " ${ZORBA_BATCHING_BATCHSIZE} )
ENDIF (ZORBA_BATCHING_TYPE GREATER 0)

# float point precision
SET(ZORBA_FLOAT_POINT_PRECISION 18 CACHE STRING
   "number of decimal places in floating point serialization")
MESSAGE( STATUS "ZORBA_FLOAT_POINT_PRECISION: " ${ZORBA_FLOAT_POINT_PRECISION} )


#specify the threading mechanism used
#for WIN32 you don't have to specify it
# one valid thread options must be present for any non win32 platform
IF (NOT DEFINED WIN32)
  SET(ZORBA_THREAD "pthread" CACHE STRING "thread mechanism")
  MESSAGE(STATUS "ZORBA_THREAD [pthread/boost/singlethread]: " ${ZORBA_THREAD})

  IF(${ZORBA_THREAD} STREQUAL "pthread")
  	SET(ZORBA_USE_PTHREAD_LIBRARY	ON )
  ENDIF(${ZORBA_THREAD} STREQUAL "pthread")
  MESSAGE( STATUS "ZORBA_USE_PTHREAD_LIBRARY:      " ${ZORBA_USE_PTHREAD_LIBRARY} )

  IF(${ZORBA_THREAD} STREQUAL "boost")
  	SET(ZORBA_USE_BOOST_THREAD_LIBRARY	 ON )
  ENDIF(${ZORBA_THREAD} STREQUAL "boost")
  MESSAGE( STATUS "ZORBA_USE_BOOST_THREAD_LIBRARY: " ${ZORBA_USE_BOOST_THREAD_LIBRARY} )

  IF(${ZORBA_THREAD} STREQUAL "singlethread")
  	SET(ZORBA_FOR_ONE_THREAD_ONLY	 ON )
  ENDIF(${ZORBA_THREAD} STREQUAL "singlethread")
  MESSAGE( STATUS "ZORBA_FOR_ONE_THREAD_ONLY:      " ${ZORBA_FOR_ONE_THREAD_ONLY} )

  IF (NOT DEFINED ZORBA_USE_PTHREAD_LIBRARY AND NOT DEFINED ZORBA_FOR_ONE_THREAD_ONLY AND NOT DEFINED ZORBA_USE_BOOST_THREAD_LIBRARY)
    MESSAGE( FATAL_ERROR "Unsupported thread system. For non WIN32 systems set ZORBA_THREAD=pthread, boost, or singlethread using the cmake -D ZORBA_THREAD=... option")
  ENDIF (NOT DEFINED ZORBA_USE_PTHREAD_LIBRARY AND NOT DEFINED ZORBA_FOR_ONE_THREAD_ONLY AND NOT DEFINED ZORBA_USE_BOOST_THREAD_LIBRARY)

ENDIF (NOT DEFINED WIN32) 


SET(ZORBA_BUILD_STATIC_LIBRARY OFF CACHE BOOL "build a static library, e.g. when creating a release")
MESSAGE(STATUS "ZORBA_BUILD_STATIC_LIBRARY:     " ${ZORBA_BUILD_STATIC_LIBRARY})

# flags for debugging the parser
SET(ZORBA_DEBUG_PARSER OFF CACHE BOOL "debug the parser")
MESSAGE(STATUS "ZORBA_DEBUG_PARSER:             " ${ZORBA_DEBUG_PARSER})     
<commit_msg>fix for cygwin<commit_after># specify the standard configuration here
# 1. set the variable you want to use and assign it a default value
#    make sure you specify the datatype and the keyword CACHE to
#    tell cmake that it needs to store the value in its cache
# 2. print out a message showing the status of your variable
# 3. add your configuration parameter to include/zorba/config.h.cmake
#    for example for string variables use
#    #define ZORBA_BATCHING_TYPE    ${ZORBA_BATCHING_TYPE}
#    ${VARIABLE} is replaced by cmake using the value
#    that is set for this variable
#    for booleans (e.g. SET (ZORBA_VARIABLE OFF CACHE BOOL "zorba variable")), use
#    #cmakedefine VARIABLE
# 4. if you want to change your variable value, execute
#    cmake -DVARIABLE=new_value
#    the variable will keep this value (i.e. store it in the cache file CMakeCache.txt)
#    until you set a new one or edit the CMakeCache.txt file

# how should batching in the runtime system be done
SET(ZORBA_BATCHING_TYPE 0 CACHE STRING 
    "0 = NO_BATCHING, 1 = SIMPLE_BATCHING, 2 = SUPER_BATCHING") 
MESSAGE( STATUS "ZORBA_BATCHING_TYPE:      " ${ZORBA_BATCHING_TYPE} )

# number of items to batch if ZORBA_BATCHING_TYPE is SIMPLE_BATCHING or SUPER_BATCHING
IF (ZORBA_BATCHING_TYPE GREATER 0)
    SET(ZORBA_BATCHING_BATCHSIZE 100 CACHE STRING 
        "the batchsize used if batching is used")
    MESSAGE( STATUS "ZORBA_BATCHING_BATCHSIZE: " ${ZORBA_BATCHING_BATCHSIZE} )
ENDIF (ZORBA_BATCHING_TYPE GREATER 0)

# float point precision
SET(ZORBA_FLOAT_POINT_PRECISION 18 CACHE STRING
   "number of decimal places in floating point serialization")
MESSAGE( STATUS "ZORBA_FLOAT_POINT_PRECISION: " ${ZORBA_FLOAT_POINT_PRECISION} )


#specify the threading mechanism used
#for WIN32 you don't have to specify it
# one valid thread options must be present for any non win32 platform
IF (DEFINED UNIX)
  SET(ZORBA_THREAD "pthread" CACHE STRING "thread mechanism")
  MESSAGE(STATUS "ZORBA_THREAD [pthread/boost/singlethread]: " ${ZORBA_THREAD})

  IF(${ZORBA_THREAD} STREQUAL "pthread")
  	SET(ZORBA_USE_PTHREAD_LIBRARY	ON )
  ENDIF(${ZORBA_THREAD} STREQUAL "pthread")
  MESSAGE( STATUS "ZORBA_USE_PTHREAD_LIBRARY:      " ${ZORBA_USE_PTHREAD_LIBRARY} )

  IF(${ZORBA_THREAD} STREQUAL "boost")
  	SET(ZORBA_USE_BOOST_THREAD_LIBRARY	 ON )
  ENDIF(${ZORBA_THREAD} STREQUAL "boost")
  MESSAGE( STATUS "ZORBA_USE_BOOST_THREAD_LIBRARY: " ${ZORBA_USE_BOOST_THREAD_LIBRARY} )

  IF(${ZORBA_THREAD} STREQUAL "singlethread")
  	SET(ZORBA_FOR_ONE_THREAD_ONLY	 ON )
  ENDIF(${ZORBA_THREAD} STREQUAL "singlethread")
  MESSAGE( STATUS "ZORBA_FOR_ONE_THREAD_ONLY:      " ${ZORBA_FOR_ONE_THREAD_ONLY} )

  IF (NOT DEFINED ZORBA_USE_PTHREAD_LIBRARY AND NOT DEFINED ZORBA_FOR_ONE_THREAD_ONLY AND NOT DEFINED ZORBA_USE_BOOST_THREAD_LIBRARY)
    MESSAGE( FATAL_ERROR "Unsupported thread system. For non WIN32 systems set ZORBA_THREAD=pthread, boost, or singlethread using the cmake -D ZORBA_THREAD=... option")
  ENDIF (NOT DEFINED ZORBA_USE_PTHREAD_LIBRARY AND NOT DEFINED ZORBA_FOR_ONE_THREAD_ONLY AND NOT DEFINED ZORBA_USE_BOOST_THREAD_LIBRARY)

ENDIF (DEFINED UNIX) 


SET(ZORBA_BUILD_STATIC_LIBRARY OFF CACHE BOOL "build a static library, e.g. when creating a release")
MESSAGE(STATUS "ZORBA_BUILD_STATIC_LIBRARY:     " ${ZORBA_BUILD_STATIC_LIBRARY})

# flags for debugging the parser
SET(ZORBA_DEBUG_PARSER OFF CACHE BOOL "debug the parser")
MESSAGE(STATUS "ZORBA_DEBUG_PARSER:             " ${ZORBA_DEBUG_PARSER})     

{'fraction_non_alphanumeric': 0.0698240091572471, 'fraction_numerical': 0.006581771354986408, 'mean_word_length': 5.0, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4973120', 'n_tokens_mistral': 2627, 'n_tokens_neox': 2454, 'n_words': 728}