<commit_before>cmake_minimum_required(VERSION 3.2)
# GOOD TO KNOW
# - to check which flag gcc are used, do make with VERBOSE=1

# FLAGS
## might add debug flag -fdump-class-hierarchy for investigating vtables entries
## might add --ggdb4  -static-libgcc -static-libstdc++
# -D_GNU_SOURCE
set(COMPILER_FLAG_STANDARD "-O3 -std=c++1z -DNDEBUG -D_REENTRANT -fPIC -m64")
set(COMPILER_FLAG_WARNING "-Wall -Wextra -Wno-deprecated-declarations -Wno-unused-local-typedefs -Wno-unused-parameter -fmessage-length=0")
if (CMAKE_COMPILER_IS_GNUCXX)
        # might add -fno-exceptions -flto=8 -fno-fat-lto-objects -mfpmath=sse -fomit-frame-pointer
        # -fomit-frame-pointer: get one more register available for the register allocator, and save 1-3 instructions on entry/exit to functions, not supported by all architecture, should not omit for measurement (e.g. vtune)
        set(COMPILER_FLAG_PERF "-fdevirtualize-at-ltrans -floop-interchange -floop-strip-mine -floop-block -fgraphite-identity -funsafe-loop-optimizations -march=native")
else()
        set(COMPILER_FLAG_PERF "-march=native")
endif()

set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMPILER_FLAG_STANDARD} ${COMPILER_FLAG_WARNING} ${COMPILER_FLAG_PERF}")
# !FLAGS

set(PATH_FLAT_HASH_MAP "${CMAKE_CURRENT_SOURCE_DIR}/external/flat_hash_map")
set(PATH_GOOGLE_BENCHMARK "${CMAKE_CURRENT_SOURCE_DIR}/external/benchmark")
set(PATH_GOOGLE_TEST "${CMAKE_CURRENT_SOURCE_DIR}/external/googletest")
set(PATH_LIB_EXPERIMENTAL "${CMAKE_CURRENT_SOURCE_DIR}/experimental/experimental")

set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)

add_subdirectory(experimental/experimental)
add_subdirectory(experimental/tests)
add_subdirectory(experimental/benchmarks)<commit_msg>fix travis<commit_after>cmake_minimum_required(VERSION 3.2)
# GOOD TO KNOW
# - to check which flag gcc are used, do make with VERBOSE=1

# FLAGS
## might add debug flag -fdump-class-hierarchy for investigating vtables entries
## might add --ggdb4  -static-libgcc -static-libstdc++
# -D_GNU_SOURCE
set(COMPILER_FLAG_STANDARD "-O3 -std=c++1z -DNDEBUG -D_REENTRANT -fPIC -m64")
set(COMPILER_FLAG_WARNING "-Wall -Wextra -Wno-deprecated-declarations -Wno-unused-local-typedefs -Wno-unused-parameter -fmessage-length=0")
if (CMAKE_COMPILER_IS_GNUCXX)
        # might add -fno-exceptions -flto=8 -fno-fat-lto-objects -mfpmath=sse -fomit-frame-pointer
        # -fomit-frame-pointer: get one more register available for the register allocator, and save 1-3 instructions on entry/exit to functions, not supported by all architecture, should not omit for measurement (e.g. vtune)
        set(COMPILER_FLAG_PERF "-fdevirtualize-at-ltrans -floop-interchange -floop-strip-mine -floop-block -fgraphite-identity -funsafe-loop-optimizations -march=native")
else()
        set(COMPILER_FLAG_PERF "-march=native")
endif()

set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMPILER_FLAG_STANDARD} ${COMPILER_FLAG_WARNING} ${COMPILER_FLAG_PERF}")

option(TRAVIS_BUILD "Disable libstdc++ ABI tests" ON)

if (${TRAVIS_BUILD})
    add_definitions(-DTRAVIS_BUILD)
endif()
# !FLAGS

set(PATH_FLAT_HASH_MAP "${CMAKE_CURRENT_SOURCE_DIR}/external/flat_hash_map")
set(PATH_GOOGLE_BENCHMARK "${CMAKE_CURRENT_SOURCE_DIR}/external/benchmark")
set(PATH_GOOGLE_TEST "${CMAKE_CURRENT_SOURCE_DIR}/external/googletest")
set(PATH_LIB_EXPERIMENTAL "${CMAKE_CURRENT_SOURCE_DIR}/experimental/experimental")

set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)

add_subdirectory(experimental/experimental)
add_subdirectory(experimental/tests)
add_subdirectory(experimental/benchmarks)
{'fraction_non_alphanumeric': 0.11081932773109243, 'fraction_numerical': 0.0063025210084033615, 'mean_word_length': 6.396116504854369, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2074743', 'n_tokens_mistral': 1434, 'n_tokens_neox': 1349, 'n_words': 250}