<commit_before>set(CMAKE_INCLUDE_CURRENT_DIR ON)
IF(APPLE)
  set(CMAKE_CXX_FLAGS "-std=c++11 -stdlib=libc++")
ELSE()
  set(CMAKE_CXX_FLAGS "-std=gnu++0x")
ENDIF()

set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
include(public-headers)
include(core)
include(async)
include(objc)
include(LibFindMacros)

IF(APPLE)
  set(additional_includes
    "${CMAKE_CURRENT_SOURCE_DIR}/../Externals/ctemplate-osx/include"
    "${CMAKE_CURRENT_SOURCE_DIR}/../Externals/uchardet-osx/include"
    "${CMAKE_CURRENT_SOURCE_DIR}/../Externals/libetpan-osx/include"
    /usr/include/tidy
    /usr/include/libxml2
  )
  
  set(additional_lib_searchpath
    "${CMAKE_CURRENT_SOURCE_DIR}/../Externals/ctemplate-osx/lib"
    "${CMAKE_CURRENT_SOURCE_DIR}/../Externals/uchardet-osx/lib"
    "${CMAKE_CURRENT_SOURCE_DIR}/../Externals/libetpan-osx/lib"
  )
  
  execute_process(COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/../scripts/get-mac.sh"
    WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/../scripts")
ELSE()
  set(additional_includes
    /usr/include/libxml2
  )
ENDIF()

# detect ctemplate

find_path(CTEMPLATE_INCLUDE_DIR
  NAMES ctemplate/template.h
  PATHS ${additional_includes}
)
find_library(CTEMPLATE_LIBRARY
  NAMES ctemplate
  PATHS ${additional_lib_searchpath}
)

if(NOT CTEMPLATE_INCLUDE_DIR OR NOT CTEMPLATE_LIBRARY)
  message(FATAL_ERROR "ERROR: Could not find ctemplate")
else()
  message(STATUS "Found ctemplate")
endif()

# detect uchardet

find_path(UCHARDET_INCLUDE_DIR
  NAMES uchardet/uchardet.h
  PATHS ${additional_includes}
)
find_library(UCHARDET_LIBRARY
  NAMES uchardet
  PATHS ${additional_lib_searchpath}
)

if(NOT UCHARDET_INCLUDE_DIR OR NOT UCHARDET_LIBRARY)
  message(FATAL_ERROR "ERROR: Could not find uchardet")
else()
  message(STATUS "Found uchardet")
endif()

if(NOT APPLE)
  
  # detect icu

  find_path(ICU4C_INCLUDE_DIR
    NAMES unicode/utf8.h
    PATHS ${additional_includes}
  )
  find_library(ICU_LIBRARY
    NAMES icuuc
    PATHS ${additional_lib_searchpath}
  )
<commit_msg>Fixed typo in CMakefile<commit_after>set(CMAKE_INCLUDE_CURRENT_DIR ON)
IF(APPLE)
  set(CMAKE_CXX_FLAGS "-std=c++11 -stdlib=libc++")
ELSE()
  set(CMAKE_CXX_FLAGS "-std=gnu++0x")
ENDIF()

set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
include(public-headers)
include(core)
include(async)
include(objc)
include(LibFindMacros)

IF(APPLE)
  set(additional_includes
    "${CMAKE_CURRENT_SOURCE_DIR}/../Externals/ctemplate-osx/include"
    "${CMAKE_CURRENT_SOURCE_DIR}/../Externals/uchardet-osx/include"
    "${CMAKE_CURRENT_SOURCE_DIR}/../Externals/libetpan-osx/include"
    /usr/include/tidy
    /usr/include/libxml2
  )
  
  set(additional_lib_searchpath
    "${CMAKE_CURRENT_SOURCE_DIR}/../Externals/ctemplate-osx/lib"
    "${CMAKE_CURRENT_SOURCE_DIR}/../Externals/uchardet-osx/lib"
    "${CMAKE_CURRENT_SOURCE_DIR}/../Externals/libetpan-osx/lib"
  )
  
  execute_process(COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/../scripts/get-mac.sh"
    WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/../scripts")
ELSE()
  set(additional_includes
    /usr/include/libxml2
  )
ENDIF()

# detect ctemplate

find_path(CTEMPLATE_INCLUDE_DIR
  NAMES ctemplate/template.h
  PATHS ${additional_includes}
)
find_library(CTEMPLATE_LIBRARY
  NAMES ctemplate
  PATHS ${additional_lib_searchpath}
)

if(NOT CTEMPLATE_INCLUDE_DIR OR NOT CTEMPLATE_LIBRARY)
  message(FATAL_ERROR "ERROR: Could not find ctemplate")
else()
  message(STATUS "Found ctemplate")
endif()

# detect uchardet

find_path(UCHARDET_INCLUDE_DIR
  NAMES uchardet/uchardet.h
  PATHS ${additional_includes}
)
find_library(UCHARDET_LIBRARY
  NAMES uchardet
  PATHS ${additional_lib_searchpath}
)

if(NOT UCHARDET_INCLUDE_DIR OR NOT UCHARDET_LIBRARY)
  message(FATAL_ERROR "ERROR: Could not find uchardet")
else()
  message(STATUS "Found uchardet")
endif()

if(NOT APPLE)
  
  # detect icu

  find_path(ICU4C_INCLUDE_DIR
    NAMES unicode/utf8.h
    PATHS ${additional_includes}
  )
  find_library(ICU4C_LIBRARY
    NAMES icuuc
    PATHS ${additional_lib_searchpath}
  )

{'fraction_non_alphanumeric': 0.11802788844621515, 'fraction_numerical': 0.0037350597609561755, 'mean_word_length': 5.2765625, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5479922', 'n_tokens_mistral': 1667, 'n_tokens_neox': 1548, 'n_words': 215}