<commit_before># - Check iconv capabilities:
# 
# ICONV_CAN_TRANSLIT      - iconv supports //TRANSLIT option
# ICONV_CAN_IGNORE        - iconv supports //IGNORE option
# 
# Just to be sure, we do not limit the calls to iconv_open, but do the full thing
# Input is FORCED with a valid ASCII and UTF-8 compliant thingy.
# Caller have the choice of destination charset and have to provide TWO TRAILING BYTES.
#
MACRO(_FINDICONVCAPABILITY tocharset fromcharset outputvariable twobytes)
  INCLUDE (CheckCSourceRuns)
  SET (CMAKE_REQUIRED_INCLUDES_PREVIOUS ${CMAKE_REQUIRED_INCLUDES})
  SET (CMAKE_REQUIRED_LIBRARIES_PREVIOUS ${CMAKE_REQUIRED_LIBRARIES})
  SET (CMAKE_REQUIRED_DEFINITIONS_PREVIOUS ${CMAKE_REQUIRED_DEFINITIONS})
  SET (CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES_PREVIOUS} ${ICONV_INCLUDE_DIR})
  SET (CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES_PREVIOUS} ${ICONV_LINK_FLAGS})
  IF (WINICONV)
    SET (CMAKE_REQUIRED_DEFINITIONS ${CMAKE_REQUIRED_DEFINITIONS_PREVIOUS} -DWINICONV=1)
  ELSE ()
    SET (CMAKE_REQUIRED_DEFINITIONS ${CMAKE_REQUIRED_DEFINITIONS_PREVIOUS} -DWINICONV=0)<commit_msg>ICONV_IS_NOT_TRUSTABLE revival - observed on FreeBSD -; [ci skip]<commit_after># - Check iconv capabilities:
# 
# ICONV_CAN_TRANSLIT      - iconv supports //TRANSLIT option
# ICONV_CAN_IGNORE        - iconv supports //IGNORE option
# ICONV_IS_NOT_TRUSTABLE  - Some bad iconv's that dup blindly IMHO
# 
# Just to be sure, we do not limit the calls to iconv_open, but do the full thing
# Input is FORCED with a valid ASCII and UTF-8 compliant thingy.
# Caller have the choice of destination charset and have to provide TWO TRAILING BYTES.
#
MACRO(_FINDICONVCAPABILITY tocharset fromcharset outputvariable twobytes)
  INCLUDE (CheckCSourceRuns)
  SET (CMAKE_REQUIRED_INCLUDES_PREVIOUS ${CMAKE_REQUIRED_INCLUDES})
  SET (CMAKE_REQUIRED_LIBRARIES_PREVIOUS ${CMAKE_REQUIRED_LIBRARIES})
  SET (CMAKE_REQUIRED_DEFINITIONS_PREVIOUS ${CMAKE_REQUIRED_DEFINITIONS})
  SET (CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES_PREVIOUS} ${ICONV_INCLUDE_DIR})
  SET (CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES_PREVIOUS} ${ICONV_LINK_FLAGS})
  IF (WINICONV)
    SET (CMAKE_REQUIRED_DEFINITIONS ${CMAKE_REQUIRED_DEFINITIONS_PREVIOUS} -DWINICONV=1)
  ELSE ()
    SET (CMAKE_REQUIRED_DEFINITIONS ${CMAKE_REQUIRED_DEFINITIONS_PREVIOUS} -DWINICONV=0)
{'fraction_non_alphanumeric': 0.06969825754356142, 'fraction_numerical': 0.0025499362515937103, 'mean_word_length': 6.198776758409786, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '501582', 'n_tokens_mistral': 959, 'n_tokens_neox': 880, 'n_words': 195}