<commit_before># Borrowed from caffe
# https://github.com/BVLC/caffe/blob/master/cmake/Cuda.cmake

# Known NVIDIA GPU achitectures Torch can be compiled for.
# This list will be used for CUDA_ARCH_NAME = All option
SET(KNOWN_GPU_ARCHITECTURES "2.0 2.1(2.0) 3.0 3.5 5.0 5.2")

################################################################################################
# Removes duplicates from LIST(s)
# Usage:
#   LIST_UNIQUE(<list_variable> [<list_variable>] [...])
MACRO(LIST_UNIQUE)
  FOREACH(__lst ${ARGN})
    IF(${__lst})
      LIST(REMOVE_DUPLICATES ${__lst})
    ENDIF()
  ENDFOREACH()<commit_msg>fixing gpu arch detection fallback<commit_after># Borrowed from caffe
# https://github.com/BVLC/caffe/blob/master/cmake/Cuda.cmake

# Known NVIDIA GPU achitectures Torch can be compiled for.
# This list will be used for CUDA_ARCH_NAME = All option
SET(KNOWN_GPU_ARCHITECTURES "2.0 2.1(2.0) 3.0 3.5 5.0")

IF (CUDA_VERSION VERSION_GREATER "6.5")
   SET(KNOWN_GPU_ARCHITECTURES "${KNOWN_GPU_ARCHITECTURES} 5.2")
ENDIF (CUDA_VERSION VERSION_GREATER "6.5")

################################################################################################
# Removes duplicates from LIST(s)
# Usage:
#   LIST_UNIQUE(<list_variable> [<list_variable>] [...])
MACRO(LIST_UNIQUE)
  FOREACH(__lst ${ARGN})
    IF(${__lst})
      LIST(REMOVE_DUPLICATES ${__lst})
    ENDIF()
  ENDFOREACH()
{'fraction_non_alphanumeric': 0.2557636887608069, 'fraction_numerical': 0.023054755043227664, 'mean_word_length': 5.52112676056338, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7285714', 'n_tokens_mistral': 538, 'n_tokens_neox': 482, 'n_words': 103}