<commit_before># This will automatically download the FAST test data zip file and unzip it
# The data will only be downloaded if not already present
if(${FAST_DOWNLOAD_TEST_DATA})
if(NOT EXISTS ${FAST_TEST_DATA_DIR})
    message("-- data folder does not exists. Downloading data ...")
    set(TEST_DATA_FILENAME ${PROJECT_SOURCE_DIR}/FAST_Test_Data.zip)
    # Download
    file(DOWNLOAD
            "http://www.idi.ntnu.no/~smistad/FAST_Test_Data.zip"
            ${TEST_DATA_FILENAME}
            SHOW_PROGRESS
            STATUS DOWNLOAD_STATUS
            INACTIVITY_TIMEOUT 30
    )
    list(GET DOWNLOAD_STATUS 0 STATUS_CODE)
    list(GET DOWNLOAD_STATUS 1 STATUS_DESC)
    if(NOT ${STATUS_CODE} EQUAL 0)
        message("-- Download of test data failed: ${STATUS_DESC}")
        # TODO Try other download URL
    else()
        # Unzip
        execute_process(
                COMMAND ${CMAKE_COMMAND}
                -E tar xzf ${TEST_DATA_FILENAME}
                WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
        )
        # Delete zip
        file(REMOVE ${TEST_DATA_FILENAME})
    endif()
else()
    message("-- data folder exists. Skip download of data.")
endif()
endif()<commit_msg>renamed TestData folder to data<commit_after># This will automatically download the FAST test data zip file and unzip it
# The data will only be downloaded if not already present
if(${FAST_DOWNLOAD_TEST_DATA})
if(NOT EXISTS ${FAST_TEST_DATA_DIR})
    message("-- Test data folder does not exists (${FAST_TEST_DATA_DIR}). Downloading data ...")
    set(TEST_DATA_FILENAME ${PROJECT_SOURCE_DIR}/FAST_Test_Data.zip)
    # Download
    file(DOWNLOAD
            "http://www.idi.ntnu.no/~smistad/FAST_Test_Data.zip"
            ${TEST_DATA_FILENAME}
            SHOW_PROGRESS
            STATUS DOWNLOAD_STATUS
            INACTIVITY_TIMEOUT 30
    )
    list(GET DOWNLOAD_STATUS 0 STATUS_CODE)
    list(GET DOWNLOAD_STATUS 1 STATUS_DESC)
    if(NOT ${STATUS_CODE} EQUAL 0)
        message("-- Download of test data failed: ${STATUS_DESC}")
        # TODO Try other download URL
    else()
        # Unzip
        execute_process(
                COMMAND ${CMAKE_COMMAND}
                -E tar xzf ${TEST_DATA_FILENAME}
                WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
        )
        # Delete zip
        file(REMOVE ${TEST_DATA_FILENAME})
    endif()
else()
    message("-- Test data folder exists (${FAST_TEST_DATA_DIR}). Skipping data download.")
endif()
endif()
{'fraction_non_alphanumeric': 0.09087221095334685, 'fraction_numerical': 0.004056795131845842, 'mean_word_length': 2.626470588235294, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6859492', 'n_tokens_mistral': 791, 'n_tokens_neox': 746, 'n_words': 202}