<commit_before># - Manage Source Archive and track its content.
# This module manages source archive by:
#  - Handle the ChangeLog
#  - Build the source archive
#  - Exclude unwanted files, either from ignore files of source version
#    control, or add them to SOURCE_ARCHIVE_IGNORE_FILES
#  - Track the contents in source archive, when it updates, the source 
#    Archive should be updated.
#
# This module provide a convenient wrapper of CPack. 
# CPack itself, by default, pack everything under the source directory,
# this is usually undesirable. 
# We avoid this by using the sane default ignore list.
#
# Included Modules:
#   - ManageFile
#   - ManageVersion
#   - CPack
#
# Defines following functions:
#   SOURCE_ARCHIVE_CONTENTS_ADD(<filename>)
#   - Add a file to source archive if the file is not in the archive.
#     * Parameters:
#       - filename: Filename to be added.
#
#   SOURCE_ARCHIVE_CONTENTS_ADD_NO_CHECK(<filename>)
#   - Add a file to source archive without check.
#     * Parameters:
#       - filename: Filename to be added.
#
# Defines following macros:
#   PACK_SOURCE_ARCHIVE([<outputDir> | OUTPUT_FILE <file>] 
#     [GENERATOR <cpackGenerator>] 
#     [GITIGNORE <gitignoreFile>] [INCLUDE <file ...>])
#     )
#     - Pack source archive..
#       * Parameters:
#         + outputDir: Directory to write source archive.
#         + OUTPUT_FILE file: Output file with path.
#           Default: SOURCES/<projectName>-<PRJ_VER>-Source.<packFormat>
#         + GENERATOR cpackGenerator: The CPack generator
#           Default: TGZ (.tar.gz) 
#         + GITIGNORE gitignoreFile: Specify path to .gitignore for using .gitignore
#             to exclude the unwanted files.
#         + INCLUDE file ...: Add back those files to source archive.
#             You can add back the files that would otherwise be excluded,
#             e.g. .pot files.
#             otherwise those will be excluded.
#       * Variables to be cached:
#         + SOURCE_ARCHIVE_CONTENTS: List of files to be packed.
#         + SOURCE_ARCHIVE_FILE: Path of source archive (with path).
#         + SOURCE_ARCHIVE_FILE_EXTENSION: File extension of 
#             the source package
#         + SOURCE_ARCHIVE_IGNORE_FILES: List of files to be 
#             ignored to archive.
#         + SOURCE_ARCHIVE_NAME: Name of source archive (without path).
#       * Targets:
#         + pack_src: Pack source files like package_source.
#         + clean_pack_src: Remove all source archives.
#         + clean_old_pack_src: Remove all old source package.
#
#
IF(DEFINED _MANAGE_ARCHIVE_CMAKE_)
    RETURN()
ENDIF(DEFINED _MANAGE_ARCHIVE_CMAKE_)

SET (_MANAGE_ARCHIVE_CMAKE_ "DEFINED")
SET(SOURCE_ARCHIVE_IGNORE_FILES_COMMON
    "/\\\\.svn/"  "/CVS/" "/\\\\.git/" "/\\\\.hg/" "NO_PACK")

SET(SOURCE_ARCHIVE_IGNORE_FILES_CMAKE 
    "/CMakeFiles/" "_CPack_Packages/" "/Testing/"
    "\\\\.directory$" "CMakeCache\\\\.txt$"
    "/install_manifest.txt$"
    "/cmake_install\\\\.cmake$" "/cmake_uninstall\\\\.cmake$"
    "/CPack.*\\\\.cmake$" "/CTestTestfile\\\\.cmake$"
    "Makefile$" "/${PROJECT_NAME}-${PRJ_VER}-SOURCE/"
    )
SET(SOURCE_ARCHIVE_IGNORE_FILES 
    ${SOURCE_ARCHIVE_IGNORE_FILES_CMAKE}
    ${SOURCE_ARCHIVE_IGNORE_FILES_COMMON}
    )

INCLUDE(ManageVersion)
INCLUDE(ManageFile)

FUNCTION(SOURCE_ARCHIVE_CONTENTS_SET value)
    SET(SOURCE_ARCHIVE_CONTENTS "${value}" CACHE INTERNAL "Source archive file list")
ENDFUNCTION(SOURCE_ARCHIVE_CONTENTS_SET)
SOURCE_ARCHIVE_CONTENTS_SET("")

## Add whatever it's told
FUNCTION(SOURCE_ARCHIVE_CONTENTS_ADD_FAST value)
    LIST(APPEND SOURCE_ARCHIVE_CONTENTS "${value}")
    SOURCE_ARCHIVE_CONTENTS_SET("${SOURCE_ARCHIVE_CONTENTS}")
ENDFUNCTION(SOURCE_ARCHIVE_CONTENTS_ADD_FAST)

FUNCTION(SOURCE_ARCHIVE_CONTENTS_ADD filename)
    GET_FILENAME_COMPONENT(_file "${filename}" ABSOLUTE)
    FILE(RELATIVE_PATH _f ${CMAKE_SOURCE_DIR} "${_file}")
    LIST(FIND SOURCE_ARCHIVE_CONTENTS "${_f}" _index)
    IF(_index LESS 0)
	SOURCE_ARCHIVE_CONTENTS_ADD_FAST("${_f}")
    ENDIF(_index LESS 0)
ENDFUNCTION(SOURCE_ARCHIVE_CONTENTS_ADD)

FUNCTION(SOURCE_ARCHIVE_CONTENTS_ADD_NO_CHECK filename)
    GET_FILENAME_COMPONENT(_file "${filename}" ABSOLUTE)
    FILE(RELATIVE_PATH _f ${CMAKE_SOURCE_DIR} "${_file}")
    SOURCE_ARCHIVE_CONTENTS_ADD_FAST("${_f}")
ENDFUNCTION(SOURCE_ARCHIVE_CONTENTS_ADD_NO_CHECK)

# Internal:  SOURCE_ARCHIVE_GET_CONTENTS()
#   - Return all source file to be packed.
#     This is called by SOURCE_ARCHIVE(),
#     So no need to call it again.
FUNCTION(SOURCE_ARCHIVE_GET_CONTENTS )
    SET(_fileList "")
    FILE(GLOB_RECURSE _ls FOLLOW_SYMLINKS "*" )
    STRING(REPLACE "\\\\" "\\" _ignore_files
        "${SOURCE_ARCHIVE_IGNORE_FILES}")

    FOREACH(_file ${_ls})
	SET(_matched 0)
	FOREACH(filePattern ${_ignore_files})
	    M_MSG(${M_INFO3} "_file=${_file} filePattern=${filePattern}")

	    IF(_file MATCHES "${filePattern}")
		SET(_matched 1)
		BREAK()
	    ENDIF(_file MATCHES "${filePattern}")<commit_msg>ManageArchive: SOURCE_ARCHIVE_CONTENTS_ADD now accept multiple files<commit_after># - Manage Source Archive and track its content.
# This module manages source archive by:
#  - Handle the ChangeLog
#  - Build the source archive
#  - Exclude unwanted files, either from ignore files of source version
#    control, or add them to SOURCE_ARCHIVE_IGNORE_FILES
#  - Track the contents in source archive, when it updates, the source 
#    Archive should be updated.
#
# This module provide a convenient wrapper of CPack. 
# CPack itself, by default, pack everything under the source directory,
# this is usually undesirable. 
# We avoid this by using the sane default ignore list.
#
# Included Modules:
#   - ManageFile
#   - ManageVersion
#   - CPack
#
# Defines following functions:
#   SOURCE_ARCHIVE_CONTENTS_ADD(<filename>)
#   - Add a file to source archive if the file is not in the archive.
#     * Parameters:
#       - filename: Filename to be added.
#
#   SOURCE_ARCHIVE_CONTENTS_ADD_NO_CHECK(<filename>)
#   - Add a file to source archive without check.
#     * Parameters:
#       - filename: Filename to be added.
#
# Defines following macros:
#   PACK_SOURCE_ARCHIVE([<outputDir> | OUTPUT_FILE <file>] 
#     [GENERATOR <cpackGenerator>] 
#     [GITIGNORE <gitignoreFile>] [INCLUDE <file ...>])
#     )
#     - Pack source archive..
#       * Parameters:
#         + outputDir: Directory to write source archive.
#         + OUTPUT_FILE file: Output file with path.
#           Default: SOURCES/<projectName>-<PRJ_VER>-Source.<packFormat>
#         + GENERATOR cpackGenerator: The CPack generator
#           Default: TGZ (.tar.gz) 
#         + GITIGNORE gitignoreFile: Specify path to .gitignore for using .gitignore
#             to exclude the unwanted files.
#         + INCLUDE file ...: Add back those files to source archive.
#             You can add back the files that would otherwise be excluded,
#             e.g. .pot files.
#             otherwise those will be excluded.
#       * Variables to be cached:
#         + SOURCE_ARCHIVE_CONTENTS: List of files to be packed.
#         + SOURCE_ARCHIVE_FILE: Path of source archive (with path).
#         + SOURCE_ARCHIVE_FILE_EXTENSION: File extension of 
#             the source package
#         + SOURCE_ARCHIVE_IGNORE_FILES: List of files to be 
#             ignored to archive.
#         + SOURCE_ARCHIVE_NAME: Name of source archive (without path).
#       * Targets:
#         + pack_src: Pack source files like package_source.
#         + clean_pack_src: Remove all source archives.
#         + clean_old_pack_src: Remove all old source package.
#
#
IF(DEFINED _MANAGE_ARCHIVE_CMAKE_)
    RETURN()
ENDIF(DEFINED _MANAGE_ARCHIVE_CMAKE_)

SET (_MANAGE_ARCHIVE_CMAKE_ "DEFINED")
SET(SOURCE_ARCHIVE_IGNORE_FILES_COMMON
    "/\\\\.svn/"  "/CVS/" "/\\\\.git/" "/\\\\.hg/" "NO_PACK")

SET(SOURCE_ARCHIVE_IGNORE_FILES_CMAKE 
    "/CMakeFiles/" "_CPack_Packages/" "/Testing/"
    "\\\\.directory$" "CMakeCache\\\\.txt$"
    "/install_manifest.txt$"
    "/cmake_install\\\\.cmake$" "/cmake_uninstall\\\\.cmake$"
    "/CPack.*\\\\.cmake$" "/CTestTestfile\\\\.cmake$"
    "Makefile$" "/${PROJECT_NAME}-${PRJ_VER}-SOURCE/"
    )
SET(SOURCE_ARCHIVE_IGNORE_FILES 
    ${SOURCE_ARCHIVE_IGNORE_FILES_CMAKE}
    ${SOURCE_ARCHIVE_IGNORE_FILES_COMMON}
    )

INCLUDE(ManageVersion)
INCLUDE(ManageFile)

FUNCTION(SOURCE_ARCHIVE_CONTENTS_SET value)
    SET(SOURCE_ARCHIVE_CONTENTS "${value}" CACHE INTERNAL "Source archive file list")
ENDFUNCTION(SOURCE_ARCHIVE_CONTENTS_SET)
SOURCE_ARCHIVE_CONTENTS_SET("")

## Add whatever it's told
FUNCTION(SOURCE_ARCHIVE_CONTENTS_ADD_FAST value)
    LIST(APPEND SOURCE_ARCHIVE_CONTENTS "${value}")
    SOURCE_ARCHIVE_CONTENTS_SET("${SOURCE_ARCHIVE_CONTENTS}")
ENDFUNCTION(SOURCE_ARCHIVE_CONTENTS_ADD_FAST)

FUNCTION(SOURCE_ARCHIVE_CONTENTS_ADD)
    FOREACH(filename ${ARGN})
	GET_FILENAME_COMPONENT(_file "${filename}" ABSOLUTE)
	FILE(RELATIVE_PATH _f ${CMAKE_SOURCE_DIR} "${_file}")
	LIST(FIND SOURCE_ARCHIVE_CONTENTS "${_f}" _index)
	IF(_index LESS 0)
	    SOURCE_ARCHIVE_CONTENTS_ADD_FAST("${_f}")
	ENDIF(_index LESS 0)
    ENDFOREACH(filename)
ENDFUNCTION(SOURCE_ARCHIVE_CONTENTS_ADD)

FUNCTION(SOURCE_ARCHIVE_CONTENTS_ADD_NO_CHECK)
    FOREACH(filename ${ARGN})
	GET_FILENAME_COMPONENT(_file "${filename}" ABSOLUTE)
	FILE(RELATIVE_PATH _f ${CMAKE_SOURCE_DIR} "${_file}")
	SOURCE_ARCHIVE_CONTENTS_ADD_FAST("${_f}")
    ENDFOREACH(filename)
ENDFUNCTION(SOURCE_ARCHIVE_CONTENTS_ADD_NO_CHECK)

# Internal:  SOURCE_ARCHIVE_GET_CONTENTS()
#   - Return all source file to be packed.
#     This is called by SOURCE_ARCHIVE(),
#     So no need to call it again.
FUNCTION(SOURCE_ARCHIVE_GET_CONTENTS )
    SET(_fileList "")
    FILE(GLOB_RECURSE _ls FOLLOW_SYMLINKS "*" )
    STRING(REPLACE "\\\\" "\\" _ignore_files
        "${SOURCE_ARCHIVE_IGNORE_FILES}")

    FOREACH(_file ${_ls})
	SET(_matched 0)
	FOREACH(filePattern ${_ignore_files})
	    M_MSG(${M_INFO3} "_file=${_file} filePattern=${filePattern}")

	    IF(_file MATCHES "${filePattern}")
		SET(_matched 1)
		BREAK()
	    ENDIF(_file MATCHES "${filePattern}")
{'fraction_non_alphanumeric': 0.11003268297514113, 'fraction_numerical': 0.0009903931860948797, 'mean_word_length': 3.627864344637947, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3190925', 'n_tokens_mistral': 3528, 'n_tokens_neox': 3287, 'n_words': 810}