<commit_before># 
# how to add packages?
# 
    # you can search for them here: https://search.nixos.org/packages
    #
    # if you find a package like: nixpkg.python38Packages.setuptools
    # then add a section like this:
    #    [[packages]]
    #    load = [ "python38Packages", "setuptools",]
    # 
    # if you need a specific version 
    # search for the package (e.g. "setuptools") on here: https://lazamar.co.uk/nix-versions/
    # each version will have a long hash like "3f50332bc4913a56ad216ca01f5d0bd24277a6b2" 
    # (which is a commit on the nixpkg repo)
    # once you copy the hash, you can add the package below like
    # 
    #    [[packages]]
    #    load = [ "python38Packages", "setuptools",]
    #    from = "3f50332bc4913a56ad216ca01f5d0bd24277a6b2"

# 
# Complex packages
# 
    # things such as platform-specific packages,
    # loading from a git repository,
    # or if you find an online solution that involves native build inputs
    # you can use the following advanced format
    # 
    #    [[packages]]
    #    load = [ "nixGLNvidia",]
    #    onlyIf = [ [ "stdenv", "isLinux",],] # for MacOS use: onlyIf = [ [ "stdenv", "isDarwin",],]
    #    # see https://discourse.nixos.org/t/opencv-with-cuda-in-nix-shell/7358/5
    #    from = { fetchGit = { url = "https://github.com/guibou/nixGL", rev = "7d6bc1b21316bab6cf4a6520c2639a11c25a220e" }, }
    # 
    # in general when looking at online forms, look at the shell.nix file
    # specifically look at the `# create a shell` and `main.packages.mkShell` part
    # and know that seeing `pkgs` online translates to `main.packages` inside the shell.nix file


# [[ you can add tools for your project right here or at the bottom ]]

# 
# 
# 
# Need-to-build-from-source tools
#
# 
# 
# Note: most projects need these, 
# and if you need them for one project nix has a shared-memory across projects
# comment these out in a situation such as deploying a project to a minimal server

# 
# Linux
# 
[[packages]]
load = [ "pkgconfig",]
asNativeBuildInput = true
onlyIf = [ [ "stdenv", "isLinux",],]

[[packages]]
load = [ "libconfig",]
asNativeBuildInput = true
onlyIf = [ [ "stdenv", "isLinux",],]

[[packages]]
load = [ "cmake",]
asNativeBuildInput = true
onlyIf = [ [ "stdenv", "isLinux",],]

[[packages]]
load = [ "stdenv", "cc",]
onlyIf = [ [ "stdenv", "isLinux",],]

# 
# Mac
# 
[[packages]]
load = [ "xcbuild",]
asNativeBuildInput = true
onlyIf = [ [ "stdenv", "isDarwin",],]

[[packages]]
load = [ "xcodebuild",]
asNativeBuildInput = true
onlyIf = [ [ "stdenv", "isDarwin",],]

[[packages]]
load = [ "darwin", "libobjc", ]
asNativeBuildInput = true
onlyIf = [ [ "stdenv", "isDarwin",],]

[[packages]]
load = [ "darwin", "apple_sdk", "frameworks", "CoreServices" ]
asNativeBuildInput = true
onlyIf = [ [ "stdenv", "isDarwin",],]

[[packages]]
load = [ "darwin", "apple_sdk", "frameworks", "CoreFoundation" ]
asNativeBuildInput = true
onlyIf = [ [ "stdenv", "isDarwin",],]

[[packages]]
load = [ "darwin", "apple_sdk", "frameworks", "Foundation" ]
asNativeBuildInput = true
onlyIf = [ [ "stdenv", "isDarwin",],]

[[packages]]
load = [ "darwin", "apple_sdk", "frameworks", "AVKit" ]
asNativeBuildInput = true
onlyIf = [ [ "stdenv", "isDarwin",],]

[[packages]]
load = [ "darwin", "apple_sdk", "frameworks", "AppKit" ]
asNativeBuildInput = true
onlyIf = [ [ "stdenv", "isDarwin",],]

[[packages]]
load = [ "darwin", "apple_sdk", "frameworks", "WebKit" ]
asNativeBuildInput = true<commit_msg>add AVFoundation<commit_after># 
# how to add packages?
# 
    # you can search for them here: https://search.nixos.org/packages
    #
    # if you find a package like: nixpkg.python38Packages.setuptools
    # then add a section like this:
    #    [[packages]]
    #    load = [ "python38Packages", "setuptools",]
    # 
    # if you need a specific version 
    # search for the package (e.g. "setuptools") on here: https://lazamar.co.uk/nix-versions/
    # each version will have a long hash like "3f50332bc4913a56ad216ca01f5d0bd24277a6b2" 
    # (which is a commit on the nixpkg repo)
    # once you copy the hash, you can add the package below like
    # 
    #    [[packages]]
    #    load = [ "python38Packages", "setuptools",]
    #    from = "3f50332bc4913a56ad216ca01f5d0bd24277a6b2"

# 
# Complex packages
# 
    # things such as platform-specific packages,
    # loading from a git repository,
    # or if you find an online solution that involves native build inputs
    # you can use the following advanced format
    # 
    #    [[packages]]
    #    load = [ "nixGLNvidia",]
    #    onlyIf = [ [ "stdenv", "isLinux",],] # for MacOS use: onlyIf = [ [ "stdenv", "isDarwin",],]
    #    # see https://discourse.nixos.org/t/opencv-with-cuda-in-nix-shell/7358/5
    #    from = { fetchGit = { url = "https://github.com/guibou/nixGL", rev = "7d6bc1b21316bab6cf4a6520c2639a11c25a220e" }, }
    # 
    # in general when looking at online forms, look at the shell.nix file
    # specifically look at the `# create a shell` and `main.packages.mkShell` part
    # and know that seeing `pkgs` online translates to `main.packages` inside the shell.nix file


# [[ you can add tools for your project right here or at the bottom ]]

# 
# 
# 
# Need-to-build-from-source tools
#
# 
# 
# Note: most projects need these, 
# and if you need them for one project nix has a shared-memory across projects
# comment these out in a situation such as deploying a project to a minimal server

# 
# Linux
# 
[[packages]]
load = [ "pkgconfig",]
asNativeBuildInput = true
onlyIf = [ [ "stdenv", "isLinux",],]

[[packages]]
load = [ "libconfig",]
asNativeBuildInput = true
onlyIf = [ [ "stdenv", "isLinux",],]

[[packages]]
load = [ "cmake",]
asNativeBuildInput = true
onlyIf = [ [ "stdenv", "isLinux",],]

[[packages]]
load = [ "stdenv", "cc",]
onlyIf = [ [ "stdenv", "isLinux",],]

# 
# Mac
# 
[[packages]]
load = [ "xcbuild",]
asNativeBuildInput = true
onlyIf = [ [ "stdenv", "isDarwin",],]

[[packages]]
load = [ "xcodebuild",]
asNativeBuildInput = true
onlyIf = [ [ "stdenv", "isDarwin",],]

[[packages]]
load = [ "darwin", "libobjc", ]
asNativeBuildInput = true
onlyIf = [ [ "stdenv", "isDarwin",],]

[[packages]]
load = [ "darwin", "apple_sdk", "frameworks", "CoreServices" ]
asNativeBuildInput = true
onlyIf = [ [ "stdenv", "isDarwin",],]

[[packages]]
load = [ "darwin", "apple_sdk", "frameworks", "CoreFoundation" ]
asNativeBuildInput = true
onlyIf = [ [ "stdenv", "isDarwin",],]

[[packages]]
load = [ "darwin", "apple_sdk", "frameworks", "Foundation" ]
asNativeBuildInput = true
onlyIf = [ [ "stdenv", "isDarwin",],]

[[packages]]
load = [ "darwin", "apple_sdk", "frameworks", "AVKit" ]
asNativeBuildInput = true
onlyIf = [ [ "stdenv", "isDarwin",],]

[[packages]]
load = [ "darwin", "apple_sdk", "frameworks", "AVFoundation" ]
asNativeBuildInput = true
onlyIf = [ [ "stdenv", "isDarwin",],]

[[packages]]
load = [ "darwin", "apple_sdk", "frameworks", "AppKit" ]
asNativeBuildInput = true
onlyIf = [ [ "stdenv", "isDarwin",],]

[[packages]]
load = [ "darwin", "apple_sdk", "frameworks", "WebKit" ]
asNativeBuildInput = true
{'fraction_non_alphanumeric': 0.1612221909321318, 'fraction_numerical': 0.02478175161926218, 'mean_word_length': 3.8286879673691367, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4647331', 'n_tokens_mistral': 2686, 'n_tokens_neox': 2532, 'n_words': 681}