<commit_before>{ runCommand
, stdenv
, storeDir ? builtins.storeDir
, writeScript
, singularity
, writeReferencesToFile
, bash
, vmTools
, gawk
, utillinux
, e2fsprogs
, squashfsTools }:

rec {
  shellScript = name: text:
    writeScript name ''
      #!${stdenv.shell}
      set -e
      ${text}
    '';

  mkLayer = {
    name,
    contents ? [],
  }:
    runCommand "singularity-layer-${name}" {
      inherit contents;
    } ''
      mkdir $out
      for f in $contents ; do
        cp -ra $f $out/
      done
    '';

  buildImage = {
    name,
    contents ? [],
    diskSize ? 1024,
    runScript ? "#!${stdenv.shell}\nexec /bin/sh",
    runAsRoot ? null,
    extraSpace ? 0
  }:
    let layer = mkLayer {
          inherit name;
          contents = contents ++ [ bash runScriptFile ];
          };
        runAsRootFile = shellScript "run-as-root.sh" runAsRoot;
        runScriptFile = shellScript "run-script.sh" runScript;
        result = vmTools.runInLinuxVM (
          runCommand "singularity-image-${name}.img" {
            buildInputs = [ singularity e2fsprogs utillinux gawk ];
            layerClosure = writeReferencesToFile layer;
            preVM = vmTools.createEmptyImage {
              size = diskSize;
              fullName = "singularity-run-disk";
            };
          }
          ''
            rm -rf $out
            mkdir disk
            mkfs -t ext3 -b 4096 /dev/${vmTools.hd}
            mount /dev/${vmTools.hd} disk
            cd disk

            # Run root script
            ${stdenv.lib.optionalString (runAsRoot != null) ''
              mkdir -p ./${storeDir}
              mount --rbind ${storeDir} ./${storeDir}
              unshare -imnpuf --mount-proc chroot ./ ${runAsRootFile}
              umount -R ./${storeDir}
            ''}

            # Build /bin and copy across closure
            mkdir -p bin nix/store
            for f in $(cat $layerClosure) ; do
              cp -ar $f ./$f
              for f in $f/bin/* ; do
                if [ ! -e bin/$(basename $f) ] ; then
                  ln -s $f bin/
                fi
              done
            done

            # Create runScript
            ln -s ${runScriptFile} singularity

            # Size calculation
            cd ..
            umount disk
            size=$(resize2fs -P /dev/${vmTools.hd} | awk '{print $NF}')
            mount /dev/${vmTools.hd} disk
            cd disk

            export PATH=$PATH:${e2fsprogs}/bin/
            singularity create -s $((1 + size * 4 / 1024 + ${toString extraSpace})) $out
            tar -c . | singularity import $out
          '');

    in result;
}<commit_msg>singularity-tools: Only copy into /bin what is specified in contents<commit_after>{ runCommand
, stdenv
, storeDir ? builtins.storeDir
, writeScript
, singularity
, writeReferencesToFile
, bash
, vmTools
, gawk
, utillinux
, e2fsprogs
, squashfsTools }:

rec {
  shellScript = name: text:
    writeScript name ''
      #!${stdenv.shell}
      set -e
      ${text}
    '';

  mkLayer = {
    name,
    contents ? [],
  }:
    runCommand "singularity-layer-${name}" {
      inherit contents;
    } ''
      mkdir $out
      for f in $contents ; do
        cp -ra $f $out/
      done
    '';

  buildImage = {
    name,
    contents ? [],
    diskSize ? 1024,
    runScript ? "#!${stdenv.shell}\nexec /bin/sh",
    runAsRoot ? null,
    extraSpace ? 0
  }:
    let layer = mkLayer {
          inherit name;
          contents = contents ++ [ bash runScriptFile ];
          };
        runAsRootFile = shellScript "run-as-root.sh" runAsRoot;
        runScriptFile = shellScript "run-script.sh" runScript;
        result = vmTools.runInLinuxVM (
          runCommand "singularity-image-${name}.img" {
            buildInputs = [ singularity e2fsprogs utillinux gawk ];
            layerClosure = writeReferencesToFile layer;
            preVM = vmTools.createEmptyImage {
              size = diskSize;
              fullName = "singularity-run-disk";
            };
          }
          ''
            rm -rf $out
            mkdir disk
            mkfs -t ext3 -b 4096 /dev/${vmTools.hd}
            mount /dev/${vmTools.hd} disk
            cd disk

            # Run root script
            ${stdenv.lib.optionalString (runAsRoot != null) ''
              mkdir -p ./${storeDir}
              mount --rbind ${storeDir} ./${storeDir}
              unshare -imnpuf --mount-proc chroot ./ ${runAsRootFile}
              umount -R ./${storeDir}
            ''}

            # Build /bin and copy across closure
            mkdir -p bin nix/store
            for f in $(cat $layerClosure) ; do
              cp -ar $f ./$f
            done

            for c in ${toString contents} ; do
              for f in $c/bin/* ; do
                if [ ! -e bin/$(basename $f) ] ; then
                  ln -s $f bin/
                fi
              done
            done

            # Create runScript
            ln -s ${runScriptFile} singularity

            # Size calculation
            cd ..
            umount disk
            size=$(resize2fs -P /dev/${vmTools.hd} | awk '{print $NF}')
            mount /dev/${vmTools.hd} disk
            cd disk

            export PATH=$PATH:${e2fsprogs}/bin/
            singularity create -s $((1 + size * 4 / 1024 + ${toString extraSpace})) $out
            tar -c . | singularity import $out
          '');

    in result;
}
{'fraction_non_alphanumeric': 0.11589648798521257, 'fraction_numerical': 0.0073937153419593345, 'mean_word_length': 1.4362899594777128, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5443977', 'n_tokens_mistral': 1665, 'n_tokens_neox': 1577, 'n_words': 426}