<commit_before>{ config, lib, pkgs, ...}:
let
  cfg = config.services.hadoop;
in
with lib;
{
  imports = [ ./yarn.nix ./hdfs.nix ];

  options.services.hadoop = {
    coreSite = mkOption {
      default = {};
      type = types.attrsOf types.anything;
      example = literalExpression ''
        {
          "fs.defaultFS" = "hdfs://localhost";
        }
      '';
      description = "Hadoop core-site.xml definition";
    };

    hdfsSite = mkOption {
      default = {
        "dfs.namenode.rpc-bind-host" = "0.0.0.0";
      };
      type = types.attrsOf types.anything;
      example = literalExpression ''
        {
          "dfs.nameservices" = "namenode1";
        }
      '';
      description = "Hadoop hdfs-site.xml definition";
    };

    mapredSite = mkOption {
      default = {
        "mapreduce.framework.name" = "yarn";
        "yarn.app.mapreduce.am.env" = "HADOOP_MAPRED_HOME=${cfg.package}/lib/${cfg.package.untarDir}";
        "mapreduce.map.env" = "HADOOP_MAPRED_HOME=${cfg.package}/lib/${cfg.package.untarDir}";
        "mapreduce.reduce.env" = "HADOOP_MAPRED_HOME=${cfg.package}/lib/${cfg.package.untarDir}";
      };
      type = types.attrsOf types.anything;
      example = literalExpression ''
        options.services.hadoop.mapredSite.default // {
          "mapreduce.map.java.opts" = "-Xmx900m -XX:+UseParallelGC";
        }
      '';
      description = "Hadoop mapred-site.xml definition";
    };

    yarnSite = mkOption {
      default = {
        "yarn.nodemanager.admin-env" = "PATH=$PATH";
        "yarn.nodemanager.aux-services" = "mapreduce_shuffle";
        "yarn.nodemanager.aux-services.mapreduce_shuffle.class" = "org.apache.hadoop.mapred.ShuffleHandler";
        "yarn.nodemanager.bind-host" = "0.0.0.0";
        "yarn.nodemanager.container-executor.class" = "org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor";
        "yarn.nodemanager.env-whitelist" = "JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,LANG,TZ";
        "yarn.nodemanager.linux-container-executor.group" = "hadoop";
        "yarn.nodemanager.linux-container-executor.path" = "/run/wrappers/yarn-nodemanager/bin/container-executor";
        "yarn.nodemanager.log-dirs" = "/var/log/hadoop/yarn/nodemanager";
        "yarn.resourcemanager.bind-host" = "0.0.0.0";
        "yarn.resourcemanager.scheduler.class" = "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler";
      };
      type = types.attrsOf types.anything;
      example = literalExpression ''
        options.services.hadoop.yarnSite.default // {
          "yarn.resourcemanager.hostname" = "''${config.networking.hostName}";
        }
      '';
      description = "Hadoop yarn-site.xml definition";
    };

    httpfsSite = mkOption {
      default = { };
      type = types.attrsOf types.anything;
      example = literalExpression ''
        {
          "hadoop.http.max.threads" = 500;
        }
      '';
      description = "Hadoop httpfs-site.xml definition";
    };

    log4jProperties = mkOption {
      default = "${cfg.package}/lib/${cfg.package.untarDir}/etc/hadoop/log4j.properties";
      type = types.path;
      example = literalExpression ''
        "''${pkgs.hadoop}/lib/''${pkgs.hadoop.untarDir}/etc/hadoop/log4j.properties";
      '';
      description = "log4j.properties file added to HADOOP_CONF_DIR";
    };

    containerExecutorCfg = mkOption {
      default = {
        # must be the same as yarn.nodemanager.linux-container-executor.group in yarnSite
        "yarn.nodemanager.linux-container-executor.group"="hadoop";
        "min.user.id"=1000;
        "feature.terminal.enabled"=1;
      };
      type = types.attrsOf types.anything;
      example = literalExpression ''
        options.services.hadoop.containerExecutorCfg.default // {
          "feature.terminal.enabled" = 0;
        }
      '';
      description = "Yarn container-executor.cfg definition";
    };

    extraConfDirs = mkOption {
      default = [];<commit_msg>nixos/hadoop: add links for config files<commit_after>{ config, lib, pkgs, ...}:
let
  cfg = config.services.hadoop;
in
with lib;
{
  imports = [ ./yarn.nix ./hdfs.nix ];

  options.services.hadoop = {
    coreSite = mkOption {
      default = {};
      type = types.attrsOf types.anything;
      example = literalExpression ''
        {
          "fs.defaultFS" = "hdfs://localhost";
        }
      '';
      description = ''
        Hadoop core-site.xml definition
        <link xlink:href="https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/core-default.xml"/>
      '';
    };

    hdfsSite = mkOption {
      default = {
        "dfs.namenode.rpc-bind-host" = "0.0.0.0";
      };
      type = types.attrsOf types.anything;
      example = literalExpression ''
        {
          "dfs.nameservices" = "namenode1";
        }
      '';
      description = ''
        Hadoop hdfs-site.xml definition
        <link xlink:href="https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/hdfs-default.xml"/>
      '';
    };

    mapredSite = mkOption {
      default = {
        "mapreduce.framework.name" = "yarn";
        "yarn.app.mapreduce.am.env" = "HADOOP_MAPRED_HOME=${cfg.package}/lib/${cfg.package.untarDir}";
        "mapreduce.map.env" = "HADOOP_MAPRED_HOME=${cfg.package}/lib/${cfg.package.untarDir}";
        "mapreduce.reduce.env" = "HADOOP_MAPRED_HOME=${cfg.package}/lib/${cfg.package.untarDir}";
      };
      type = types.attrsOf types.anything;
      example = literalExpression ''
        options.services.hadoop.mapredSite.default // {
          "mapreduce.map.java.opts" = "-Xmx900m -XX:+UseParallelGC";
        }
      '';
      description = ''
        Hadoop mapred-site.xml definition
        <link xlink:href="https://hadoop.apache.org/docs/current/hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml"/>
      '';
    };

    yarnSite = mkOption {
      default = {
        "yarn.nodemanager.admin-env" = "PATH=$PATH";
        "yarn.nodemanager.aux-services" = "mapreduce_shuffle";
        "yarn.nodemanager.aux-services.mapreduce_shuffle.class" = "org.apache.hadoop.mapred.ShuffleHandler";
        "yarn.nodemanager.bind-host" = "0.0.0.0";
        "yarn.nodemanager.container-executor.class" = "org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor";
        "yarn.nodemanager.env-whitelist" = "JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,LANG,TZ";
        "yarn.nodemanager.linux-container-executor.group" = "hadoop";
        "yarn.nodemanager.linux-container-executor.path" = "/run/wrappers/yarn-nodemanager/bin/container-executor";
        "yarn.nodemanager.log-dirs" = "/var/log/hadoop/yarn/nodemanager";
        "yarn.resourcemanager.bind-host" = "0.0.0.0";
        "yarn.resourcemanager.scheduler.class" = "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler";
      };
      type = types.attrsOf types.anything;
      example = literalExpression ''
        options.services.hadoop.yarnSite.default // {
          "yarn.resourcemanager.hostname" = "''${config.networking.hostName}";
        }
      '';
      description = ''
        Hadoop yarn-site.xml definition
        <link xlink:href="https://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-common/yarn-default.xml"/>
      '';
    };

    httpfsSite = mkOption {
      default = { };
      type = types.attrsOf types.anything;
      example = literalExpression ''
        {
          "hadoop.http.max.threads" = 500;
        }
      '';
      description = ''
        Hadoop httpfs-site.xml definition
        <link xlink:href="https://hadoop.apache.org/docs/current/hadoop-hdfs-httpfs/httpfs-default.html"/>
      '';
    };

    log4jProperties = mkOption {
      default = "${cfg.package}/lib/${cfg.package.untarDir}/etc/hadoop/log4j.properties";
      type = types.path;
      example = literalExpression ''
        "''${pkgs.hadoop}/lib/''${pkgs.hadoop.untarDir}/etc/hadoop/log4j.properties";
      '';
      description = "log4j.properties file added to HADOOP_CONF_DIR";
    };

    containerExecutorCfg = mkOption {
      default = {
        # must be the same as yarn.nodemanager.linux-container-executor.group in yarnSite
        "yarn.nodemanager.linux-container-executor.group"="hadoop";
        "min.user.id"=1000;
        "feature.terminal.enabled"=1;
      };
      type = types.attrsOf types.anything;
      example = literalExpression ''
        options.services.hadoop.containerExecutorCfg.default // {
          "feature.terminal.enabled" = 0;
        }
      '';
      description = ''
        Yarn container-executor.cfg definition
        <link xlink:href="https://hadoop.apache.org/docs/r2.7.2/hadoop-yarn/hadoop-yarn-site/SecureContainer.html"/>
      '';
    };

    extraConfDirs = mkOption {
      default = [];
{'fraction_non_alphanumeric': 0.1489219081666853, 'fraction_numerical': 0.006814881018880572, 'mean_word_length': 2.5355450236966823, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 14}, 'pii_count': 6, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6338978', 'n_tokens_mistral': 2997, 'n_tokens_neox': 2943, 'n_words': 343}