<commit_before>

{% macro job_deploy(job) -%}

{% set job_name = job['name'] -%}
{% set job_merged = salt['chronos_client.merge'](job, pillar[job_name]) %}
{% set tmp_dir = pillar['system']['tmp'] -%}
{% set nameservice_names = salt['hdfs.nameservice_names']() -%}
{% set uris = job_merged.get('uris', []) -%}

{% for uri in uris -%}

{% set uri_basename = salt['system.basename'](uri) -%}

job-uri-file-{{ uri_basename }}:
  file.managed:
    - name: {{ tmp_dir }}/{{ uri_basename  }}
    - source: {{ uri }}
    - user: root
    - group: root
    - mode: 755

{% for nameservice in nameservice_names %}

{% set basepath = "hdfs://{0}{1}".format(nameservice, pillar['hdfs']['pkgs_path']) -%}
{% set filepath = "{0}/{1}".format(basepath, uri_basename) -%}

job-uri-file-in-hdfs-{{ nameservice }}-{{ uri_basename }}:
  cmd.run:
    - name: |
        hadoop fs -mkdir -p {{ basepath }}
        hadoop fs -chmod -R 1777 {{ basepath }}
        hadoop fs -copyFromLocal {{ tmp_dir }}/{{ uri_basename  }} {{ filepath }}<commit_msg>Pillar should be optional<commit_after>

{% macro job_deploy(job) -%}

{% set job_name = job['name'] -%}
{% set job_merged = salt['chronos_client.merge'](job, pillar.get(job_name, {})) %}
{% set tmp_dir = pillar['system']['tmp'] -%}
{% set nameservice_names = salt['hdfs.nameservice_names']() -%}
{% set uris = job_merged.get('uris', []) -%}

{% for uri in uris -%}

{% set uri_basename = salt['system.basename'](uri) -%}

job-uri-file-{{ uri_basename }}:
  file.managed:
    - name: {{ tmp_dir }}/{{ uri_basename  }}
    - source: {{ uri }}
    - user: root
    - group: root
    - mode: 755

{% for nameservice in nameservice_names %}

{% set basepath = "hdfs://{0}{1}".format(nameservice, pillar['hdfs']['pkgs_path']) -%}
{% set filepath = "{0}/{1}".format(basepath, uri_basename) -%}

job-uri-file-in-hdfs-{{ nameservice }}-{{ uri_basename }}:
  cmd.run:
    - name: |
        hadoop fs -mkdir -p {{ basepath }}
        hadoop fs -chmod -R 1777 {{ basepath }}
        hadoop fs -copyFromLocal {{ tmp_dir }}/{{ uri_basename  }} {{ filepath }}
{'fraction_non_alphanumeric': 0.21545893719806763, 'fraction_numerical': 0.010628019323671498, 'mean_word_length': 3.4063829787234043, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2925470', 'n_tokens_mistral': 828, 'n_tokens_neox': 772, 'n_words': 151}