<commit_before>---

- name: Create group for Airflow
  group:
    name: '{{ airflow_group }}'
    state: present

- name: Create user for Airflow
  user:
    name: '{{ airflow_user }}'
    group: '{{ airflow_group }}'
    home: '{{ airflow_home }}'
  when: airflow_user != 'vagrant'

- name: Create airflow home
  file:
    path: '{{ airflow_home}}'
    state: directory
    owner: '{{ airflow_user }}'

- name: Install required Debian packages
  apt:
    pkg: '{{ item }}'
    state: present
    update_cache: yes
    cache_valid_time: 600
  with_items: "{{ airflow_deb_packages }}"

- name: Install required Python packages
  pip:
    name: '{{ item }}'
    state: present
  with_items: "{{ airflow_pip_packages }}"

- name: Install required Debian packages for plugins
  apt:
    pkg: '{{ item }}'
    state: present
    update_cache: yes
    cache_valid_time: 600
  with_flattened: "{{ airflow_plugins|map(attribute='required_debs')|list }}"

- name: Install required Python packages for plugins
  pip:
    name: '{{ item }}'
    state: present
  with_flattened: "{{ airflow_plugins|map(attribute='required_pips')|list }}"

- name: Install airflow (first time installation takes a long time)
  pip:
    name: 'airflow=={{ airflow_version }}'
    state: present

- name: Install airflow plugins
  pip:
    name: 'airflow[{{ item.name }}]=={{ airflow_version }}'
    state: present
  with_items: "{{ airflow_plugins }}"

- name: Generate Fernet key
  ferney_key:
    config_file: '{{ airflow_config}}'

- name: Setup configuration for airflow
  template:
    src: airflow.cfg.j2
    dest: '{{ airflow_config}}'
    owner: '{{ airflow_user }}'

- name: Remove uncompatible DAG example (example_twitter_dag.py)
  file:
    path: /usr/local/lib/python2.7/dist-packages/airflow/example_dags/example_twitter_dag.py
    state: absent

- name: Launch Airflow Initdb
  command: "/usr/local/bin/airflow initdb"
  register: airflow_initdb_result
  environment:
    AIRFLOW_HOME: "{{ airflow_home }}"
    PYTHONPATH: "{{ airflow_python_path }}"
  become: true
  become_user: '{{ airflow_user }}'
  changed_when: "airflow_initdb_result.rc != 0 or airflow_initdb_result.stdout.find('Running upgrade') >= 0"

- name: Launch Airflow Upgradedb
  command: "/usr/local/bin/airflow upgradedb"
  register: airflow_upgradedb_result
  environment:
    AIRFLOW_HOME: "{{ airflow_home }}"
    PYTHONPATH: "{{ airflow_python_path }}"
  become: true
  become_user: '{{ airflow_user }}'
  changed_when: "airflow_upgradedb_result.rc != 0 or airflow_upgradedb_result.stdout.find('Running upgrade') >= 0"

- name: Install Airflow scripts
  git:
    repo: '{{ item.url }}'
    version: "{{ item.version | default('latest') }}"
    verify_commit: "{{ item.verify_commit | default(False) }}"
    key_file: "{{ item.key_file | default(None) }}"
    dest: '{{ item.dest }}'
    depth: 1
    force: yes
    update: yes
  with_items: "{{ airflow_scripts }}"

- include: marathon.yml
  when: airflow_startup == 'marathon'<commit_msg>Add tag airflow_scripts<commit_after>---

- name: Create group for Airflow
  group:
    name: '{{ airflow_group }}'
    state: present

- name: Create user for Airflow
  user:
    name: '{{ airflow_user }}'
    group: '{{ airflow_group }}'
    home: '{{ airflow_home }}'
  when: airflow_user != 'vagrant'

- name: Create airflow home
  file:
    path: '{{ airflow_home}}'
    state: directory
    owner: '{{ airflow_user }}'

- name: Install required Debian packages
  apt:
    pkg: '{{ item }}'
    state: present
    update_cache: yes
    cache_valid_time: 600
  with_items: "{{ airflow_deb_packages }}"

- name: Install required Python packages
  pip:
    name: '{{ item }}'
    state: present
  with_items: "{{ airflow_pip_packages }}"

- name: Install required Debian packages for plugins
  apt:
    pkg: '{{ item }}'
    state: present
    update_cache: yes
    cache_valid_time: 600
  with_flattened: "{{ airflow_plugins|map(attribute='required_debs')|list }}"

- name: Install required Python packages for plugins
  pip:
    name: '{{ item }}'
    state: present
  with_flattened: "{{ airflow_plugins|map(attribute='required_pips')|list }}"

- name: Install airflow (first time installation takes a long time)
  pip:
    name: 'airflow=={{ airflow_version }}'
    state: present

- name: Install airflow plugins
  pip:
    name: 'airflow[{{ item.name }}]=={{ airflow_version }}'
    state: present
  with_items: "{{ airflow_plugins }}"

- name: Generate Fernet key
  ferney_key:
    config_file: '{{ airflow_config}}'

- name: Setup configuration for airflow
  template:
    src: airflow.cfg.j2
    dest: '{{ airflow_config}}'
    owner: '{{ airflow_user }}'

- name: Remove uncompatible DAG example (example_twitter_dag.py)
  file:
    path: /usr/local/lib/python2.7/dist-packages/airflow/example_dags/example_twitter_dag.py
    state: absent

- name: Launch Airflow Initdb
  command: "/usr/local/bin/airflow initdb"
  register: airflow_initdb_result
  environment:
    AIRFLOW_HOME: "{{ airflow_home }}"
    PYTHONPATH: "{{ airflow_python_path }}"
  become: true
  become_user: '{{ airflow_user }}'
  changed_when: "airflow_initdb_result.rc != 0 or airflow_initdb_result.stdout.find('Running upgrade') >= 0"

- name: Launch Airflow Upgradedb
  command: "/usr/local/bin/airflow upgradedb"
  register: airflow_upgradedb_result
  environment:
    AIRFLOW_HOME: "{{ airflow_home }}"
    PYTHONPATH: "{{ airflow_python_path }}"
  become: true
  become_user: '{{ airflow_user }}'
  changed_when: "airflow_upgradedb_result.rc != 0 or airflow_upgradedb_result.stdout.find('Running upgrade') >= 0"

- name: Install Airflow scripts
  git:
    repo: '{{ item.url }}'
    version: "{{ item.version | default('latest') }}"
    verify_commit: "{{ item.verify_commit | default(False) }}"
    key_file: "{{ item.key_file | default(None) }}"
    dest: '{{ item.dest }}'
    depth: 1
    force: yes
    update: yes
  with_items: "{{ airflow_scripts }}"
  tags:
    - airflow_scripts

- include: marathon.yml
  when: airflow_startup == 'marathon'
{'fraction_non_alphanumeric': 0.1365594961882665, 'fraction_numerical': 0.004640371229698376, 'mean_word_length': 3.7783056215360253, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '56306', 'n_tokens_mistral': 2111, 'n_tokens_neox': 2066, 'n_words': 477}