<commit_before>
{% from 'lib.sls' import createuser, apache, uwsgi  %}

include:
  - apache
  - uwsgi
  - postgres10
  - letsencrypt

360-datastore-packages:
  pkg.installed:
    - pkgs:
      - virtualenv
      - python3-dev
      - build-essential
      - postgresql-10
      - postgresql-server-dev-10
      - postgresql-client
      - libapache2-mod-proxy-uwsgi
      - uwsgi-plugin-python3
    - watch_in:
      - service: apache2
      - service: uwsgi

  apache_module.enabled:
    - name: proxy proxy_uwsgi
    - watch_in:
      - service: apache2
      - service: uwsgi

{{ createuser(pillar.datastore.user) }}

##### Git checkout

/home/{{ pillar.datastore.user }}/datastore:
  git.latest:
    - name: {{ pillar.datastore.git_url }}
    - rev: {{ pillar.datastore.branch }}
    - target: /home/{{ pillar.datastore.user }}/datastore
    - user: {{ pillar.datastore.user }}
    - force_fetch: True
    - force_reset: True
    - require:
      - pkg: git

/home/{{ pillar.datastore.user }}/datatester:
  git.latest:
    - name: https://github.com/ThreeSixtyGiving/datatester.git
    - rev: {{ pillar.datatester.branch }}
    - target: /home/{{ pillar.datastore.user }}/datatester
    - user: {{ pillar.datastore.user }}
    - force_fetch: True
    - force_reset: True
    - require:
      - pkg: git


##### Install python deps in virtualenvs

/home/{{ pillar.datastore.user }}/datastore/.ve:
  virtualenv.managed:
    - python: /usr/bin/python3
    - user: {{ pillar.datastore.user }}
    - system_site_packages: False
    - requirements: /home/{{ pillar.datastore.user }}/datastore/requirements.txt
    - require:
      - git: /home/{{ pillar.datastore.user }}/datastore

/home/{{ pillar.datastore.user }}/datatester/.ve:
  virtualenv.managed:
    - python: /usr/bin/python3
    - user: {{ pillar.datastore.user }}
    - system_site_packages: False
    - requirements: /home/{{ pillar.datastore.user }}/datatester/requirements.txt
    - require:
      - git: /home/{{ pillar.datastore.user }}/datatester

##### Web user / pass

/etc/apache2/360.htpasswd:
  file.managed:
    - source: salt://private/datastore/360.htpasswd
    - makedirs: True


##### Datastore configs setup

/home/{{ pillar.datastore.user }}/datastore/datastore/settings/settings_360g.py:
  file.managed:
    - source: salt://private/datastore/settings_360g.py
    - template: jinja
    - context:
       datastore: {{ pillar.datastore }}
       datastore_private: {{ pillar.datastore_private }}
    - require:
      - git: /home/{{ pillar.datastore.user }}/datastore


/home/{{ pillar.datastore.user }}/data_run_config.sh:
  file.managed:
    - source: salt://private/datastore/data_run_config.sh
    - template: jinja
    - context:
       datastore: {{ pillar.datastore }}
       datastore_private: {{ pillar.datastore_private }}
    - require:
      - git: /home/{{ pillar.datastore.user }}/datastore

##### Database

# Note on some VMs if the locale isn't setup correctly this will fail silently
postgresql:
  service:
    - running
    - enable: True
    - reload: True

database_setup:

  postgres_user.present:
    - name: {{ pillar.datastore.user }}
    - password: {{ pillar.datastore_private.postgresql_password }}
    - require:
      - service: postgresql

  postgres_database.present:
    - name: {{ pillar.datastore.database_name }}
    - owner: {{ pillar.datastore.user }}
    - require:
      - service: postgresql
      - postgres_user: {{ pillar.datastore.user }}


##### Colab

/etc/postgresql/10/main/pg_hba.conf:
  file.managed:
    - source: salt://postgres/pg10_hba_360_datastore.conf

/etc/postgresql/10/main/conf.d/listen_all.conf:
  file.managed:
    - source: salt://postgres/postgres_conf_d_listen_all.conf

colab_notebooks:
  postgres_user.present:
    - password: {{ pillar.datastore_private.colab_pg_password }}
    - groups: readaccess


##### Datastore directories

{% for dir in [pillar.datastore.reports_dir,
               pillar.datastore.grantnav_packages_dir,
               pillar.datastore.static_dir]
%}

{{ dir }}:
  file.directory:
    - user: datastore
    - group: www-data
    - mode: 755
    - makedirs: True
<commit_msg>360 datastore Put postgresql config updates into separate task<commit_after>
{% from 'lib.sls' import createuser, apache, uwsgi  %}

include:
  - apache
  - uwsgi
  - postgres10
  - letsencrypt

360-datastore-packages:
  pkg.installed:
    - pkgs:
      - virtualenv
      - python3-dev
      - build-essential
      - postgresql-10
      - postgresql-server-dev-10
      - postgresql-client
      - libapache2-mod-proxy-uwsgi
      - uwsgi-plugin-python3
    - watch_in:
      - service: apache2
      - service: uwsgi

  apache_module.enabled:
    - name: proxy proxy_uwsgi
    - watch_in:
      - service: apache2
      - service: uwsgi

{{ createuser(pillar.datastore.user) }}

##### Git checkout

/home/{{ pillar.datastore.user }}/datastore:
  git.latest:
    - name: {{ pillar.datastore.git_url }}
    - rev: {{ pillar.datastore.branch }}
    - target: /home/{{ pillar.datastore.user }}/datastore
    - user: {{ pillar.datastore.user }}
    - force_fetch: True
    - force_reset: True
    - require:
      - pkg: git

/home/{{ pillar.datastore.user }}/datatester:
  git.latest:
    - name: https://github.com/ThreeSixtyGiving/datatester.git
    - rev: {{ pillar.datatester.branch }}
    - target: /home/{{ pillar.datastore.user }}/datatester
    - user: {{ pillar.datastore.user }}
    - force_fetch: True
    - force_reset: True
    - require:
      - pkg: git


##### Install python deps in virtualenvs

/home/{{ pillar.datastore.user }}/datastore/.ve:
  virtualenv.managed:
    - python: /usr/bin/python3
    - user: {{ pillar.datastore.user }}
    - system_site_packages: False
    - requirements: /home/{{ pillar.datastore.user }}/datastore/requirements.txt
    - require:
      - git: /home/{{ pillar.datastore.user }}/datastore

/home/{{ pillar.datastore.user }}/datatester/.ve:
  virtualenv.managed:
    - python: /usr/bin/python3
    - user: {{ pillar.datastore.user }}
    - system_site_packages: False
    - requirements: /home/{{ pillar.datastore.user }}/datatester/requirements.txt
    - require:
      - git: /home/{{ pillar.datastore.user }}/datatester

##### Web user / pass

/etc/apache2/360.htpasswd:
  file.managed:
    - source: salt://private/datastore/360.htpasswd
    - makedirs: True


##### Datastore configs setup

/home/{{ pillar.datastore.user }}/datastore/datastore/settings/settings_360g.py:
  file.managed:
    - source: salt://private/datastore/settings_360g.py
    - template: jinja
    - context:
       datastore: {{ pillar.datastore }}
       datastore_private: {{ pillar.datastore_private }}
    - require:
      - git: /home/{{ pillar.datastore.user }}/datastore


/home/{{ pillar.datastore.user }}/data_run_config.sh:
  file.managed:
    - source: salt://private/datastore/data_run_config.sh
    - template: jinja
    - context:
       datastore: {{ pillar.datastore }}
       datastore_private: {{ pillar.datastore_private }}
    - require:
      - git: /home/{{ pillar.datastore.user }}/datastore

##### Database

# Note on some VMs if the locale isn't setup correctly this will fail silently
postgresql:
  service:
    - running
    - enable: True
    - reload: True

database_setup:

  postgres_user.present:
    - name: {{ pillar.datastore.user }}
    - password: {{ pillar.datastore_private.postgresql_password }}
    - require:
      - service: postgresql

  postgres_database.present:
    - name: {{ pillar.datastore.database_name }}
    - owner: {{ pillar.datastore.user }}
    - require:
      - service: postgresql
      - postgres_user: {{ pillar.datastore.user }}


##### Colab

colab_notebooks:
  postgres_user.present:
    - password: {{ pillar.datastore_private.colab_pg_password }}
    - groups: readaccess
    - require:
      - service: postgresql

update_postgres_config_colab_user_access:
  file.managed:
    - name: /etc/postgresql/10/main/pg_hba.conf
    - source: salt://postgres/pg10_hba_360_datastore.conf

update_postgres_listen_all:
  file.managed:
    - name: /etc/postgresql/10/main/conf.d/listen_all.conf
    - source: salt://postgres/postgres_conf_d_listen_all.conf

##### Datastore directories

{% for dir in [pillar.datastore.reports_dir,
               pillar.datastore.grantnav_packages_dir,
               pillar.datastore.static_dir]
%}

{{ dir }}:
  file.directory:
    - user: datastore
    - group: www-data
    - mode: 755
    - makedirs: True

{'fraction_non_alphanumeric': 0.13869560089633212, 'fraction_numerical': 0.010024767071588631, 'mean_word_length': 2.6504520017219115, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7622518', 'n_tokens_mistral': 3134, 'n_tokens_neox': 2956, 'n_words': 509}