<commit_before>{% set environment = salt.grains.get('environment', 'data-qa') %}
{% set open_env_map = {
  'data-qa': 'rc-apps',
  'data-production': 'production-apps'
} %}

dagster:
  pipeline_configs:
    open-discussions:
      resources:
        postgres_db:
          config:
            postgres_db_name: opendiscussions
            postgres_hostname: {{ open_env_map[environment] }}-rds-postgresql-opendiscussions.cbnm7ajau6mi.us-east-1.rds.amazonaws.com
            postgres_password: __vault__:cache:postgres-{{ open_env_map[environment] }}-opendiscussions/creds/readonly>data>password
            postgres_port: 5432
            postgres_username: __vault__:cache:postgres-{{ open_env_map[environment] }}-opendiscussions/creds/readonly>data>username
      solids:
        fetch_open_run_data:
          config:
            outputs_dir: s3://mitodl-data-lake/mit-open/course-runs/
            file_base: run_data
        fetch_open_user_data:
          config:
            file_base: user_data
            outputs_dir: s3://mitodl-data-lake/mit-open/users/<commit_msg>Adding solid config for updated Dagster pipeline<commit_after>{% set environment = salt.grains.get('environment', 'data-qa') %}
{% set open_env_map = {
  'data-qa': 'rc-apps',
  'data-production': 'production-apps'
} %}

dagster:
  pipeline_configs:
    open-discussions:
      resources:
        postgres_db:
          config:
            postgres_db_name: opendiscussions
            postgres_hostname: {{ open_env_map[environment] }}-rds-postgresql-opendiscussions.cbnm7ajau6mi.us-east-1.rds.amazonaws.com
            postgres_password: __<PASSWORD>__:cache:postgres-{{ open_env_map[environment] }}-opendiscussions/creds/readonly>data>password
            postgres_port: 5432
            postgres_username: __vault__:cache:postgres-{{ open_env_map[environment] }}-opendiscussions/creds/readonly>data>username
      solids:
        fetch_open_run_data:
          config:
            outputs_dir: s3://mitodl-data-lake/mit-open/course-runs/
        fetch_open_user_data:
          config:
            outputs_dir: s3://mitodl-data-lake/mit-open/users/
        fetch_open_course_data:
          config:
            outputs_dir: s3://mitodl-data-lake/mit-open/courses/
{'fraction_non_alphanumeric': 0.11586452762923351, 'fraction_numerical': 0.008467023172905526, 'mean_word_length': 2.864027538726334, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '636092', 'n_tokens_mistral': 767, 'n_tokens_neox': 739, 'n_words': 95}