<commit_before>remove_works:
  cmd.run:
    - name: docker run
            -v "{{ pillar['migration-data-folder'] }}:/migration/data"
            deichman/migration:{{ pillar['migration']['image-tag'] }}
            bash -c "sed -e 's/__HOST__/{{ pillar['redef']['services']['host'] }}:{{ pillar['redef']['services']['port'] }}/g' /sparql/000_delete_works_and_links_to_works.sparql | curl http://{{ pillar['redef']['fuseki']['host'] }}:{{ pillar['redef']['fuseki']['port'] }}/ds/query --data-urlencode query@-"
    - failhard: True

generate_works_from_publications:
  cmd.run:
    - name: docker run
            -v "{{ pillar['migration-data-folder'] }}:/migration/data"
            deichman/migration:{{ pillar['migration']['image-tag'] }}
            bash -c "sed -e 's/__HOST__/{{ pillar['redef']['services']['host'] }}:{{ pillar['redef']['services']['port'] }}/g' /sparql/001_generate_works_from_publications.sparql | curl http://{{ pillar['redef']['fuseki']['host'] }}:{{ pillar['redef']['fuseki']['port'] }}/ds/query --data-urlencode query@- > /migration/data/works_aggregated.ttl"
    - failhard: True

load_aggregated_works:
  cmd.run:
    - name: >
            docker run
            -v "{{ pillar['migration-data-folder'] }}:/migration/data" 
            deichman/migration:{{ pillar['migration']['image-tag'] }}
            bash -c "curl -X POST -H 'Content-Type: text/turtle' -d @/migration/data/works_aggregated.ttl http://{{ pillar['redef']['fuseki']['host'] }}:{{ pillar['redef']['fuseki']['port'] }}/ds/data?graph=default"
    - require:
      - cmd: generate_works_from_publications
    - failhard: True
<commit_msg>Use Fuseki's separate endpoint for update queries<commit_after>remove_works:
  cmd.run:
    - name: docker run
            -v "{{ pillar['migration-data-folder'] }}:/migration/data"
            deichman/migration:{{ pillar['migration']['image-tag'] }}
            bash -c "sed -e 's/__HOST__/{{ pillar['redef']['services']['host'] }}:{{ pillar['redef']['services']['port'] }}/g' /sparql/000_delete_works_and_links_to_works.sparql | curl http://{{ pillar['redef']['fuseki']['host'] }}:{{ pillar['redef']['fuseki']['port'] }}/ds/update --data-urlencode update@-"
    - failhard: True

generate_works_from_publications:
  cmd.run:
    - name: docker run
            -v "{{ pillar['migration-data-folder'] }}:/migration/data"
            deichman/migration:{{ pillar['migration']['image-tag'] }}
            bash -c "sed -e 's/__HOST__/{{ pillar['redef']['services']['host'] }}:{{ pillar['redef']['services']['port'] }}/g' /sparql/001_generate_works_from_publications.sparql | curl http://{{ pillar['redef']['fuseki']['host'] }}:{{ pillar['redef']['fuseki']['port'] }}/ds/query --data-urlencode query@- > /migration/data/works_aggregated.ttl"
    - failhard: True

load_aggregated_works:
  cmd.run:
    - name: >
            docker run
            -v "{{ pillar['migration-data-folder'] }}:/migration/data" 
            deichman/migration:{{ pillar['migration']['image-tag'] }}
            bash -c "curl -X POST -H 'Content-Type: text/turtle' -d @/migration/data/works_aggregated.ttl http://{{ pillar['redef']['fuseki']['host'] }}:{{ pillar['redef']['fuseki']['port'] }}/ds/data?graph=default"
    - require:
      - cmd: generate_works_from_publications
    - failhard: True

{'fraction_non_alphanumeric': 0.21869328493647913, 'fraction_numerical': 0.003629764065335753, 'mean_word_length': 3.6251748251748253, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6530110', 'n_tokens_mistral': 1169, 'n_tokens_neox': 1106, 'n_words': 178}