<commit_before>---

- name: Install percolator dependencies
  apt: pkg={{ item }} state=installed
  with_items: percolator_deps

  - name: Patch xsds stream-extraction.hxx. Ugly.
    lineinfile: dest=/usr/include/xsd/cxx/tree/stream-extraction.hxx regexp='^(.*) push_back(.*)$' line={{ item }} backrefs=yes 
    with_items:
      - '\1 this->push_back\2' 
      - '\1  this->push_back\2' 

- name: Install pycolator dependencies
  apt: pkg={{ item }} state=installed
  with_items: pycolator_deps

- name: Install pycolator libraries from mirror
  pip: name={{ item }} state=present extra_args="--index-url=http://{{master_ip}}/pypimirror/simple"
       executable=pip3
  with_items: pycolator_pip_deps
  when: use_mirror is defined and use_mirror
- name: Install pycolator libraries from internet
  pip: name={{ item }} state=present executable=pip3
  with_items: pycolator_pip_deps
  when: use_mirror is not defined or not use_mirror

- name: Install msgfplus dependencies
  apt: pkg={{ item }} state=installed
  with_items: msgfplus_deps

<commit_msg>Bug in monkeypatching xsd<commit_after>---

- name: Install percolator dependencies
  apt: pkg={{ item }} state=installed
  with_items: percolator_deps

- name: Patch xsds stream-extraction.hxx. Ugly.
  lineinfile: dest=/usr/include/xsd/cxx/tree/stream-extraction.hxx regexp='^(.*) push_back(.*)$' line="{{ item }}" backrefs=yes 
  with_items:
    - '\1 this->push_back\2' 
    - '\1  this->push_back\2' 

- name: Install pycolator dependencies
  apt: pkg={{ item }} state=installed
  with_items: pycolator_deps

- name: Install pycolator libraries from mirror
  pip: name={{ item }} state=present extra_args="--index-url=http://{{master_ip}}/pypimirror/simple"
       executable=pip3
  with_items: pycolator_pip_deps
  when: use_mirror is defined and use_mirror
- name: Install pycolator libraries from internet
  pip: name={{ item }} state=present executable=pip3
  with_items: pycolator_pip_deps
  when: use_mirror is not defined or not use_mirror

- name: Install msgfplus dependencies
  apt: pkg={{ item }} state=installed
  with_items: msgfplus_deps


{'fraction_non_alphanumeric': 0.12037914691943127, 'fraction_numerical': 0.005687203791469194, 'mean_word_length': 4.251243781094527, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2496466', 'n_tokens_mistral': 750, 'n_tokens_neox': 709, 'n_words': 181}