<commit_before>---
# Whether to configure the project application code.
# Allows skipping the code tasks if only the database
# tasks should be run, which is useful in a multi-host setup.
fpsd_crawler_configure_service: true

# General system dependencies requiring for running
# Tor Browser Bundle (tbb).
fpsd_crawler_apt_packages:
  - aptitude # needed by Ansible for apt 'upgrade'
  - git #tbs
  - gnupg-curl # needed for https keyservers
  - libasound2 # tbb (firefox)
  - libdbus-glib-1-2 # tbb (firefox)
  - libfontconfig1 # tbb (firefox)
  - libgtk2.0-0 # tbb (firefox)
  - libpq-dev # needed to install psycopg2
  - libxrender1 # tbb (firefox)
  - python3-pip
  - python3.5
  - tmux
  - vim
  - xvfb # tbs

# General Python tooling for development. Specific Python dependencies
# for the crawler code are installed from `requirements.txt` files.
fpsd_crawler_pypi_packages:
  - pip
  - pytest
  - ipython
  - ipdb

# Specify TBB version to download and install
fpsd_crawler_tbb_release: 6.0.7
fpsd_crawler_tbb_locale: en-US
fpsd_crawler_tbb_arch: 64
fpsd_crawler_tbb_directory: "/opt/tbb"

# Specify Tor version to download, patch, build, and install
fpsd_crawler_tor_release: 0.2.8.8
fpsd_crawler_tor_directory: "/usr/local/src/tor"

# Clone the git repo by default, but allow disabling, e.g. in vagrant,
# where it can be mounted in.
fpsd_crawler_clone_git_repo: true
fpsd_crawler_project_directory: "/opt/fingerprint-securedrop"
fpsd_crawler_git_repo_url: "https://github.com/freedomofpress/fingerprint-securedrop"

# Username for running the crawler systemd service.
fpsd_crawler_system_account: fpsd

# Configuration options for Postgres database.
fpsd_database_apt_packages:
  - postgresql
  - postgresql-contrib
  - libpq-dev
  - python-psycopg2

# List of hidden service directories to sort
fpsd_onion_dirs:
  - http://secrdrop5wyphb5x.onion/sites/securedrop.org/files/securedrop_list.txt
  - http://msydqstlz2kzerdg.onion/address/
  - http://skunksworkedp2cg.onion/sites.html

# List of preferred EntryNodes. Tor will make the first sequentially listed one
# reachable the client guard.
fpsd_entry_nodes: 
  - 237C733E2D1BAD26FAABEAFB612BDA0CD3BC6AF4
  - 95CDFAA852FF075FCD23102C06636F2B84B3371A
  - E727D0B4179549BB3F82ED3C256F209E54CE0B23
  - 41E2FB97690EED79DD4BD6BAC00A974B69F49858

# If left undefined, a randomly generated password will be created and written
# to /tmp on the Ansible controller.
fpsd_database_password: "{{ lookup('password', '/tmp/passwordfile chars=ascii_letters,digits') }}"

# All of these vars except password will be set as environment variables in
# `/etc/bash.bashrc`. If password is not set, then Ansible will generate one for
# you at provision-time. The password will not be set as an environment
# variable, but is instead saved to a PGPASSFILE, `~{{ ssh_username }}/.pgpass`,
# along with the corresponding information from the other vars according to the
# PGPASSFILE spec.
#
# Note: the database role will not overwrite an existing PGPASSFILE, so in order
# to do so, you'll have to first remove it then re-run the role:
# `ANSIBLE_ARGS="--tags=database vagrant provision"`.
fpsd_database_psql_env:
  PGHOST: localhost
  PGUSER: fp_user
  PGPORT: "5432"
  PGDATABASE: fpsd
  PGPASSWORD: "{{ fpsd_database_password }}"

# Defaults to true. Set to false if you want to use a remote database. If you
# wish not to use the database at all, set the environment variable
# `ANSIBLE_ARGS="--skip-tags=database"`. If set to false you must fill out all
# vars in the field below so Ansible can still set the appropriate environment
# variables and create the PGPASSFILE for it just works™ access to your remote
# database.
fpsd_database_initialize_local_db: true

# Whether to configure credentials for user accounts.
fpsd_database_configure_credentials: true
fpsd_database_usernames:
  - "{{ ansible_user|default(ansible_ssh_user|default(lookup('env', 'USER'))) }}"
<commit_msg>Bump Tor Browser version to 6.0.8 so that playbooks do not fail<commit_after>---
# Whether to configure the project application code.
# Allows skipping the code tasks if only the database
# tasks should be run, which is useful in a multi-host setup.
fpsd_crawler_configure_service: true

# General system dependencies requiring for running
# Tor Browser Bundle (tbb).
fpsd_crawler_apt_packages:
  - aptitude # needed by Ansible for apt 'upgrade'
  - git #tbs
  - gnupg-curl # needed for https keyservers
  - libasound2 # tbb (firefox)
  - libdbus-glib-1-2 # tbb (firefox)
  - libfontconfig1 # tbb (firefox)
  - libgtk2.0-0 # tbb (firefox)
  - libpq-dev # needed to install psycopg2
  - libxrender1 # tbb (firefox)
  - python3-pip
  - python3.5
  - tmux
  - vim
  - xvfb # tbs

# General Python tooling for development. Specific Python dependencies
# for the crawler code are installed from `requirements.txt` files.
fpsd_crawler_pypi_packages:
  - pip
  - pytest
  - ipython
  - ipdb

# Specify TBB version to download and install
fpsd_crawler_tbb_release: 6.0.8
fpsd_crawler_tbb_locale: en-US
fpsd_crawler_tbb_arch: 64
fpsd_crawler_tbb_directory: "/opt/tbb"

# Specify Tor version to download, patch, build, and install
fpsd_crawler_tor_release: 0.2.8.8
fpsd_crawler_tor_directory: "/usr/local/src/tor"

# Clone the git repo by default, but allow disabling, e.g. in vagrant,
# where it can be mounted in.
fpsd_crawler_clone_git_repo: true
fpsd_crawler_project_directory: "/opt/fingerprint-securedrop"
fpsd_crawler_git_repo_url: "https://github.com/freedomofpress/fingerprint-securedrop"

# Username for running the crawler systemd service.
fpsd_crawler_system_account: fpsd

# Configuration options for Postgres database.
fpsd_database_apt_packages:
  - postgresql
  - postgresql-contrib
  - libpq-dev
  - python-psycopg2

# List of hidden service directories to sort
fpsd_onion_dirs:
  - http://secrdrop5wyphb5x.onion/sites/securedrop.org/files/securedrop_list.txt
  - http://msydqstlz2kzerdg.onion/address/
  - http://skunksworkedp2cg.onion/sites.html

# List of preferred EntryNodes. Tor will make the first sequentially listed one
# reachable the client guard.
fpsd_entry_nodes:
  - 237C733E2D1BAD26FAABEAFB612BDA0CD3BC6AF4
  - 95CDFAA852FF075FCD23102C06636F2B84B3371A
  - E727D0B4179549BB3F82ED3C256F209E54CE0B23
  - 41E2FB97690EED79DD4BD6BAC00A974B69F49858

# If left undefined, a randomly generated password will be created and written
# to /tmp on the Ansible controller.
fpsd_database_password: "{{ lookup('password', '/tmp/passwordfile chars=ascii_letters,digits') }}"

# All of these vars except password will be set as environment variables in
# `/etc/bash.bashrc`. If password is not set, then Ansible will generate one for
# you at provision-time. The password will not be set as an environment
# variable, but is instead saved to a PGPASSFILE, `~{{ ssh_username }}/.pgpass`,
# along with the corresponding information from the other vars according to the
# PGPASSFILE spec.
#
# Note: the database role will not overwrite an existing PGPASSFILE, so in order
# to do so, you'll have to first remove it then re-run the role:
# `ANSIBLE_ARGS="--tags=database vagrant provision"`.
fpsd_database_psql_env:
  PGHOST: localhost
  PGUSER: fp_user
  PGPORT: "5432"
  PGDATABASE: fpsd
  PGPASSWORD: "{{ <PASSWORD>database_password }}"

# Defaults to true. Set to false if you want to use a remote database. If you
# wish not to use the database at all, set the environment variable
# `ANSIBLE_ARGS="--skip-tags=database"`. If set to false you must fill out all
# vars in the field below so Ansible can still set the appropriate environment
# variables and create the PGPASSFILE for it just works™ access to your remote
# database.
fpsd_database_initialize_local_db: true

# Whether to configure credentials for user accounts.
fpsd_database_configure_credentials: true
fpsd_database_usernames:
  - "{{ ansible_user|default(ansible_ssh_user|default(lookup('env', 'USER'))) }}"

{'fraction_non_alphanumeric': 0.07842145206172527, 'fraction_numerical': 0.031242094611687327, 'mean_word_length': 4.619758351101635, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6415341', 'n_tokens_mistral': 2909, 'n_tokens_neox': 2713, 'n_words': 827}