<commit_before>
variables:
  pip: pip3 --timeout 100 --retries 10


test:
  image: registry.gitlab.com/fdroid/ci-images-server:latest
  script:
    - $pip install -e .
    - cd tests
    - ./complete-ci-tests

# Test that the parsing of the .txt format didn't change from last
# released version. This uses the commit ID of the release tags,
# rather than the release tag itself so that contributor forks do not
# need to include the tags in them for this test to work.
#
# The COMMIT_ID should be bumped after each release, so that the list
# of sed hacks needed does not continuously grow.
metadata_v0:
  image: registry.gitlab.com/fdroid/ci-images-server:latest
  variables:
    RELEASE_COMMIT_ID: 4655e2e24ebd043be6faa4adf552db391caf2be9  # 1.1a~
  script:
    - git fetch https://gitlab.com/fdroid/fdroidserver $RELEASE_COMMIT_ID
    - cd tests
    - export GITCOMMIT=`git describe`
    - git checkout $RELEASE_COMMIT_ID
    - cd ..
    - git clone --depth 1 https://gitlab.com/fdroid/fdroiddata
    - cd fdroiddata
    - ../tests/dump_internal_metadata_format.py
    - cd ..
    - git reset --hard
    - git checkout $GITCOMMIT
    - cd fdroiddata
    - echo "accepted_formats = ('txt', 'yml')" >> config.py
    - ../tests/dump_internal_metadata_format.py
    - sed -i
          -e '/kivy:\sfalse/d'
          -e '/OpenCollective/d'
          metadata/dump_*/*.yaml
    - diff -uw metadata/dump_*

.apt-template: &apt-template
  variables:
    DEBIAN_FRONTEND: noninteractive
    LANG: C.UTF-8
  before_script:
    - echo Etc/UTC > /etc/timezone
    - echo 'APT::Install-Recommends "0";'
           'APT::Install-Suggests "0";'
           'APT::Acquire::Retries "20";'
           'APT::Get::Assume-Yes "true";'
           'Dpkg::Use-Pty "0";'
           'quiet "1";'
        >> /etc/apt/apt.conf.d/99gitlab
    - apt-get update
    - apt-get dist-upgrade

debian_testing:
  image: debian:testing
  <<: *apt-template
  only:
    - master@fdroid/fdroidserver
  script:
    - apt-get install
        aapt
        androguard
        fdroidserver
        git
        gnupg
        python3-defusedxml
        python3-setuptools
        zipalign
    - python3 -c 'import fdroidserver'
    - python3 -c 'import androguard'
    - cd tests
    - ./run-tests

# test using LTS set up with the PPA, including Recommends
ubuntu_lts_ppa:
  image: ubuntu:latest
  <<: *apt-template
  only:
    - master@fdroid/fdroidserver
  script:
    - export ANDROID_HOME=/usr/lib/android-sdk
    - apt-get install gnupg
    - while ! apt-key adv --keyserver hkp://pool.sks-keyservers.net --recv-key <KEY>; do sleep 15; done
    - export RELEASE=`sed -n 's,^deb [^ ][^ ]* \([a-z]*\).*,\1,p' /etc/apt/sources.list | head -1`
    - echo "deb http://ppa.launchpad.net/fdroid/fdroidserver/ubuntu $RELEASE main" >> /etc/apt/sources.list
    - apt-get update
    - apt-get dist-upgrade
    - apt-get install --install-recommends binfmt-support fdroidserver git python3-defusedxml python3-setuptools
    - grep binfmt /proc/modules || apt -qy purge apksigner
    - cd tests
    - ./run-tests

# test using Xenial LTS with all depends from pypi
# apksigner is recommended, but requires binfmt support in the kernel
ubuntu_xenial_pip:
  image: ubuntu:xenial
  <<: *apt-template
  only:
    - master@fdroid/fdroidserver
  script:
    - apt-get install git default-jdk-headless python3-pip python3-venv rsync zipalign libarchive13
    - rm -rf env
    - pyvenv env
    - . env/bin/activate
    - $pip install --upgrade babel pip setuptools
    - $pip install -e .
    - ./setup.py compile_catalog
    - ./tests/run-tests

pip_install:
  image: archlinux/base
  only:
    - master@fdroid/fdroidserver
  script:
    - pacman --sync --sysupgrade --refresh --noconfirm git grep python-pip python-virtualenv tar
    # setup venv to act as release build machine
    - python -m venv sdist-env
    - . sdist-env/bin/activate
    - ./setup.py compile_catalog sdist
    - deactivate
    - tar tzf dist/fdroidserver-*.tar.gz | grep locale/de/LC_MESSAGES/fdroidserver.mo
    # back to bare machine to act as user's install machine
    - pip install dist/fdroidserver-*.tar.gz
    - test -e /usr/share/locale/de/LC_MESSAGES/fdroidserver.mo
    - fdroid
    - fdroid readmeta
    - fdroid update --help

lint_format_safety_bandit_checks:
  image: alpine:3.7
  variables:
    LANG: C.UTF-8
  script:
    - apk add --no-cache bash dash ca-certificates python3
    - python3 -m ensurepip
    - $pip install Babel 'bandit<1.6.0' pycodestyle pyflakes 'pylint<2.0' safety
    - export EXITVALUE=0
    - ./hooks/pre-commit || export EXITVALUE=1
    - ./tests/test-gradlew-fdroid || export EXITVALUE=1
    - bandit
        -ii
        -s B110,B322,B404,B408,B410,B603,B607
        -r $CI_PROJECT_DIR fdroid
        || export EXITVALUE=1
    - safety check --full-report || export EXITVALUE=1
    - pylint --rcfile=.pylint-rcfile --output-format=colorized --reports=n
            fdroid
            makebuildserver
            setup.py
            fdroidserver/*.py
            tests/*.py
            tests/*.TestCase
        || export EXITVALUE=1
    - apk add --no-cache gettext make
    - make -C locale compile || export EXITVALUE=1
    - rm -f locale/*/*/*.mo
    - pybabel compile --domain=fdroidserver --directory locale 2>&1 | (grep -F "error:" && exit 1) || true
    - exit $EXITVALUE

fedora_latest:
  image: fedora:latest
  only:
    - master@fdroid/fdroidserver
  script:
    # tricks to hopefully make runs more reliable
    - echo "timeout=600" >> /etc/dnf/dnf.conf
    - echo "retries=50" >> /etc/dnf/dnf.conf
    - echo "keepcache=True" >> /etc/dnf/dnf.conf
<commit_msg>gitlab-ci: show clear error message when one step of job fails<commit_after>
variables:
  pip: pip3 --timeout 100 --retries 10


test:
  image: registry.gitlab.com/fdroid/ci-images-server:latest
  script:
    - $pip install -e .
    - cd tests
    - ./complete-ci-tests

# Test that the parsing of the .txt format didn't change from last
# released version. This uses the commit ID of the release tags,
# rather than the release tag itself so that contributor forks do not
# need to include the tags in them for this test to work.
#
# The COMMIT_ID should be bumped after each release, so that the list
# of sed hacks needed does not continuously grow.
metadata_v0:
  image: registry.gitlab.com/fdroid/ci-images-server:latest
  variables:
    RELEASE_COMMIT_ID: 4655e2e24ebd043be6faa4adf552db391caf2be9  # 1.1a~
  script:
    - git fetch https://gitlab.com/fdroid/fdroidserver $RELEASE_COMMIT_ID
    - cd tests
    - export GITCOMMIT=`git describe`
    - git checkout $RELEASE_COMMIT_ID
    - cd ..
    - git clone --depth 1 https://gitlab.com/fdroid/fdroiddata
    - cd fdroiddata
    - ../tests/dump_internal_metadata_format.py
    - cd ..
    - git reset --hard
    - git checkout $GITCOMMIT
    - cd fdroiddata
    - echo "accepted_formats = ('txt', 'yml')" >> config.py
    - ../tests/dump_internal_metadata_format.py
    - sed -i
          -e '/kivy:\sfalse/d'
          -e '/OpenCollective/d'
          metadata/dump_*/*.yaml
    - diff -uw metadata/dump_*

.apt-template: &apt-template
  variables:
    DEBIAN_FRONTEND: noninteractive
    LANG: C.UTF-8
  before_script:
    - echo Etc/UTC > /etc/timezone
    - echo 'APT::Install-Recommends "0";'
           'APT::Install-Suggests "0";'
           'APT::Acquire::Retries "20";'
           'APT::Get::Assume-Yes "true";'
           'Dpkg::Use-Pty "0";'
           'quiet "1";'
        >> /etc/apt/apt.conf.d/99gitlab
    - apt-get update
    - apt-get dist-upgrade

debian_testing:
  image: debian:testing
  <<: *apt-template
  only:
    - master@fdroid/fdroidserver
  script:
    - apt-get install
        aapt
        androguard
        fdroidserver
        git
        gnupg
        python3-defusedxml
        python3-setuptools
        zipalign
    - python3 -c 'import fdroidserver'
    - python3 -c 'import androguard'
    - cd tests
    - ./run-tests

# test using LTS set up with the PPA, including Recommends
ubuntu_lts_ppa:
  image: ubuntu:latest
  <<: *apt-template
  only:
    - master@fdroid/fdroidserver
  script:
    - export ANDROID_HOME=/usr/lib/android-sdk
    - apt-get install gnupg
    - while ! apt-key adv --keyserver hkp://pool.sks-keyservers.net --recv-key <KEY>; do sleep 15; done
    - export RELEASE=`sed -n 's,^deb [^ ][^ ]* \([a-z]*\).*,\1,p' /etc/apt/sources.list | head -1`
    - echo "deb http://ppa.launchpad.net/fdroid/fdroidserver/ubuntu $RELEASE main" >> /etc/apt/sources.list
    - apt-get update
    - apt-get dist-upgrade
    - apt-get install --install-recommends binfmt-support fdroidserver git python3-defusedxml python3-setuptools
    - grep binfmt /proc/modules || apt -qy purge apksigner
    - cd tests
    - ./run-tests

# test using Xenial LTS with all depends from pypi
# apksigner is recommended, but requires binfmt support in the kernel
ubuntu_xenial_pip:
  image: ubuntu:xenial
  <<: *apt-template
  only:
    - master@fdroid/fdroidserver
  script:
    - apt-get install git default-jdk-headless python3-pip python3-venv rsync zipalign libarchive13
    - rm -rf env
    - pyvenv env
    - . env/bin/activate
    - $pip install --upgrade babel pip setuptools
    - $pip install -e .
    - ./setup.py compile_catalog
    - ./tests/run-tests

pip_install:
  image: archlinux/base
  only:
    - master@fdroid/fdroidserver
  script:
    - pacman --sync --sysupgrade --refresh --noconfirm git grep python-pip python-virtualenv tar
    # setup venv to act as release build machine
    - python -m venv sdist-env
    - . sdist-env/bin/activate
    - ./setup.py compile_catalog sdist
    - deactivate
    - tar tzf dist/fdroidserver-*.tar.gz | grep locale/de/LC_MESSAGES/fdroidserver.mo
    # back to bare machine to act as user's install machine
    - pip install dist/fdroidserver-*.tar.gz
    - test -e /usr/share/locale/de/LC_MESSAGES/fdroidserver.mo
    - fdroid
    - fdroid readmeta
    - fdroid update --help

lint_format_safety_bandit_checks:
  image: alpine:3.7
  variables:
    LANG: C.UTF-8
  script:
    - apk add --no-cache bash dash ca-certificates python3
    - python3 -m ensurepip
    - $pip install Babel 'bandit<1.6.0' pycodestyle pyflakes 'pylint<2.0' safety
    - export EXITVALUE=0
    - function set_error() { export EXITVALUE=1; printf "\x1b[31mERROR `history|tail -2|head -1|cut -b 6-500`\x1b[0m\n"; }
    - ./hooks/pre-commit || set_error
    - ./tests/test-gradlew-fdroid || set_error
    - bandit
        -ii
        -s B110,B322,B404,B408,B410,B603,B607
        -r $CI_PROJECT_DIR fdroid
        || set_error
    - safety check --full-report || set_error
    - pylint --rcfile=.pylint-rcfile --output-format=colorized --reports=n
            fdroid
            makebuildserver
            setup.py
            fdroidserver/*.py
            tests/*.py
            tests/*.TestCase
        || set_error
    - apk add --no-cache gettext make
    - make -C locale compile || set_error
    - rm -f locale/*/*/*.mo
    - pybabel compile --domain=fdroidserver --directory locale 2>&1 | (grep -F "error:" && exit 1) || true
    - exit $EXITVALUE

fedora_latest:
  image: fedora:latest
  only:
    - master@fdroid/fdroidserver
  script:
    # tricks to hopefully make runs more reliable
    - echo "timeout=600" >> /etc/dnf/dnf.conf
    - echo "retries=50" >> /etc/dnf/dnf.conf
    - echo "keepcache=True" >> /etc/dnf/dnf.conf

{'fraction_non_alphanumeric': 0.11652021089630932, 'fraction_numerical': 0.017926186291739893, 'mean_word_length': 2.3751482799525503, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 33, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5123782', 'n_tokens_mistral': 4119, 'n_tokens_neox': 3892, 'n_words': 989}