<commit_before>sudo: required
dist: trusty
language: generic

env:
    matrix:
        # Basic build only running nosetests, one using apt and one running in
        # a virtualenv.
        - INSTALL=venv UPGRADE=false
        - INSTALL=apt UPGRADE=false
        # Test upgrade by first installing stable package from official repo
        # and then build and upgrade to new packages. Not applicable to
        # virtualenv since the upgrade process specifically tests the apt
        # package upgrade procedure.
        - INSTALL=apt UPGRADE=true

matrix:
    # don't wait for the above jobs, which are allowed to fail, before marking
    # the whole build complete
    fast_finish: true

install:
    # -- common operations to apt install and virtualenv -----------------------
    # install dependencies for building packages and build NIPAP debian packages
    - sudo apt-get update
    - sudo apt-get install -qq -y --force-yes devscripts python-dev python3-dev python-docutils debhelper dh-python junit4 python3-pip python-nose default-jdk gradle
    # configure .nipaprc
    - sed -e 's/username = guest/username = unittest/' -e 's/password = <PASSWORD>/password = <PASSWORD>/' nipap-cli/nipaprc > ~/.nipaprc
    - chmod 0600 ~/.nipaprc
    # install postgresql + ip4r + contrib
    - sudo apt-get install -qq -y postgresql-9.2 postgresql-9.2-ip4r postgresql-contrib-9.2

    # -- virtualenv build ------------------------------------------------------
    # install nipap dependencies
    - if [ "$INSTALL" == "venv" ]; then sudo -H pip install -r nipap/requirements.txt; fi
    # SQL
    - if [ "$INSTALL" == "venv" ]; then sudo su -c "cd nipap/sql; PGPASSWORD=papin make install" postgres; fi
    # move configuration file into place
    - if [ "$INSTALL" == "venv" ]; then sudo mkdir /etc/nipap; sudo cp nipap/nipap.conf.dist /etc/nipap/nipap.conf; sudo sed -e "s/{{LISTEN_ADDRESS}}/127.0.0.1/" -e "s/{{LISTEN_PORT}}/1337/" -e "s/{{DB_USERNAME}}/nipap/" -e "s/{{DB_NAME}}/nipap/" -e "s/{{DB_PASSWORD}}/papin/" -e "s/{{DB_SSLMODE}}/require/" -e "s/{{DB_PORT}}/5432/" -e "s/{{DB_HOST}}/localhost/" -e "s/{{SYSLOG}}/true/" -i /etc/nipap/nipap.conf; fi
    # create local user for unittest
    - if [ "$INSTALL" == "venv" ]; then sudo nipap/nipap-passwd create-database; sudo nipap/nipap-passwd add -u unittest -p gottatest -n unittest; fi
    - if [ "$INSTALL" == "venv" ]; then sudo nipap/nipap-passwd add -u readonly -p gottatest --readonly -n "Read-only user for running unit tests"; fi
    # install pynipap
    - if [ "$INSTALL" == "venv" ]; then cd pynipap; sudo python setup.py install; cd ..; fi
    - if [ "$INSTALL" == "venv" ]; then cd pynipap; sudo python3 setup.py install; cd ..; fi
    # install nipap-cli dependencies
    - if [ "$INSTALL" == "venv" ]; then sudo -H pip3 install -r nipap-cli/requirements.txt; fi
    # start nipap backend
    - if [ "$INSTALL" == "venv" ]; then nipap/nipapd --no-pid-file -c /etc/nipap/nipap.conf; fi

    # -- APT build --------------------------------------------------------------
    # add NIPAP official Debian repo and keys, we use it to get ip4r
    - if [ "$INSTALL" == "apt" ]; then echo "deb http://spritelink.github.io/NIPAP/repos/apt stable main extra" | sudo tee /etc/apt/sources.list.d/nipap.list; fi
    - if [ "$INSTALL" == "apt" ]; then wget -O - https://spritelink.github.io/NIPAP/nipap.gpg.key | sudo apt-key add -; fi
    - if [ "$INSTALL" == "apt" ]; then sudo apt-get update -qq; fi
    # install dependencies for installing & running nipap
    - if [ "$INSTALL" == "apt" ]; then sudo apt-get install -qq -y --force-yes python-pysqlite2 python-psycopg2 python-ipy python3-ipy python-docutils postgresql-9.2 postgresql-9.2-ip4r python-tornado python-flask python-flask-xml-rpc python-flask-compress python-parsedatetime python-pylons python-tz python-dateutil python-psutil python-pyparsing; fi
    # if we are testing the upgrade, first install NIPAP packages from official repo
    - if [ "$INSTALL" == "apt" ] && [ "$UPGRADE" == "true" ]; then sudo apt-get install -qq nipapd nipap-www nipap-cli; fi
    # bump version so that we know we are upgrading beyond what is installed
    - if [ "$INSTALL" == "apt" ] && [ "$UPGRADE" == "true" ]; then (echo -e 'Version 9999.9.9\n------------------\n * Test version for Travis-CI automatic upgrade test'; cat NEWS) > NEWS2; mv NEWS2 NEWS; make bumpversion; fi
    # populate answers to nipapd package install questions and reconfigure
    - if [ "$INSTALL" == "apt" ] && [ "$UPGRADE" == "true" ]; then echo 'set nipapd/autoconf true' | sudo debconf-communicate; echo 'set nipapd/database_host localhost' | sudo debconf-communicate; echo 'set nipapd/local_db_autoconf true' | sudo debconf-communicate; echo 'set nipapd/startup true' | sudo debconf-communicate; echo 'set nipapd/local_db_upgrade true' | sudo debconf-communicate; sudo dpkg-reconfigure nipapd; fi
    # create local user for unittest and restart
    - if [ "$INSTALL" == "apt" ] && [ "$UPGRADE" == "true" ]; then sudo nipap/nipap-passwd add -u unittest -p gottatest -f /etc/nipap/local_auth.db -n unittest; sudo /etc/init.d/nipapd restart; fi
    # if upgrade, add some data to the database that we can verify later
    - if [ "$INSTALL" == "apt" ] && [ "$UPGRADE" == "true" ]; then nosetests tests/upgrade-before.py; fi
    # build new NIPAP packages
    - if [ "$INSTALL" == "apt" ]; then make builddeb; fi
    # install the newly built nipap packages
    - if [ "$INSTALL" == "apt" ]; then sudo dpkg --force-confnew -i nipap*.deb python*-pynipap*.deb; fi
    # populate answers to nipapd package install questions and reconfigure
    - if [ "$INSTALL" == "apt" ]; then echo 'set nipapd/database_host localhost' | sudo debconf-communicate; echo 'set nipapd/local_db_autoconf true' | sudo debconf-communicate; echo 'set nipapd/startup true' | sudo debconf-communicate; echo 'set nipapd/local_db_upgrade true' | sudo debconf-communicate; sudo dpkg-reconfigure nipapd; fi
    # create local user for unittests
    - if [ "$INSTALL" == "apt" ]; then sudo nipap/nipap-passwd add -u unittest -p gottatest -f /etc/nipap/local_auth.db -n "User for running unit tests"; fi
    - if [ "$INSTALL" == "apt" ]; then sudo nipap/nipap-passwd add -u readonly -p gottatest -f /etc/nipap/local_auth.db --readonly -n "Read-only user for running unit tests"; fi
    - if [ "$INSTALL" == "apt" ]; then sudo sed -e "s/db_host *= *[^ ]\+/db_host = localhost/" -i /etc/nipap/nipap.conf; fi
    # Don't need to restart, but I would like to make sure that all is updated
    # (like local_auth.db is re-read and so forth). Read #654 for more
    # information.
    #- if [ "$INSTALL" == "apt" ]; then sudo /etc/init.d/nipapd restart; fi

script:
    - cd tests
    # if upgrade, verify data loaded before upgrade looks correct post upgrade
    - if [ "$INSTALL" == "apt" ] && [ "$UPGRADE" == "true" ]; then nosetests upgrade-after.py; fi<commit_msg>ci: Upgrade dpkg to add support for xc<commit_after>sudo: required
dist: trusty
language: generic

env:
    matrix:
        # Basic build only running nosetests, one using apt and one running in
        # a virtualenv.
        - INSTALL=venv UPGRADE=false
        - INSTALL=apt UPGRADE=false
        # Test upgrade by first installing stable package from official repo
        # and then build and upgrade to new packages. Not applicable to
        # virtualenv since the upgrade process specifically tests the apt
        # package upgrade procedure.
        - INSTALL=apt UPGRADE=true

matrix:
    # don't wait for the above jobs, which are allowed to fail, before marking
    # the whole build complete
    fast_finish: true

install:
    # -- common operations to apt install and virtualenv -----------------------
    # install dependencies for building packages and build NIPAP debian packages
    - sudo apt-get update
    - sudo apt-get install -qq -y --force-yes devscripts python-dev python3-dev python-docutils debhelper dh-python junit4 python3-pip python-nose default-jdk gradle
    # configure .nipaprc
    - sed -e 's/username = guest/username = unittest/' -e 's/password = <PASSWORD>/password = <PASSWORD>/' nipap-cli/nipaprc > ~/.nipaprc
    - chmod 0600 ~/.nipaprc
    # install postgresql + ip4r + contrib
    - sudo apt-get install -qq -y postgresql-9.2 postgresql-9.2-ip4r postgresql-contrib-9.2

    # -- virtualenv build ------------------------------------------------------
    # install nipap dependencies
    - if [ "$INSTALL" == "venv" ]; then sudo -H pip install -r nipap/requirements.txt; fi
    # SQL
    - if [ "$INSTALL" == "venv" ]; then sudo su -c "cd nipap/sql; PGPASSWORD=<PASSWORD> make install" postgres; fi
    # move configuration file into place
    - if [ "$INSTALL" == "venv" ]; then sudo mkdir /etc/nipap; sudo cp nipap/nipap.conf.dist /etc/nipap/nipap.conf; sudo sed -e "s/{{LISTEN_ADDRESS}}/127.0.0.1/" -e "s/{{LISTEN_PORT}}/1337/" -e "s/{{DB_USERNAME}}/nipap/" -e "s/{{DB_NAME}}/nipap/" -e "s/{{DB_PASSWORD}}/papin/" -e "s/{{DB_SSLMODE}}/require/" -e "s/{{DB_PORT}}/5432/" -e "s/{{DB_HOST}}/localhost/" -e "s/{{SYSLOG}}/true/" -i /etc/nipap/nipap.conf; fi
    # create local user for unittest
    - if [ "$INSTALL" == "venv" ]; then sudo nipap/nipap-passwd create-database; sudo nipap/nipap-passwd add -u unittest -p gottatest -n unittest; fi
    - if [ "$INSTALL" == "venv" ]; then sudo nipap/nipap-passwd add -u readonly -p gottatest --readonly -n "Read-only user for running unit tests"; fi
    # install pynipap
    - if [ "$INSTALL" == "venv" ]; then cd pynipap; sudo python setup.py install; cd ..; fi
    - if [ "$INSTALL" == "venv" ]; then cd pynipap; sudo python3 setup.py install; cd ..; fi
    # install nipap-cli dependencies
    - if [ "$INSTALL" == "venv" ]; then sudo -H pip3 install -r nipap-cli/requirements.txt; fi
    # start nipap backend
    - if [ "$INSTALL" == "venv" ]; then nipap/nipapd --no-pid-file -c /etc/nipap/nipap.conf; fi

    # -- APT build --------------------------------------------------------------
    # add NIPAP official Debian repo and keys, we use it to get ip4r
    - if [ "$INSTALL" == "apt" ]; then echo "deb http://spritelink.github.io/NIPAP/repos/apt stable main extra" | sudo tee /etc/apt/sources.list.d/nipap.list; fi
    - if [ "$INSTALL" == "apt" ]; then wget -O - https://spritelink.github.io/NIPAP/nipap.gpg.key | sudo apt-key add -; fi
    - if [ "$INSTALL" == "apt" ]; then sudo apt-get update -qq; fi
    # install dependencies for installing & running nipap
    - if [ "$INSTALL" == "apt" ]; then sudo apt-get install -qq -y --force-yes dpkg python-pysqlite2 python-psycopg2 python-ipy python3-ipy python-docutils postgresql-9.2 postgresql-9.2-ip4r python-tornado python-flask python-flask-xml-rpc python-flask-compress python-parsedatetime python-pylons python-tz python-dateutil python-psutil python-pyparsing; fi
    # if we are testing the upgrade, first install NIPAP packages from official repo
    - if [ "$INSTALL" == "apt" ] && [ "$UPGRADE" == "true" ]; then sudo apt-get install -qq nipapd nipap-www nipap-cli; fi
    # bump version so that we know we are upgrading beyond what is installed
    - if [ "$INSTALL" == "apt" ] && [ "$UPGRADE" == "true" ]; then (echo -e 'Version 9999.9.9\n------------------\n * Test version for Travis-CI automatic upgrade test'; cat NEWS) > NEWS2; mv NEWS2 NEWS; make bumpversion; fi
    # populate answers to nipapd package install questions and reconfigure
    - if [ "$INSTALL" == "apt" ] && [ "$UPGRADE" == "true" ]; then echo 'set nipapd/autoconf true' | sudo debconf-communicate; echo 'set nipapd/database_host localhost' | sudo debconf-communicate; echo 'set nipapd/local_db_autoconf true' | sudo debconf-communicate; echo 'set nipapd/startup true' | sudo debconf-communicate; echo 'set nipapd/local_db_upgrade true' | sudo debconf-communicate; sudo dpkg-reconfigure nipapd; fi
    # create local user for unittest and restart
    - if [ "$INSTALL" == "apt" ] && [ "$UPGRADE" == "true" ]; then sudo nipap/nipap-passwd add -u unittest -p gottatest -f /etc/nipap/local_auth.db -n unittest; sudo /etc/init.d/nipapd restart; fi
    # if upgrade, add some data to the database that we can verify later
    - if [ "$INSTALL" == "apt" ] && [ "$UPGRADE" == "true" ]; then nosetests tests/upgrade-before.py; fi
    # build new NIPAP packages
    - if [ "$INSTALL" == "apt" ]; then make builddeb; fi
    # install the newly built nipap packages
    - if [ "$INSTALL" == "apt" ]; then sudo dpkg --force-confnew -i nipap*.deb python*-pynipap*.deb; fi
    # populate answers to nipapd package install questions and reconfigure
    - if [ "$INSTALL" == "apt" ]; then echo 'set nipapd/database_host localhost' | sudo debconf-communicate; echo 'set nipapd/local_db_autoconf true' | sudo debconf-communicate; echo 'set nipapd/startup true' | sudo debconf-communicate; echo 'set nipapd/local_db_upgrade true' | sudo debconf-communicate; sudo dpkg-reconfigure nipapd; fi
    # create local user for unittests
    - if [ "$INSTALL" == "apt" ]; then sudo nipap/nipap-passwd add -u unittest -p gottatest -f /etc/nipap/local_auth.db -n "User for running unit tests"; fi
    - if [ "$INSTALL" == "apt" ]; then sudo nipap/nipap-passwd add -u readonly -p gottatest -f /etc/nipap/local_auth.db --readonly -n "Read-only user for running unit tests"; fi
    - if [ "$INSTALL" == "apt" ]; then sudo sed -e "s/db_host *= *[^ ]\+/db_host = localhost/" -i /etc/nipap/nipap.conf; fi
    # Don't need to restart, but I would like to make sure that all is updated
    # (like local_auth.db is re-read and so forth). Read #654 for more
    # information.
    #- if [ "$INSTALL" == "apt" ]; then sudo /etc/init.d/nipapd restart; fi

script:
    - cd tests
    # if upgrade, verify data loaded before upgrade looks correct post upgrade
    - if [ "$INSTALL" == "apt" ] && [ "$UPGRADE" == "true" ]; then nosetests upgrade-after.py; fi
{'fraction_non_alphanumeric': 0.1539848539487919, 'fraction_numerical': 0.007356653443923548, 'mean_word_length': 2.8894810659186536, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 12, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4244927', 'n_tokens_mistral': 4825, 'n_tokens_neox': 4514, 'n_words': 1499}