<commit_before>#!/bin/bash

CONSOLE_LOG=/home/jenkins-slave/logs/console-log.$ZUUL_UUID.manila.log

echo "Collecting logs"
ssh -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" -i $DEVSTACK_SSH_KEY ubuntu@$DEVSTACK_FLOATING_IP "/home/ubuntu/bin/collect_logs.sh"

if [ "$IS_DEBUG_JOB" != "yes" ]
  then
    echo "Creating logs destination folder"
    ssh -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" -i $LOGS_SSH_KEY <EMAIL> "if [ -z '$ZUUL_CHANGE' ] || [ -z '$ZUUL_PATCHSET' ]; then echo 'Missing parameters!'; exit 1; elif [ ! -d /srv/logs/manila/$ZUUL_CHANGE/$ZUUL_PATCHSET ]; then mkdir -p /srv/logs/manila/$ZUUL_CHANGE/$ZUUL_PATCHSET; else rm -rf /srv/logs/manila/$ZUUL_CHANGE/$ZUUL_PATCHSET/*; fi"

    echo "Downloading logs"
    scp -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" -i $DEVSTACK_SSH_KEY ubuntu@$DEVSTACK_FLOATING_IP:/home/ubuntu/aggregate.tar.gz "aggregate-$NAME.tar.gz"

    echo "Uploading logs"
    scp -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" -i $LOGS_SSH_KEY "aggregate-$NAME.tar.gz" <EMAIL>:/srv/logs/manila/$ZUUL_CHANGE/$ZUUL_PATCHSET/aggregate-logs.tar.gz

    echo "Before gzip:"
    ls -lia `dirname $CONSOLE_LOG`

    echo "GZIP:"
    gzip -v9 $CONSOLE_LOG

    echo "After gzip:"
    ls -lia `dirname $CONSOLE_LOG`

    scp -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" -i $LOGS_SSH_KEY $CONSOLE_LOG.gz <EMAIL>.tld:/srv/logs/manila/$ZUUL_CHANGE/$ZUUL_PATCHSET/ && rm -f $CONSOLE_LOG*

    echo "Extracting logs"
    ssh -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" -i $LOGS_SSH_KEY <EMAIL> "tar -xzf /srv/logs/manila/$ZUUL_CHANGE/$ZUUL_PATCHSET/aggregate-logs.tar.gz -C /srv/logs/manila/$ZUUL_CHANGE/$ZUUL_PATCHSET/"
    echo "Fixing permissions on all log files"
    ssh -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" -i $LOGS_SSH_KEY <EMAIL>@logs.<EMAIL>.tld "chmod a+rx -R /srv/logs/manila/$ZUUL_CHANGE/$ZUUL_PATCHSET/"
  else
    TIMESTAMP=$(date +%d-%m-%Y_%H-%M)
    echo "Creating logs destination folder"
    ssh -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" -i $LOGS_SSH_KEY <EMAIL>@logs.openstack.tld "if [ -z '$ZUUL_CHANGE' ] || [ -z '$ZUUL_PATCHSET' ]; then echo 'Missing parameters!'; exit 1; elif [ ! -d /srv/logs/debug/manila/$ZUUL_CHANGE/$ZUUL_PATCHSET/$TIMESTAMP ]; then mkdir -p /srv/logs/debug/manila/$ZUUL_CHANGE/$ZUUL_PATCHSET/$TIMESTAMP; else rm -rf /srv/logs/debug/manila/$ZUUL_CHANGE/$ZUUL_PATCHSET/$TIMESTAMP/*; fi"

		echo "Downloading logs"
    scp -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" -i $DEVSTACK_SSH_KEY ubuntu@$FLOATING_IP:/home/ubuntu/aggregate.tar.gz "aggregate-$NAME.tar.gz"

		echo "Uploading logs"
    scp -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" -i $LOGS_SSH_KEY "aggregate-$NAME.tar.gz" <EMAIL>.tld:/srv/logs/debug/manila/$ZUUL_CHANGE/$ZUUL_PATCHSET/$TIMESTAMP/aggregate-logs.tar.gz
    gzip -9 /home/jenkins-slave/console-$NAME.log
    scp -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" -i $LOGS_SSH_KEY "/home/jenkins-slave/console-$NAME.log.gz" <EMAIL>:/srv/logs/debug/manila/$ZUUL_CHANGE/$ZUUL_PATCHSET/$TIMESTAMP/console.log.gz && rm -f /home/jenkins-slave/console-$NAME.log.gz

		echo "Extracting logs"
    ssh -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" -i $LOGS_SSH_KEY <EMAIL>@logs.<EMAIL>ld "tar -xzf /srv/logs/debug/manila/$ZUUL_CHANGE/$ZUUL_PATCHSET/$TIMESTAMP/aggregate-logs.tar.gz -C /srv/logs/debug/$ZUUL_CHANGE/$ZUUL_PATCHSET/$TIMESTAMP/"

		echo "Fixing permissions on all log files"
    ssh -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" -i $LOGS_SSH_KEY <EMAIL>@logs.<EMAIL>ld "chmod a+rx -R /srv/logs/debug/manila/$ZUUL_CHANGE/$ZUUL_PATCHSET/$TIMESTAMP"    
fi
<commit_msg>Fixed wrong CONSOLE_LOG for normal jobs<commit_after>#!/bin/bash

echo "Collecting logs"
ssh -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" -i $DEVSTACK_SSH_KEY ubuntu@$DEVSTACK_FLOATING_IP "/home/ubuntu/bin/collect_logs.sh"

if [ "$IS_DEBUG_JOB" != "yes" ]
  then
    echo "Creating logs destination folder"
    ssh -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" -i $LOGS_SSH_KEY <EMAIL>@logs.openstack.tld "if [ -z '$ZUUL_CHANGE' ] || [ -z '$ZUUL_PATCHSET' ]; then echo 'Missing parameters!'; exit 1; elif [ ! -d /srv/logs/manila/$ZUUL_CHANGE/$ZUUL_PATCHSET ]; then mkdir -p /srv/logs/manila/$ZUUL_CHANGE/$ZUUL_PATCHSET; else rm -rf /srv/logs/manila/$ZUUL_CHANGE/$ZUUL_PATCHSET/*; fi"

    echo "Downloading logs"
    scp -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" -i $DEVSTACK_SSH_KEY ubuntu@$DEVSTACK_FLOATING_IP:/home/ubuntu/aggregate.tar.gz "aggregate-$NAME.tar.gz"

    echo "Uploading logs"
    scp -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" -i $LOGS_SSH_KEY "aggregate-$NAME.tar.gz" <EMAIL>:/srv/logs/manila/$ZUUL_CHANGE/$ZUUL_PATCHSET/aggregate-logs.tar.gz

    echo "Before gzip:"
    ls -lia `dirname $CONSOLE_LOG`

    echo "GZIP:"
    gzip -v9 $CONSOLE_LOG

    echo "After gzip:"
    ls -lia `dirname $CONSOLE_LOG`

    scp -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" -i $LOGS_SSH_KEY $CONSOLE_LOG.gz <EMAIL>:/srv/logs/manila/$ZUUL_CHANGE/$ZUUL_PATCHSET/console.log.gz && rm -f $CONSOLE_LOG*

    echo "Extracting logs"
    ssh -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" -i $LOGS_SSH_KEY <EMAIL> "tar -xzf /srv/logs/manila/$ZUUL_CHANGE/$ZUUL_PATCHSET/aggregate-logs.tar.gz -C /srv/logs/manila/$ZUUL_CHANGE/$ZUUL_PATCHSET/"

    echo "Fixing permissions on all log files"
    ssh -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" -i $LOGS_SSH_KEY <EMAIL> "chmod a+rx -R /srv/logs/manila/$ZUUL_CHANGE/$ZUUL_PATCHSET/"
  else
    TIMESTAMP=$(date +%d-%m-%Y_%H-%M)
    echo "Creating logs destination folder"
    ssh -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" -i $LOGS_SSH_KEY <EMAIL>ld "if [ -z '$ZUUL_CHANGE' ] || [ -z '$ZUUL_PATCHSET' ]; then echo 'Missing parameters!'; exit 1; elif [ ! -d /srv/logs/debug/manila/$ZUUL_CHANGE/$ZUUL_PATCHSET/$TIMESTAMP ]; then mkdir -p /srv/logs/debug/manila/$ZUUL_CHANGE/$ZUUL_PATCHSET/$TIMESTAMP; else rm -rf /srv/logs/debug/manila/$ZUUL_CHANGE/$ZUUL_PATCHSET/$TIMESTAMP/*; fi"
    
    echo "Downloading logs"
    scp -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" -i $DEVSTACK_SSH_KEY ubuntu@$FLOATING_IP:/home/ubuntu/aggregate.tar.gz "aggregate-$NAME.tar.gz"

    echo "Before gzip:"
    ls -lia `dirname $CONSOLE_LOG`

    echo "GZIP:"
    gzip -v9 $CONSOLE_LOG

    echo "After gzip:"
    ls -lia `dirname $CONSOLE_LOG`
    
    echo "Uploading logs"
    scp -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" -i $LOGS_SSH_KEY "aggregate-$NAME.tar.gz" <EMAIL>:/srv/logs/debug/manila/$ZUUL_CHANGE/$ZUUL_PATCHSET/$TIMESTAMP/aggregate-logs.tar.gz
    scp -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" -i $LOGS_SSH_KEY $CONSOLE_LOG <EMAIL>:/srv/logs/debug/manila/$ZUUL_CHANGE/$ZUUL_PATCHSET/$TIMESTAMP/console.log.gz && rm -f $CONSOLE_LOG.gz
    
    echo "Extracting logs"
    ssh -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" -i $LOGS_SSH_KEY <EMAIL> "tar -xzf /srv/logs/debug/manila/$ZUUL_CHANGE/$ZUUL_PATCHSET/$TIMESTAMP/aggregate-logs.tar.gz -C /srv/logs/debug/$ZUUL_CHANGE/$ZUUL_PATCHSET/$TIMESTAMP/"
    
    echo "Fixing permissions on all log files"
    ssh -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" -i $LOGS_SSH_KEY <EMAIL> "chmod a+rx -R /srv/logs/debug/manila/$ZUUL_CHANGE/$ZUUL_PATCHSET/$TIMESTAMP"    
fi

{'fraction_non_alphanumeric': 0.1479926528470218, 'fraction_numerical': 0.0010495932826029913, 'mean_word_length': 4.185714285714286, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6396726', 'n_tokens_mistral': 3294, 'n_tokens_neox': 3196, 'n_words': 588}