<commit_before>#!/usr/bin/env python3
#
# build.py - part of the FDroid server tools
# Copyright (C) 2010-2014, <NAME>, <EMAIL>
# Copyright (C) 2013-2014 <NAME> <<EMAIL>>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import sys
import os
import shutil
import glob
import subprocess
import re
import resource
import tarfile
import traceback
import time
import requests
import tempfile
from configparser import ConfigParser
from argparse import ArgumentParser
import logging
from gettext import ngettext

from . import _
from . import common
from . import net
from . import metadata
from . import scanner
from . import vmtools
from .common import FDroidPopen, SdkToolsPopen
from .exception import FDroidException, BuildException, VCSException

try:
    import paramiko
except ImportError:
    pass


# Note that 'force' here also implies test mode.
def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
    """Do a build on the builder vm.

    :param app: app metadata dict
    :param build:
    :param vcs: version control system controller object
    :param build_dir: local source-code checkout of app
    :param output_dir: target folder for the build result
    :param force:
    """

    global buildserverid

    try:
        paramiko
    except NameError:
        raise BuildException("Paramiko is required to use the buildserver")
    if options.verbose:
        logging.getLogger("paramiko").setLevel(logging.INFO)
    else:
        logging.getLogger("paramiko").setLevel(logging.WARN)

    sshinfo = vmtools.get_clean_builder('builder')

    try:
        if not buildserverid:
            buildserverid = subprocess.check_output(['vagrant', 'ssh', '-c',
                                                     'cat /home/vagrant/buildserverid'],
                                                    cwd='builder').rstrip()
            logging.debug(_('Fetched buildserverid from VM: ') + buildserverid)

        # Open SSH connection...
        logging.info("Connecting to virtual machine...")
        sshs = paramiko.SSHClient()
        sshs.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        sshs.connect(sshinfo['hostname'], username=sshinfo['user'],
                     port=sshinfo['port'], timeout=300,
                     look_for_keys=False, key_filename=sshinfo['idfile'])

        homedir = '/home/' + sshinfo['user']

        # Get an SFTP connection...
        ftp = sshs.open_sftp()
        ftp.get_channel().settimeout(60)

        # Put all the necessary files in place...
        ftp.chdir(homedir)

        # Helper to copy the contents of a directory to the server...
        def send_dir(path):
            logging.debug("rsyncing " + path + " to " + ftp.getcwd())
            # TODO this should move to `vagrant rsync` from >= v1.5
            try:
                subprocess.check_output(['rsync', '--recursive', '--perms', '--links', '--quiet', '--rsh=' +
                                         'ssh -o StrictHostKeyChecking=no' +
                                         ' -o UserKnownHostsFile=/dev/null' +<commit_msg>build: fix str vs. bytes error in buildserverid<commit_after>#!/usr/bin/env python3
#
# build.py - part of the FDroid server tools
# Copyright (C) 2010-2014, <NAME>, <EMAIL>
# Copyright (C) 2013-2014 <NAME> <<EMAIL>>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import sys
import os
import shutil
import glob
import subprocess
import re
import resource
import tarfile
import traceback
import time
import requests
import tempfile
from configparser import ConfigParser
from argparse import ArgumentParser
import logging
from gettext import ngettext

from . import _
from . import common
from . import net
from . import metadata
from . import scanner
from . import vmtools
from .common import FDroidPopen, SdkToolsPopen
from .exception import FDroidException, BuildException, VCSException

try:
    import paramiko
except ImportError:
    pass


# Note that 'force' here also implies test mode.
def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
    """Do a build on the builder vm.

    :param app: app metadata dict
    :param build:
    :param vcs: version control system controller object
    :param build_dir: local source-code checkout of app
    :param output_dir: target folder for the build result
    :param force:
    """

    global buildserverid

    try:
        paramiko
    except NameError:
        raise BuildException("Paramiko is required to use the buildserver")
    if options.verbose:
        logging.getLogger("paramiko").setLevel(logging.INFO)
    else:
        logging.getLogger("paramiko").setLevel(logging.WARN)

    sshinfo = vmtools.get_clean_builder('builder')

    try:
        if not buildserverid:
            buildserverid = subprocess.check_output(['vagrant', 'ssh', '-c',
                                                     'cat /home/vagrant/buildserverid'],
                                                    cwd='builder').rstrip()
            logging.debug(_('Fetched buildserverid from VM: {buildserverid}')
                          .format(buildserverid=buildserverid.decode()))

        # Open SSH connection...
        logging.info("Connecting to virtual machine...")
        sshs = paramiko.SSHClient()
        sshs.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        sshs.connect(sshinfo['hostname'], username=sshinfo['user'],
                     port=sshinfo['port'], timeout=300,
                     look_for_keys=False, key_filename=sshinfo['idfile'])

        homedir = '/home/' + sshinfo['user']

        # Get an SFTP connection...
        ftp = sshs.open_sftp()
        ftp.get_channel().settimeout(60)

        # Put all the necessary files in place...
        ftp.chdir(homedir)

        # Helper to copy the contents of a directory to the server...
        def send_dir(path):
            logging.debug("rsyncing " + path + " to " + ftp.getcwd())
            # TODO this should move to `vagrant rsync` from >= v1.5
            try:
                subprocess.check_output(['rsync', '--recursive', '--perms', '--links', '--quiet', '--rsh=' +
                                         'ssh -o StrictHostKeyChecking=no' +
                                         ' -o UserKnownHostsFile=/dev/null' +
{'fraction_non_alphanumeric': 0.09179452968645764, 'fraction_numerical': 0.0066711140760507, 'mean_word_length': 2.579751671442216, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5733130', 'n_tokens_mistral': 2100, 'n_tokens_neox': 1989, 'n_words': 720}