<commit_before>#!/usr/bin/env python2
# -*- coding: utf-8 -*-
#
# build.py - part of the FDroid server tools
# Copyright (C) 2010, <NAME>, <EMAIL>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import sys
import os
import shutil
import subprocess
import re
import zipfile
import tarfile
import md5
import shlex
from xml.dom.minidom import Document
from optparse import OptionParser

import common

#Read configuration...
execfile('config.py')


# Parse command line...
parser = OptionParser()
parser.add_option("-v", "--verbose", action="store_true", default=False,
                  help="Spew out even more information than normal")
parser.add_option("-p", "--package", default=None,
                  help="Build only the specified package")
parser.add_option("-c", "--clean", action="store_true", default=False,
                  help="Clean mode - build everything from scratch")
(options, args) = parser.parse_args()

# Get all apps...
apps = common.read_metadata(options.verbose)

#Clear and/or create the 'built' directory, depending on mode:
built_dir = 'built'
if options.clean:
    if os.path.exists(built_dir):
        shutil.rmtree(built_dir)
if not os.path.exists(built_dir):
    os.mkdir(built_dir)

if not os.path.isdir('build'):
    os.makedirs('build')

for app in apps:

    if app['disabled']:
        print "Skipping %s: disabled" % app['id']
    elif not app['builds']:
        print "Skipping %s: no builds specified" % app['id']

    if (app['disabled'] is None and app['repo'] != '' 
            and app['repotype'] != '' and (options.package is None or
            options.package == app['id']) and len(app['builds']) > 0):

        print "Processing " + app['id']

        build_dir = 'build/' + app['id']

        got_source = False


        for thisbuild in app['builds']:

            dest = os.path.join(built_dir, app['id'] + '_' +
                    thisbuild['vercode'] + '.apk')
            dest_unsigned = dest + "_unsigned"

            if os.path.exists(dest):
                print "..version " + thisbuild['version'] + " already exists"
            elif thisbuild['commit'].startswith('!'):
                print ("..skipping version " + thisbuild['version'] + " - " +
                        thisbuild['commit'][1:])
            else:
                print "..building version " + thisbuild['version']

                if not got_source:

                    got_source = True

                    # Remove the build directory if it already exists...
                    if os.path.exists(build_dir):
                        shutil.rmtree(build_dir)

                    # Strip username/password out of repo address if specified (relevant
                    # only for SVN) and store for use later.
                    repo = app['repo']
                    index = repo.find('@')
                    if index != -1:
                        username = repo[:index]
                        repo = repo[index+1:]
                        index = username.find(':')
                        if index == -1:
                            print "Password required with username"
                            sys.exit(1)
                        password = username[index+1:]
                        username = username[:index]
                        repouserargs = ['--username', username, 
                                '--password', password, '--non-interactive']
                    else:
                        repouserargs = []

                    # Get the source code...
                    if app['repotype'] == 'git':
                        if subprocess.call(['git', 'clone', repo, build_dir]) != 0:
                            print "Git clone failed"
                            sys.exit(1)
                    elif app['repotype'] == 'svn':
                        if not repo.endswith("*"):
                            if subprocess.call(['svn', 'checkout', repo, build_dir] +
                                    repouserargs) != 0:
                                print "Svn checkout failed"
                                sys.exit(1)
                    elif app['repotype'] == 'hg':
                        if subprocess.call(['hg', 'clone', repo, build_dir]) !=0:
                            print "Hg clone failed"
                            sys.exit(1)
                    elif app['repotype'] == 'bzr':
                        if subprocess.call(['bzr', 'branch', repo, build_dir]) !=0:
                            print "Bzr branch failed"
                            sys.exit(1)
                    else:
                        print "Invalid repo type " + app['repotype'] + " in " + app['id']
                        sys.exit(1)

                # Optionally, the actual app source can be in a subdirectory...
                doupdate = True
                if thisbuild.has_key('subdir'):
                    if app['repotype'] == 'svn' and repo.endswith("*"):
                        root_dir = build_dir
                        # Remove the build directory if it already exists...
                        if os.path.exists(build_dir):
                            shutil.rmtree(build_dir)
                        if subprocess.call(['svn', 'checkout',
                                repo[:-1] + thisbuild['subdir'],
                                '-r', thisbuild['commit'],
                                build_dir] + repouserargs) != 0:
                            print "Svn checkout failed"
                            sys.exit(1)
                        # Because we're checking out for every version we build,
                        # we've already checked out the repo at the correct revision
                        # and don't need to update to it...
                        doupdate = False
                    else:
                        root_dir = os.path.join(build_dir, thisbuild['subdir'])
                else:
                    root_dir = build_dir

                if doupdate:
                    if app['repotype'] == 'git':
                        if subprocess.call(['git', 'reset', '--hard', thisbuild['commit']],
                                cwd=build_dir) != 0:
                            print "Git reset failed"
                            sys.exit(1)
                        if subprocess.call(['git', 'clean', '-dfx'],
                                cwd=build_dir) != 0:
                            print "Git clean failed"
                            sys.exit(1)
                    elif app['repotype'] == 'svn':
                        for svncommand in (
                        'svn revert -R .',
                        r"svn status | awk '/\?/ {print $2}' | xargs rm -rf",
                        'svn update --force -r '+thisbuild['commit'],):
                            if subprocess.call(svncommand, cwd=build_dir,
                                    shell=True) != 0:
                                print "Svn update failed"
                                sys.exit(1)
                    elif app['repotype'] == 'hg':
                        if subprocess.call(['hg', 'checkout', thisbuild['commit']],
                                cwd=build_dir) != 0:
                            print "Hg checkout failed"
                            sys.exit(1)
                        if subprocess.call('hg status -u -0 | xargs rm -rf',
                                cwd=build_dir, shell=True) != 0:
                            print "Hg clean failed"
                            sys.exit(1)
                    elif app['repotype'] == 'bzr':
                        if subprocess.call(['bzr', 'revert', '-r', thisbuild['commit']],
                                cwd=build_dir) != 0:
                            print "Bzr revert failed"
                            sys.exit(1)
                        if subprocess.call(['bzr', 'clean-tree', '--unknown', '--ignored'],
                                cwd=build_dir) != 0:
                            print "Bzr revert failed"
                            sys.exit(1)
                    else:
                        print "Invalid repo type " + app['repotype']
                        sys.exit(1)

                # Initialise submodules if requred...
                if thisbuild.get('submodules', 'no')  == 'yes':
                        if subprocess.call(['git', 'submodule', 'init'],
                                cwd=build_dir) != 0:<commit_msg>Cleaning the tree for bzr needs --force<commit_after>#!/usr/bin/env python2
# -*- coding: utf-8 -*-
#
# build.py - part of the FDroid server tools
# Copyright (C) 2010, <NAME>, <EMAIL>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import sys
import os
import shutil
import subprocess
import re
import zipfile
import tarfile
import md5
import shlex
from xml.dom.minidom import Document
from optparse import OptionParser

import common

#Read configuration...
execfile('config.py')


# Parse command line...
parser = OptionParser()
parser.add_option("-v", "--verbose", action="store_true", default=False,
                  help="Spew out even more information than normal")
parser.add_option("-p", "--package", default=None,
                  help="Build only the specified package")
parser.add_option("-c", "--clean", action="store_true", default=False,
                  help="Clean mode - build everything from scratch")
(options, args) = parser.parse_args()

# Get all apps...
apps = common.read_metadata(options.verbose)

#Clear and/or create the 'built' directory, depending on mode:
built_dir = 'built'
if options.clean:
    if os.path.exists(built_dir):
        shutil.rmtree(built_dir)
if not os.path.exists(built_dir):
    os.mkdir(built_dir)

if not os.path.isdir('build'):
    os.makedirs('build')

for app in apps:

    if app['disabled']:
        print "Skipping %s: disabled" % app['id']
    elif not app['builds']:
        print "Skipping %s: no builds specified" % app['id']

    if (app['disabled'] is None and app['repo'] != '' 
            and app['repotype'] != '' and (options.package is None or
            options.package == app['id']) and len(app['builds']) > 0):

        print "Processing " + app['id']

        build_dir = 'build/' + app['id']

        got_source = False


        for thisbuild in app['builds']:

            dest = os.path.join(built_dir, app['id'] + '_' +
                    thisbuild['vercode'] + '.apk')
            dest_unsigned = dest + "_unsigned"

            if os.path.exists(dest):
                print "..version " + thisbuild['version'] + " already exists"
            elif thisbuild['commit'].startswith('!'):
                print ("..skipping version " + thisbuild['version'] + " - " +
                        thisbuild['commit'][1:])
            else:
                print "..building version " + thisbuild['version']

                if not got_source:

                    got_source = True

                    # Remove the build directory if it already exists...
                    if os.path.exists(build_dir):
                        shutil.rmtree(build_dir)

                    # Strip username/password out of repo address if specified (relevant
                    # only for SVN) and store for use later.
                    repo = app['repo']
                    index = repo.find('@')
                    if index != -1:
                        username = repo[:index]
                        repo = repo[index+1:]
                        index = username.find(':')
                        if index == -1:
                            print "Password required with username"
                            sys.exit(1)
                        password = username[index+1:]
                        username = username[:index]
                        repouserargs = ['--username', username, 
                                '--password', password, '--non-interactive']
                    else:
                        repouserargs = []

                    # Get the source code...
                    if app['repotype'] == 'git':
                        if subprocess.call(['git', 'clone', repo, build_dir]) != 0:
                            print "Git clone failed"
                            sys.exit(1)
                    elif app['repotype'] == 'svn':
                        if not repo.endswith("*"):
                            if subprocess.call(['svn', 'checkout', repo, build_dir] +
                                    repouserargs) != 0:
                                print "Svn checkout failed"
                                sys.exit(1)
                    elif app['repotype'] == 'hg':
                        if subprocess.call(['hg', 'clone', repo, build_dir]) !=0:
                            print "Hg clone failed"
                            sys.exit(1)
                    elif app['repotype'] == 'bzr':
                        if subprocess.call(['bzr', 'branch', repo, build_dir]) !=0:
                            print "Bzr branch failed"
                            sys.exit(1)
                    else:
                        print "Invalid repo type " + app['repotype'] + " in " + app['id']
                        sys.exit(1)

                # Optionally, the actual app source can be in a subdirectory...
                doupdate = True
                if thisbuild.has_key('subdir'):
                    if app['repotype'] == 'svn' and repo.endswith("*"):
                        root_dir = build_dir
                        # Remove the build directory if it already exists...
                        if os.path.exists(build_dir):
                            shutil.rmtree(build_dir)
                        if subprocess.call(['svn', 'checkout',
                                repo[:-1] + thisbuild['subdir'],
                                '-r', thisbuild['commit'],
                                build_dir] + repouserargs) != 0:
                            print "Svn checkout failed"
                            sys.exit(1)
                        # Because we're checking out for every version we build,
                        # we've already checked out the repo at the correct revision
                        # and don't need to update to it...
                        doupdate = False
                    else:
                        root_dir = os.path.join(build_dir, thisbuild['subdir'])
                else:
                    root_dir = build_dir

                if doupdate:
                    if app['repotype'] == 'git':
                        if subprocess.call(['git', 'reset', '--hard', thisbuild['commit']],
                                cwd=build_dir) != 0:
                            print "Git reset failed"
                            sys.exit(1)
                        if subprocess.call(['git', 'clean', '-dfx'],
                                cwd=build_dir) != 0:
                            print "Git clean failed"
                            sys.exit(1)
                    elif app['repotype'] == 'svn':
                        for svncommand in (
                        'svn revert -R .',
                        r"svn status | awk '/\?/ {print $2}' | xargs rm -rf",
                        'svn update --force -r '+thisbuild['commit'],):
                            if subprocess.call(svncommand, cwd=build_dir,
                                    shell=True) != 0:
                                print "Svn update failed"
                                sys.exit(1)
                    elif app['repotype'] == 'hg':
                        if subprocess.call(['hg', 'checkout', thisbuild['commit']],
                                cwd=build_dir) != 0:
                            print "Hg checkout failed"
                            sys.exit(1)
                        if subprocess.call('hg status -u -0 | xargs rm -rf',
                                cwd=build_dir, shell=True) != 0:
                            print "Hg clean failed"
                            sys.exit(1)
                    elif app['repotype'] == 'bzr':
                        if subprocess.call(['bzr', 'revert', '-r', thisbuild['commit']],
                                cwd=build_dir) != 0:
                            print "Bzr revert failed"
                            sys.exit(1)
                        if subprocess.call(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'],
                                cwd=build_dir) != 0:
                            print "Bzr revert failed"
                            sys.exit(1)
                    else:
                        print "Invalid repo type " + app['repotype']
                        sys.exit(1)

                # Initialise submodules if requred...
                if thisbuild.get('submodules', 'no')  == 'yes':
                        if subprocess.call(['git', 'submodule', 'init'],
                                cwd=build_dir) != 0:
{'fraction_non_alphanumeric': 0.1123142367824982, 'fraction_numerical': 0.004972100988895641, 'mean_word_length': 1.2492544731610338, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5459596', 'n_tokens_mistral': 4679, 'n_tokens_neox': 4347, 'n_words': 1312}