<commit_before># cerbero - a multi-platform build system for Open Source software
# Copyright (C) 2020 <NAME> <<EMAIL>>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Library General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Library General Public License for more details.
#
# You should have received a copy of the GNU Library General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.

import os, sys
import json
import tempfile
import shutil
from hashlib import sha256

from cerbero.commands import Command, register_command
from cerbero.enums import Platform
from cerbero.errors import FatalError
from cerbero.utils import _, N_, ArgparseArgument, git, shell, run_until_complete, to_unixpath
from cerbero.utils import messages as m
from cerbero.config import Distro

class BaseCache(Command):
    base_url = 'https://artifacts.gstreamer-foundation.net/cerbero-deps/%s/%s/%s'
    ssh_address = '<EMAIL>bero-deps-uploader@artifacts.gstreamer-foundation.net'
    # FIXME: fetch this value from CI env vars
    build_dir = '/builds/%s/cerbero/cerbero-build'
    deps_filename = 'cerbero-deps.tar.xz'
    log_filename = 'cerbero-deps.log'
    log_size = 10

    def __init__(self, args=[]):
        args.append(ArgparseArgument('--commit', action='store', type=str,
                    default='HEAD', help=_('the commit to pick artifact from')))
        args.append(ArgparseArgument('--branch', action='store', type=str,
                    default='master', help=_('Git branch to search from')))
        Command.__init__(self, args)

    # FIXME: move this to utils
    def checksum(self, fname):
        h = sha256()
        with open(fname, 'rb') as f:
            # Read in chunks of 512k till f.read() returns b'' instead of reading
            # the whole file at once which will fail on systems with low memory
            for block in iter(lambda: f.read(512 * 1024), b''):
                h.update(block)
        return h.hexdigest()

    def get_git_sha(self, args):
        git_dir = os.path.abspath(os.path.dirname(sys.argv[0]))
        return git.get_hash(git_dir, args.commit)

    def json_get(self, url):
        m.message("GET %s" % url)

        tmpdir = tempfile.mkdtemp()
        tmpfile = os.path.join(tmpdir, 'deps.json')
        run_until_complete(shell.download(url, destination=tmpfile,
              logfile=open(os.devnull, 'w')))

        with open(tmpfile, 'r') as f:
            resp = f.read()
        shutil.rmtree(tmpdir)

        return json.loads(resp)

    def get_distro_and_arch(self, config):
        distro = config.target_distro
        arch = config.target_arch
        if distro == Distro.REDHAT:
            distro = 'fedora'
        if distro == Distro.OS_X:
            distro = 'macos'
        if distro == Distro.WINDOWS:
            # When targeting Windows, we need to differentiate between mingw,
            # msvc, and uwp (debug/release) jobs. When cross-compiling this
            # will always be 'cross-windows-mingw' right now, but that might
            # change at some point.
            toolchain, _ = config._get_toolchain_target_platform_arch()
            distro = 'windows-' + toolchain
        if config.cross_compiling():
            distro = 'cross-' + distro
        return distro, arch

    def make_url(self, config, args, filename):
        branch = args.branch
        distro, arch = self.get_distro_and_arch(config)
        base_url = self.base_url % (branch, distro, arch)
        return "%s/%s" % (base_url, filename)

    def get_deps(self, config, args):
        url = self.make_url(config, args, self.log_filename)
        deps = []

        try:
            deps = self.json_get(url)
        except FatalError as e:
            m.warning("Could not get cache list: %s" % e.msg)

        return deps

    def get_deps_filename(self, config):
        return os.path.join(config.home_dir, self.deps_filename)

    def get_log_filename(self, config):
        return os.path.join(config.home_dir, self.log_filename)

    def run(self, config, args):
        if not config.uninstalled:
            raise FatalError(_("fetch-cache is only available with "
                        "cerbero-uninstalled"))

class FetchCache(BaseCache):
    doc = N_('Fetch a cached build from external storage based on cerbero git '
            'revision.')
    name = 'fetch-cache'

    def __init__(self, args=[]):
        args.append(ArgparseArgument('--namespace', action='store', type=str,
                    default='gstreamer', help=_('GitLab namespace to search from')))
        BaseCache.__init__(self, args)

    def find_dep(self, deps, sha):
        for dep in deps:
            if dep['commit'] == sha:
                return dep

        m.warning("Did not find cache for commit %s" % sha)
        return None

    async def fetch_dep(self, config, dep, namespace):
        try:
            dep_path = os.path.join(config.home_dir, os.path.basename(dep['url']))
            await shell.download(dep['url'], dep_path, check_cert=True, overwrite=True)
            if dep['checksum'] == self.checksum(dep_path):
                await shell.unpack(dep_path, config.home_dir)
            else:
                m.warning("Corrupted dependency file, ignoring.")
            os.remove(dep_path)

            # Don't need to relocate on Windows since binaries in the prefix
            # are meant to be used in a relocatable manner. In case this needs
            # to be re-enabled at some point, note that the current
            # self.build_dir value is an absolute path, and should instead be
            # derived from CI env vars.
            if config.target_platform != Platform.WINDOWS:
                origin = self.build_dir % namespace
                m.message("Relocating from %s to %s" % (origin, config.home_dir))
                # FIXME: Just a quick hack for now
                shell.call(("grep -lnrIU %(origin)s | xargs "
                            "sed \"s#%(origin)s#%(dest)s#g\" -i") % {
                                'origin': origin, 'dest': config.home_dir},
                            config.home_dir)
        except FatalError as e:
            m.warning("Could not retrieve dependencies for commit %s: %s" % (
                        dep['commit'], e.msg))

    def run(self, config, args):
        BaseCache.run(self, config, args)

        sha = self.get_git_sha(args)
        deps = self.get_deps(config, args)
        dep = self.find_dep(deps, sha)
        if dep:
            run_until_complete(self.fetch_dep(config, dep, args.namespace))
        m.message('All done!')

class GenCache(BaseCache):
    doc = N_('Generate build cache from current state.')
    name = 'gen-cache'

    def __init__(self, args=[]):
        BaseCache.__init__(self, args)

    def gen_dep(self, config, args, deps, sha):
        deps_filename = self.get_deps_filename(config)<commit_msg>cache: Use the build platform to decide when to relocate<commit_after># cerbero - a multi-platform build system for Open Source software
# Copyright (C) 2020 <NAME> <<EMAIL>>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Library General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Library General Public License for more details.
#
# You should have received a copy of the GNU Library General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.

import os, sys
import json
import tempfile
import shutil
from hashlib import sha256

from cerbero.commands import Command, register_command
from cerbero.enums import Platform
from cerbero.errors import FatalError
from cerbero.utils import _, N_, ArgparseArgument, git, shell, run_until_complete, to_unixpath
from cerbero.utils import messages as m
from cerbero.config import Distro

class BaseCache(Command):
    base_url = 'https://artifacts.gstreamer-foundation.net/cerbero-deps/%s/%s/%s'
    ssh_address = '<EMAIL>'
    # FIXME: fetch this value from CI env vars
    build_dir = '/builds/%s/cerbero/cerbero-build'
    deps_filename = 'cerbero-deps.tar.xz'
    log_filename = 'cerbero-deps.log'
    log_size = 10

    def __init__(self, args=[]):
        args.append(ArgparseArgument('--commit', action='store', type=str,
                    default='HEAD', help=_('the commit to pick artifact from')))
        args.append(ArgparseArgument('--branch', action='store', type=str,
                    default='master', help=_('Git branch to search from')))
        Command.__init__(self, args)

    # FIXME: move this to utils
    def checksum(self, fname):
        h = sha256()
        with open(fname, 'rb') as f:
            # Read in chunks of 512k till f.read() returns b'' instead of reading
            # the whole file at once which will fail on systems with low memory
            for block in iter(lambda: f.read(512 * 1024), b''):
                h.update(block)
        return h.hexdigest()

    def get_git_sha(self, args):
        git_dir = os.path.abspath(os.path.dirname(sys.argv[0]))
        return git.get_hash(git_dir, args.commit)

    def json_get(self, url):
        m.message("GET %s" % url)

        tmpdir = tempfile.mkdtemp()
        tmpfile = os.path.join(tmpdir, 'deps.json')
        run_until_complete(shell.download(url, destination=tmpfile,
              logfile=open(os.devnull, 'w')))

        with open(tmpfile, 'r') as f:
            resp = f.read()
        shutil.rmtree(tmpdir)

        return json.loads(resp)

    def get_distro_and_arch(self, config):
        distro = config.target_distro
        arch = config.target_arch
        if distro == Distro.REDHAT:
            distro = 'fedora'
        if distro == Distro.OS_X:
            distro = 'macos'
        if distro == Distro.WINDOWS:
            # When targeting Windows, we need to differentiate between mingw,
            # msvc, and uwp (debug/release) jobs. When cross-compiling this
            # will always be 'cross-windows-mingw' right now, but that might
            # change at some point.
            toolchain, _ = config._get_toolchain_target_platform_arch()
            distro = 'windows-' + toolchain
        if config.cross_compiling():
            distro = 'cross-' + distro
        return distro, arch

    def make_url(self, config, args, filename):
        branch = args.branch
        distro, arch = self.get_distro_and_arch(config)
        base_url = self.base_url % (branch, distro, arch)
        return "%s/%s" % (base_url, filename)

    def get_deps(self, config, args):
        url = self.make_url(config, args, self.log_filename)
        deps = []

        try:
            deps = self.json_get(url)
        except FatalError as e:
            m.warning("Could not get cache list: %s" % e.msg)

        return deps

    def get_deps_filename(self, config):
        return os.path.join(config.home_dir, self.deps_filename)

    def get_log_filename(self, config):
        return os.path.join(config.home_dir, self.log_filename)

    def run(self, config, args):
        if not config.uninstalled:
            raise FatalError(_("fetch-cache is only available with "
                        "cerbero-uninstalled"))

class FetchCache(BaseCache):
    doc = N_('Fetch a cached build from external storage based on cerbero git '
            'revision.')
    name = 'fetch-cache'

    def __init__(self, args=[]):
        args.append(ArgparseArgument('--namespace', action='store', type=str,
                    default='gstreamer', help=_('GitLab namespace to search from')))
        BaseCache.__init__(self, args)

    def find_dep(self, deps, sha):
        for dep in deps:
            if dep['commit'] == sha:
                return dep

        m.warning("Did not find cache for commit %s" % sha)
        return None

    async def fetch_dep(self, config, dep, namespace):
        try:
            dep_path = os.path.join(config.home_dir, os.path.basename(dep['url']))
            await shell.download(dep['url'], dep_path, check_cert=True, overwrite=True)
            if dep['checksum'] == self.checksum(dep_path):
                await shell.unpack(dep_path, config.home_dir)
            else:
                m.warning("Corrupted dependency file, ignoring.")
            os.remove(dep_path)

            # Don't need to relocate on Windows and macOS since we build
            # pkg-config with --enable-define-prefix.
            # In case this needs to be re-enabled at some point, note that the
            # current self.build_dir value is hard-coded and is wrong on macOS
            # and Windows. It should instead be derived from CI env vars.
            if config.platform == Platform.LINUX:
                origin = self.build_dir % namespace
                m.message("Relocating from %s to %s" % (origin, config.home_dir))
                # FIXME: Just a quick hack for now
                shell.call(("grep -lnrIU %(origin)s | xargs "
                            "sed \"s#%(origin)s#%(dest)s#g\" -i") % {
                                'origin': origin, 'dest': config.home_dir},
                            config.home_dir)
        except FatalError as e:
            m.warning("Could not retrieve dependencies for commit %s: %s" % (
                        dep['commit'], e.msg))

    def run(self, config, args):
        BaseCache.run(self, config, args)

        sha = self.get_git_sha(args)
        deps = self.get_deps(config, args)
        dep = self.find_dep(deps, sha)
        if dep:
            run_until_complete(self.fetch_dep(config, dep, args.namespace))
        m.message('All done!')

class GenCache(BaseCache):
    doc = N_('Generate build cache from current state.')
    name = 'gen-cache'

    def __init__(self, args=[]):
        BaseCache.__init__(self, args)

    def gen_dep(self, config, args, deps, sha):
        deps_filename = self.get_deps_filename(config)
{'fraction_non_alphanumeric': 0.10320600272851296, 'fraction_numerical': 0.005184174624829468, 'mean_word_length': 2.4039934989551894, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5969882', 'n_tokens_mistral': 4428, 'n_tokens_neox': 4263, 'n_words': 1327}