<commit_before># Copyright 2013 The Servo Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.

from __future__ import absolute_import, print_function, unicode_literals

import base64
import json
import os
import os.path as path
import re
import shutil
import subprocess
import sys
import urllib2
import glob

from mach.decorators import (
    CommandArgument,
    CommandProvider,
    Command,
)

import servo.bootstrap as bootstrap
from servo.command_base import CommandBase, BIN_SUFFIX, cd
from servo.util import delete, download_bytes, download_file, extract, host_triple


@CommandProvider
class MachCommands(CommandBase):
    @Command('env',
             description='Print environment setup commands',
             category='bootstrap')
    def env(self):
        env = self.build_env()
        print("export PATH=%s" % env["PATH"])
        if sys.platform == "darwin":
            print("export DYLD_LIBRARY_PATH=%s" % env["DYLD_LIBRARY_PATH"])
        else:
            print("export LD_LIBRARY_PATH=%s" % env["LD_LIBRARY_PATH"])

    @Command('bootstrap',
             description='Install required packages for building.',
             category='bootstrap')
    @CommandArgument('--force', '-f',
                     action='store_true',
                     help='Boostrap without confirmation')
    def bootstrap(self, force=False):
        return bootstrap.bootstrap(self.context, force=force)

    @Command('bootstrap-rust',
             description='Download the Rust compiler',
             category='bootstrap')
    @CommandArgument('--force', '-f',
                     action='store_true',
                     help='Force download even if a copy already exists')
    @CommandArgument('--target',
                     action='append',
                     default=[],
                     help='Download rust stdlib for specified target')
    @CommandArgument('--stable',
                     action='store_true',
                     help='Use stable rustc version')
    def bootstrap_rustc(self, force=False, target=[], stable=False):
        self.set_use_stable_rust(stable)
        version = self.rust_version()
        rust_path = self.rust_path()
        rust_dir = path.join(self.context.sharedir, "rust", rust_path)
        install_dir = path.join(self.context.sharedir, "rust", version)
        if not self.config["build"]["llvm-assertions"]:
            if not self.use_stable_rust():
                install_dir += "-alt"

        if not force and path.exists(path.join(rust_dir, "rustc", "bin", "rustc" + BIN_SUFFIX)):
            print("Rust compiler already downloaded.", end=" ")
            print("Use |bootstrap-rust --force| to download again.")
        else:
            if path.isdir(rust_dir):
                shutil.rmtree(rust_dir)
            os.makedirs(rust_dir)

            # The nightly Rust compiler is hosted on the nightly server under the date with a name
            # rustc-nightly-HOST-TRIPLE.tar.gz, whereas the stable compiler is named
            # rustc-VERSION-HOST-TRIPLE.tar.gz. We just need to pull down and extract it,
            # giving a directory name that will be the same as the tarball name (rustc is
            # in that directory).
            if stable:
                tarball = "rustc-%s-%s.tar.gz" % (version, host_triple())
                rustc_url = "https://static-rust-lang-org.s3.amazonaws.com/dist/" + tarball
            else:
                tarball = "%s/rustc-nightly-%s.tar.gz" % (version, host_triple())
                base_url = "https://s3.amazonaws.com/rust-lang-ci/rustc-builds"
                if not self.config["build"]["llvm-assertions"]:
                    base_url += "-alt"
                rustc_url = base_url + "/" + tarball
            tgz_file = rust_dir + '-rustc.tar.gz'

            download_file("Rust compiler", rustc_url, tgz_file)

            print("Extracting Rust compiler...")
            extract(tgz_file, install_dir)
            print("Rust compiler ready.")

        # Each Rust stdlib has a name of the form `rust-std-nightly-TRIPLE.tar.gz` for the nightly
        # releases, or rust-std-VERSION-TRIPLE.tar.gz for stable releases, with
        # a directory of the name `rust-std-TRIPLE` inside and then a `lib` directory.
        # This `lib` directory needs to be extracted and merged with the `rustc/lib`
        # directory from the host compiler above.
        nightly_suffix = "" if stable else "-nightly"
        stable_version = "-{}".format(version) if stable else ""
        lib_dir = path.join(install_dir,
                            "rustc{}{}-{}".format(nightly_suffix, stable_version, host_triple()),
                            "rustc", "lib", "rustlib")

        # ensure that the libs for the host's target is downloaded
        host_target = host_triple()
        if host_target not in target:
            target.append(host_target)

        for target_triple in target:
            target_lib_dir = path.join(lib_dir, target_triple)
            if path.exists(target_lib_dir):
                # No need to check for force. If --force the directory is already deleted
                print("Rust lib for target {} already downloaded.".format(target_triple), end=" ")
                print("Use |bootstrap-rust --force| to download again.")
                continue

            if self.use_stable_rust():
                std_url = ("https://static-rust-lang-org.s3.amazonaws.com/dist/rust-std-%s-%s.tar.gz"
                           % (version, target_triple))
                tgz_file = install_dir + ('rust-std-%s-%s.tar.gz' % (version, target_triple))
            else:
                std_url = ("https://s3.amazonaws.com/rust-lang-ci/rustc-builds/%s/rust-std-nightly-%s.tar.gz"
                           % (version, target_triple))
                tgz_file = install_dir + ('rust-std-nightly-%s.tar.gz' % target_triple)

            download_file("Host rust library for target %s" % target_triple, std_url, tgz_file)
            print("Extracting Rust stdlib for target %s..." % target_triple)
            extract(tgz_file, install_dir)
            shutil.copytree(path.join(install_dir,
                                      "rust-std%s%s-%s" % (nightly_suffix, stable_version, target_triple),
                                      "rust-std-%s" % target_triple, "lib", "rustlib", target_triple),
                            path.join(install_dir,
                                      "rustc%s%s-%s" % (nightly_suffix, stable_version, host_triple()),
                                      "rustc", "lib", "rustlib", target_triple))
            shutil.rmtree(path.join(install_dir,
                          "rust-std%s%s-%s" % (nightly_suffix, stable_version, target_triple)))

            print("Rust {} libs ready.".format(target_triple))

    @Command('bootstrap-rust-docs',<commit_msg>Fetch "alt" rust std build when we do so for rustc<commit_after># Copyright 2013 The Servo Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.

from __future__ import absolute_import, print_function, unicode_literals

import base64
import json
import os
import os.path as path
import re
import shutil
import subprocess
import sys
import urllib2
import glob

from mach.decorators import (
    CommandArgument,
    CommandProvider,
    Command,
)

import servo.bootstrap as bootstrap
from servo.command_base import CommandBase, BIN_SUFFIX, cd
from servo.util import delete, download_bytes, download_file, extract, host_triple


@CommandProvider
class MachCommands(CommandBase):
    @Command('env',
             description='Print environment setup commands',
             category='bootstrap')
    def env(self):
        env = self.build_env()
        print("export PATH=%s" % env["PATH"])
        if sys.platform == "darwin":
            print("export DYLD_LIBRARY_PATH=%s" % env["DYLD_LIBRARY_PATH"])
        else:
            print("export LD_LIBRARY_PATH=%s" % env["LD_LIBRARY_PATH"])

    @Command('bootstrap',
             description='Install required packages for building.',
             category='bootstrap')
    @CommandArgument('--force', '-f',
                     action='store_true',
                     help='Boostrap without confirmation')
    def bootstrap(self, force=False):
        return bootstrap.bootstrap(self.context, force=force)

    @Command('bootstrap-rust',
             description='Download the Rust compiler',
             category='bootstrap')
    @CommandArgument('--force', '-f',
                     action='store_true',
                     help='Force download even if a copy already exists')
    @CommandArgument('--target',
                     action='append',
                     default=[],
                     help='Download rust stdlib for specified target')
    @CommandArgument('--stable',
                     action='store_true',
                     help='Use stable rustc version')
    def bootstrap_rustc(self, force=False, target=[], stable=False):
        self.set_use_stable_rust(stable)
        version = self.rust_version()
        rust_path = self.rust_path()
        rust_dir = path.join(self.context.sharedir, "rust", rust_path)
        install_dir = path.join(self.context.sharedir, "rust", version)
        if not self.config["build"]["llvm-assertions"]:
            if not self.use_stable_rust():
                install_dir += "-alt"

        if not force and path.exists(path.join(rust_dir, "rustc", "bin", "rustc" + BIN_SUFFIX)):
            print("Rust compiler already downloaded.", end=" ")
            print("Use |bootstrap-rust --force| to download again.")
        else:
            if path.isdir(rust_dir):
                shutil.rmtree(rust_dir)
            os.makedirs(rust_dir)

            # The nightly Rust compiler is hosted on the nightly server under the date with a name
            # rustc-nightly-HOST-TRIPLE.tar.gz, whereas the stable compiler is named
            # rustc-VERSION-HOST-TRIPLE.tar.gz. We just need to pull down and extract it,
            # giving a directory name that will be the same as the tarball name (rustc is
            # in that directory).
            if stable:
                tarball = "rustc-%s-%s.tar.gz" % (version, host_triple())
                rustc_url = "https://static-rust-lang-org.s3.amazonaws.com/dist/" + tarball
            else:
                tarball = "%s/rustc-nightly-%s.tar.gz" % (version, host_triple())
                base_url = "https://s3.amazonaws.com/rust-lang-ci/rustc-builds"
                if not self.config["build"]["llvm-assertions"]:
                    base_url += "-alt"
                rustc_url = base_url + "/" + tarball
            tgz_file = rust_dir + '-rustc.tar.gz'

            download_file("Rust compiler", rustc_url, tgz_file)

            print("Extracting Rust compiler...")
            extract(tgz_file, install_dir)
            print("Rust compiler ready.")

        # Each Rust stdlib has a name of the form `rust-std-nightly-TRIPLE.tar.gz` for the nightly
        # releases, or rust-std-VERSION-TRIPLE.tar.gz for stable releases, with
        # a directory of the name `rust-std-TRIPLE` inside and then a `lib` directory.
        # This `lib` directory needs to be extracted and merged with the `rustc/lib`
        # directory from the host compiler above.
        nightly_suffix = "" if stable else "-nightly"
        stable_version = "-{}".format(version) if stable else ""
        lib_dir = path.join(install_dir,
                            "rustc{}{}-{}".format(nightly_suffix, stable_version, host_triple()),
                            "rustc", "lib", "rustlib")

        # ensure that the libs for the host's target is downloaded
        host_target = host_triple()
        if host_target not in target:
            target.append(host_target)

        for target_triple in target:
            target_lib_dir = path.join(lib_dir, target_triple)
            if path.exists(target_lib_dir):
                # No need to check for force. If --force the directory is already deleted
                print("Rust lib for target {} already downloaded.".format(target_triple), end=" ")
                print("Use |bootstrap-rust --force| to download again.")
                continue

            if self.use_stable_rust():
                std_url = ("https://static-rust-lang-org.s3.amazonaws.com/dist/rust-std-%s-%s.tar.gz"
                           % (version, target_triple))
                tgz_file = install_dir + ('rust-std-%s-%s.tar.gz' % (version, target_triple))
            else:
                tarball = "%s/rust-std-nightly-%s.tar.gz" % (version, target_triple)
                base_url = "https://s3.amazonaws.com/rust-lang-ci/rustc-builds"
                if not self.config["build"]["llvm-assertions"]:
                    base_url += "-alt"
                std_url = base_url + "/" + tarball
                tgz_file = install_dir + ('rust-std-nightly-%s.tar.gz' % target_triple)

            download_file("Host rust library for target %s" % target_triple, std_url, tgz_file)
            print("Extracting Rust stdlib for target %s..." % target_triple)
            extract(tgz_file, install_dir)
            shutil.copytree(path.join(install_dir,
                                      "rust-std%s%s-%s" % (nightly_suffix, stable_version, target_triple),
                                      "rust-std-%s" % target_triple, "lib", "rustlib", target_triple),
                            path.join(install_dir,
                                      "rustc%s%s-%s" % (nightly_suffix, stable_version, host_triple()),
                                      "rustc", "lib", "rustlib", target_triple))
            shutil.rmtree(path.join(install_dir,
                          "rust-std%s%s-%s" % (nightly_suffix, stable_version, target_triple)))

            print("Rust {} libs ready.".format(target_triple))

    @Command('bootstrap-rust-docs',
{'fraction_non_alphanumeric': 0.11281346616283064, 'fraction_numerical': 0.0020611473720371005, 'mean_word_length': 2.1350419987077323, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 8, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 34, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '954161', 'n_tokens_mistral': 4157, 'n_tokens_neox': 4008, 'n_words': 1039}