<commit_before># Copyright 2013 The Servo Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.

from __future__ import print_function, unicode_literals

import base64
import json
import os
import os.path as path
import re
import shutil
import subprocess
import sys
import StringIO
import tarfile
import urllib2
from distutils.version import LooseVersion

from mach.decorators import (
    CommandArgument,
    CommandProvider,
    Command,
)

from servo.command_base import CommandBase, cd, host_triple


def download(desc, src, writer):
    print("Downloading %s..." % desc)
    dumb = (os.environ.get("TERM") == "dumb") or (not sys.stdout.isatty())

    try:
        resp = urllib2.urlopen(src)

        fsize = None
        if resp.info().getheader('Content-Length'):
            fsize = int(resp.info().getheader('Content-Length').strip())

        recved = 0
        chunk_size = 8192

        while True:
            chunk = resp.read(chunk_size)
            if not chunk:
                break
            recved += len(chunk)
            if not dumb:
                if fsize is not None:
                    pct = recved * 100.0 / fsize
                    print("\rDownloading %s: %5.1f%%" % (desc, pct), end="")

                sys.stdout.flush()
            writer.write(chunk)

        if not dumb:
            print()
    except urllib2.HTTPError, e:
        print("Download failed (%d): %s - %s" % (e.code, e.reason, src))

        cpu_type = subprocess.check_output(["uname", "-m"]).strip().lower()
        if e.code == 404 and cpu_type in ["i386", "i486", "i686", "i768", "x86"]:
            # i686
            print("Note: Servo does not currently bootstrap 32bit snapshots of Rust")
            print("See https://github.com/servo/servo/issues/3899")

        sys.exit(1)


def download_file(desc, src, dst):
    with open(dst, 'wb') as fd:
        download(desc, src, fd)


def download_bytes(desc, src):
    content_writer = StringIO.StringIO()
    download(desc, src, content_writer)
    return content_writer.getvalue()


def extract(src, dst, movedir=None):
    tarfile.open(src).extractall(dst)

    if movedir:
        for f in os.listdir(movedir):
            frm = path.join(movedir, f)
            to = path.join(dst, f)
            os.rename(frm, to)
        os.rmdir(movedir)

    os.remove(src)


@CommandProvider
class MachCommands(CommandBase):
    @Command('env',
             description='Print environment setup commands',
             category='bootstrap')
    def env(self):
        env = self.build_env()
        print("export PATH=%s" % env["PATH"])
        if sys.platform == "darwin":
            print("export DYLD_LIBRARY_PATH=%s" % env["DYLD_LIBRARY_PATH"])
        else:
            print("export LD_LIBRARY_PATH=%s" % env["LD_LIBRARY_PATH"])

    @Command('bootstrap-rust',
             description='Download the Rust compiler snapshot',
             category='bootstrap')
    @CommandArgument('--force', '-f',
                     action='store_true',
                     help='Force download even if a snapshot already exists')
    def bootstrap_rustc(self, force=False):
        rust_dir = path.join(
            self.context.sharedir, "rust", self.rust_snapshot_path())
        if not force and path.exists(path.join(rust_dir, "rustc", "bin", "rustc")):
            print("Snapshot Rust compiler already downloaded.", end=" ")
            print("Use |bootstrap-rust --force| to download again.")
            return

        if path.isdir(rust_dir):
            shutil.rmtree(rust_dir)
        os.makedirs(rust_dir)

        date = self.rust_snapshot_path().split("/")[0]
        install_dir = path.join(self.context.sharedir, "rust", date)

        # The Rust compiler is hosted on the nightly server under the date with a name
        # rustc-nightly-HOST-TRIPLE.tar.gz. We just need to pull down and extract it,
        # giving a directory name that will be the same as the tarball name (rustc is
        # in that directory).
        snapshot_url = ("https://static-rust-lang-org.s3.amazonaws.com/dist/%s.tar.gz"
                        % self.rust_snapshot_path())
        tgz_file = rust_dir + '-rustc.tar.gz'

        download_file("Rust compiler", snapshot_url, tgz_file)

        print("Extracting Rust compiler...")
        extract(tgz_file, install_dir)

        # Each Rust stdlib has a name of the form `rust-std-nightly-TRIPLE.tar.gz`, with
        # a directory of the name `rust-std-TRIPLE` inside and then a `lib` directory.
        # This `lib` directory needs to be extracted and merged with the `rustc/lib`
        # directory from the host compiler above.
        # TODO: make it possible to request an additional cross-target to add to this
        # list.
        stdlibs = [host_triple(), "arm-linux-androideabi"]
        for target in stdlibs:
            snapshot_url = ("https://static-rust-lang-org.s3.amazonaws.com/dist/%s/rust-std-nightly-%s.tar.gz"
                            % (date, target))
            tgz_file = install_dir + ('rust-std-nightly-%s.tar.gz' % target)

            download_file("Host rust library for target %s" % target, snapshot_url, tgz_file)
            print("Extracting Rust stdlib for target %s..." % target)
            extract(tgz_file, install_dir)
            shutil.copytree(path.join(install_dir, "rust-std-nightly-%s" % target,
                                      "rust-std-%s" % target, "lib", "rustlib", target),
                            path.join(install_dir, "rustc-nightly-%s" % host_triple(),
                                      "rustc", "lib", "rustlib", target))
            shutil.rmtree(path.join(install_dir, "rust-std-nightly-%s" % target))

        print("Snapshot Rust ready.")

    @Command('bootstrap-rust-docs',
             description='Download the Rust documentation',
             category='bootstrap')
    @CommandArgument('--force', '-f',
                     action='store_true',
                     help='Force download even if docs already exist')
    def bootstrap_rustc_docs(self, force=False):
        self.ensure_bootstrapped()
        hash_dir = path.join(self.context.sharedir, "rust",
                             self.rust_snapshot_path().split("/")[0])
        docs_dir = path.join(hash_dir, "doc")
        if not force and path.exists(docs_dir):
            print("Snapshot Rust docs already downloaded.", end=" ")
            print("Use |bootstrap-rust-docs --force| to download again.")
            return

        if path.isdir(docs_dir):
            shutil.rmtree(docs_dir)
        docs_name = self.rust_snapshot_path().replace("rustc-", "rust-docs-")
        snapshot_url = ("https://static-rust-lang-org.s3.amazonaws.com/dist/rust-docs-nightly-%s.tar.gz"
                        % host_triple())
        tgz_file = path.join(hash_dir, 'doc.tar.gz')

        download_file("Rust docs", snapshot_url, tgz_file)

        print("Extracting Rust docs...")
        temp_dir = path.join(hash_dir, "temp_docs")
        if path.isdir(temp_dir):
            shutil.rmtree(temp_dir)
        extract(tgz_file, temp_dir)
        shutil.move(path.join(temp_dir, docs_name.split("/")[1],
                              "rust-docs", "share", "doc", "rust", "html"),
                    docs_dir)
        shutil.rmtree(temp_dir)
        print("Rust docs ready.")

    @Command('bootstrap-cargo',
             description='Download the Cargo build tool',
             category='bootstrap')
    @CommandArgument('--force', '-f',
                     action='store_true',<commit_msg>Auto merge of #8697 - Ms2ger:docs, r=SimonSapin<commit_after># Copyright 2013 The Servo Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.

from __future__ import print_function, unicode_literals

import base64
import json
import os
import os.path as path
import re
import shutil
import subprocess
import sys
import StringIO
import tarfile
import urllib2
from distutils.version import LooseVersion

from mach.decorators import (
    CommandArgument,
    CommandProvider,
    Command,
)

from servo.command_base import CommandBase, cd, host_triple


def download(desc, src, writer):
    print("Downloading %s..." % desc)
    dumb = (os.environ.get("TERM") == "dumb") or (not sys.stdout.isatty())

    try:
        resp = urllib2.urlopen(src)

        fsize = None
        if resp.info().getheader('Content-Length'):
            fsize = int(resp.info().getheader('Content-Length').strip())

        recved = 0
        chunk_size = 8192

        while True:
            chunk = resp.read(chunk_size)
            if not chunk:
                break
            recved += len(chunk)
            if not dumb:
                if fsize is not None:
                    pct = recved * 100.0 / fsize
                    print("\rDownloading %s: %5.1f%%" % (desc, pct), end="")

                sys.stdout.flush()
            writer.write(chunk)

        if not dumb:
            print()
    except urllib2.HTTPError, e:
        print("Download failed (%d): %s - %s" % (e.code, e.reason, src))

        cpu_type = subprocess.check_output(["uname", "-m"]).strip().lower()
        if e.code == 404 and cpu_type in ["i386", "i486", "i686", "i768", "x86"]:
            # i686
            print("Note: Servo does not currently bootstrap 32bit snapshots of Rust")
            print("See https://github.com/servo/servo/issues/3899")

        sys.exit(1)


def download_file(desc, src, dst):
    with open(dst, 'wb') as fd:
        download(desc, src, fd)


def download_bytes(desc, src):
    content_writer = StringIO.StringIO()
    download(desc, src, content_writer)
    return content_writer.getvalue()


def extract(src, dst, movedir=None):
    tarfile.open(src).extractall(dst)

    if movedir:
        for f in os.listdir(movedir):
            frm = path.join(movedir, f)
            to = path.join(dst, f)
            os.rename(frm, to)
        os.rmdir(movedir)

    os.remove(src)


@CommandProvider
class MachCommands(CommandBase):
    @Command('env',
             description='Print environment setup commands',
             category='bootstrap')
    def env(self):
        env = self.build_env()
        print("export PATH=%s" % env["PATH"])
        if sys.platform == "darwin":
            print("export DYLD_LIBRARY_PATH=%s" % env["DYLD_LIBRARY_PATH"])
        else:
            print("export LD_LIBRARY_PATH=%s" % env["LD_LIBRARY_PATH"])

    @Command('bootstrap-rust',
             description='Download the Rust compiler snapshot',
             category='bootstrap')
    @CommandArgument('--force', '-f',
                     action='store_true',
                     help='Force download even if a snapshot already exists')
    def bootstrap_rustc(self, force=False):
        rust_dir = path.join(
            self.context.sharedir, "rust", self.rust_snapshot_path())
        if not force and path.exists(path.join(rust_dir, "rustc", "bin", "rustc")):
            print("Snapshot Rust compiler already downloaded.", end=" ")
            print("Use |bootstrap-rust --force| to download again.")
            return

        if path.isdir(rust_dir):
            shutil.rmtree(rust_dir)
        os.makedirs(rust_dir)

        date = self.rust_snapshot_path().split("/")[0]
        install_dir = path.join(self.context.sharedir, "rust", date)

        # The Rust compiler is hosted on the nightly server under the date with a name
        # rustc-nightly-HOST-TRIPLE.tar.gz. We just need to pull down and extract it,
        # giving a directory name that will be the same as the tarball name (rustc is
        # in that directory).
        snapshot_url = ("https://static-rust-lang-org.s3.amazonaws.com/dist/%s.tar.gz"
                        % self.rust_snapshot_path())
        tgz_file = rust_dir + '-rustc.tar.gz'

        download_file("Rust compiler", snapshot_url, tgz_file)

        print("Extracting Rust compiler...")
        extract(tgz_file, install_dir)

        # Each Rust stdlib has a name of the form `rust-std-nightly-TRIPLE.tar.gz`, with
        # a directory of the name `rust-std-TRIPLE` inside and then a `lib` directory.
        # This `lib` directory needs to be extracted and merged with the `rustc/lib`
        # directory from the host compiler above.
        # TODO: make it possible to request an additional cross-target to add to this
        # list.
        stdlibs = [host_triple(), "arm-linux-androideabi"]
        for target in stdlibs:
            snapshot_url = ("https://static-rust-lang-org.s3.amazonaws.com/dist/%s/rust-std-nightly-%s.tar.gz"
                            % (date, target))
            tgz_file = install_dir + ('rust-std-nightly-%s.tar.gz' % target)

            download_file("Host rust library for target %s" % target, snapshot_url, tgz_file)
            print("Extracting Rust stdlib for target %s..." % target)
            extract(tgz_file, install_dir)
            shutil.copytree(path.join(install_dir, "rust-std-nightly-%s" % target,
                                      "rust-std-%s" % target, "lib", "rustlib", target),
                            path.join(install_dir, "rustc-nightly-%s" % host_triple(),
                                      "rustc", "lib", "rustlib", target))
            shutil.rmtree(path.join(install_dir, "rust-std-nightly-%s" % target))

        print("Snapshot Rust ready.")

    @Command('bootstrap-rust-docs',
             description='Download the Rust documentation',
             category='bootstrap')
    @CommandArgument('--force', '-f',
                     action='store_true',
                     help='Force download even if docs already exist')
    def bootstrap_rustc_docs(self, force=False):
        self.ensure_bootstrapped()
        rust_root = self.config["tools"]["rust-root"]
        docs_dir = path.join(rust_root, "doc")
        if not force and path.exists(docs_dir):
            print("Snapshot Rust docs already downloaded.", end=" ")
            print("Use |bootstrap-rust-docs --force| to download again.")
            return

        if path.isdir(docs_dir):
            shutil.rmtree(docs_dir)
        docs_name = self.rust_snapshot_path().replace("rustc-", "rust-docs-")
        snapshot_url = ("https://static-rust-lang-org.s3.amazonaws.com/dist/rust-docs-nightly-%s.tar.gz"
                        % host_triple())
        tgz_file = path.join(rust_root, 'doc.tar.gz')

        download_file("Rust docs", snapshot_url, tgz_file)

        print("Extracting Rust docs...")
        temp_dir = path.join(rust_root, "temp_docs")
        if path.isdir(temp_dir):
            shutil.rmtree(temp_dir)
        extract(tgz_file, temp_dir)
        shutil.move(path.join(temp_dir, docs_name.split("/")[1],
                              "rust-docs", "share", "doc", "rust", "html"),
                    docs_dir)
        shutil.rmtree(temp_dir)
        print("Rust docs ready.")

    @Command('bootstrap-cargo',
             description='Download the Cargo build tool',
             category='bootstrap')
    @CommandArgument('--force', '-f',
                     action='store_true',
{'fraction_non_alphanumeric': 0.12387746003439272, 'fraction_numerical': 0.0075154448761225396, 'mean_word_length': 2.2395296059418195, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 8, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 46, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2472231', 'n_tokens_mistral': 4781, 'n_tokens_neox': 4582, 'n_words': 1168}