<commit_before>#!/usr/bin/env python3
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

from __future__ import absolute_import, division, print_function, unicode_literals

import errno
import hashlib
import os
import re
import shutil
import stat
import subprocess
import sys
import tarfile
import time
import zipfile
from datetime import datetime
from typing import Dict, NamedTuple

from .copytree import prefetch_dir_if_eden
from .envfuncs import Env
from .errors import TransientFailure
from .platform import is_windows
from .runcmd import run_cmd


try:
    from urllib import urlretrieve
    from urlparse import urlparse
except ImportError:
    from urllib.parse import urlparse
    from urllib.request import urlretrieve


def file_name_is_cmake_file(file_name):
    file_name = file_name.lower()
    base = os.path.basename(file_name)
    return (
        base.endswith(".cmake")
        or base.endswith(".cmake.in")
        or base == "cmakelists.txt"
    )


class ChangeStatus(object):
    """Indicates the nature of changes that happened while updating
    the source directory.  There are two broad uses:
    * When extracting archives for third party software we want to
      know that we did something (eg: we either extracted code or
      we didn't do anything)
    * For 1st party code where we use shipit to transform the code,
      we want to know if we changed anything so that we can perform
      a build, but we generally want to be a little more nuanced
      and be able to distinguish between just changing a source file
      and whether we might need to reconfigure the build system.
    """

    def __init__(self, all_changed=False):
        """Construct a ChangeStatus object.  The default is to create
        a status that indicates no changes, but passing all_changed=True
        will create one that indicates that everything changed"""
        if all_changed:
            self.source_files = 1
            self.make_files = 1
        else:
            self.source_files = 0
            self.make_files = 0

    def record_change(self, file_name):
        """Used by the shipit fetcher to record changes as it updates
        files in the destination.  If the file name might be one used
        in the cmake build system that we use for 1st party code, then
        record that as a "make file" change.  We could broaden this
        to match any file used by various build systems, but it is
        only really useful for our internal cmake stuff at this time.
        If the file isn't a build file and is under the `fbcode_builder`
        dir then we don't class that as an interesting change that we
        might need to rebuild, so we ignore it.
        Otherwise we record the file as a source file change."""

        file_name = file_name.lower()
        if file_name_is_cmake_file(file_name):
            self.make_files += 1
        elif "/fbcode_builder/cmake" in file_name:
            self.source_files += 1
        elif "/fbcode_builder/" not in file_name:
            self.source_files += 1

    def sources_changed(self):
        """Returns true if any source files were changed during
        an update operation.  This will typically be used to decide
        that the build system to be run on the source dir in an
        incremental mode"""
        return self.source_files > 0

    def build_changed(self):
        """Returns true if any build files were changed during
        an update operation.  This will typically be used to decidfe
        that the build system should be reconfigured and re-run
        as a full build"""
        return self.make_files > 0


class Fetcher(object):
    """The Fetcher is responsible for fetching and extracting the
    sources for project.  The Fetcher instance defines where the
    extracted data resides and reports this to the consumer via
    its `get_src_dir` method."""

    def update(self):
        """Brings the src dir up to date, ideally minimizing
        changes so that a subsequent build doesn't over-build.
        Returns a ChangeStatus object that helps the caller to
        understand the nature of the changes required during
        the update."""
        return ChangeStatus()

    def clean(self):
        """Reverts any changes that might have been made to
        the src dir"""
        pass

    def hash(self):
        """Returns a hash that identifies the version of the code in the
        working copy.  For a git repo this is commit hash for the working
        copy.  For other Fetchers this should relate to the version of
        the code in the src dir.  The intent is that if a manifest
        changes the version/rev of a project that the hash be different.
        Importantly, this should be computable without actually fetching
        the code, as we want this to factor into a hash used to download
        a pre-built version of the code, without having to first download
        and extract its sources (eg: boost on windows is pretty painful).
        """
        pass

    def get_src_dir(self):
        """Returns the source directory that the project was
        extracted into"""
        pass


class LocalDirFetcher(object):
    """This class exists to override the normal fetching behavior, and
    use an explicit user-specified directory for the project sources.

    This fetcher cannot update or track changes.  It always reports that the
    project has changed, forcing it to always be built."""

    def __init__(self, path):
        self.path = os.path.realpath(path)

    def update(self):
        return ChangeStatus(all_changed=True)

    def hash(self):
        return "0" * 40

    def get_src_dir(self):
        return self.path


class SystemPackageFetcher(object):
    def __init__(self, build_options, packages):
        self.manager = build_options.host_type.get_package_manager()
        self.packages = packages.get(self.manager)
        if self.packages:
            self.installed = None
        else:
            self.installed = False

    def packages_are_installed(self):
        if self.installed is not None:
            return self.installed

        if self.manager == "rpm":
            result = run_cmd(["rpm", "-q"] + self.packages, allow_fail=True)
            self.installed = result == 0
        elif self.manager == "deb":
            result = run_cmd(["dpkg", "-s"] + self.packages, allow_fail=True)
            self.installed = result == 0
        else:
            self.installed = False

        return self.installed

    def update(self):
        assert self.installed
        return ChangeStatus(all_changed=False)

    def hash(self):
        return "0" * 40

    def get_src_dir(self):
        return None


class PreinstalledNopFetcher(SystemPackageFetcher):
    def __init__(self):
        self.installed = True


class GitFetcher(Fetcher):
    DEFAULT_DEPTH = 1

    def __init__(self, build_options, manifest, repo_url, rev, depth):
        # Extract the host/path portions of the URL and generate a flattened
        # directory name.  eg:
        # github.com/facebook/folly.git -> github.com-facebook-folly.git
        url = urlparse(repo_url)
        directory = "%s%s" % (url.netloc, url.path)<commit_msg>include system package state in cache key<commit_after>#!/usr/bin/env python3
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

from __future__ import absolute_import, division, print_function, unicode_literals

import errno
import hashlib
import os
import re
import shutil
import stat
import subprocess
import sys
import tarfile
import time
import zipfile
from datetime import datetime
from typing import Dict, NamedTuple

from .copytree import prefetch_dir_if_eden
from .envfuncs import Env
from .errors import TransientFailure
from .platform import is_windows
from .runcmd import run_cmd


try:
    from urllib import urlretrieve
    from urlparse import urlparse
except ImportError:
    from urllib.parse import urlparse
    from urllib.request import urlretrieve


def file_name_is_cmake_file(file_name):
    file_name = file_name.lower()
    base = os.path.basename(file_name)
    return (
        base.endswith(".cmake")
        or base.endswith(".cmake.in")
        or base == "cmakelists.txt"
    )


class ChangeStatus(object):
    """Indicates the nature of changes that happened while updating
    the source directory.  There are two broad uses:
    * When extracting archives for third party software we want to
      know that we did something (eg: we either extracted code or
      we didn't do anything)
    * For 1st party code where we use shipit to transform the code,
      we want to know if we changed anything so that we can perform
      a build, but we generally want to be a little more nuanced
      and be able to distinguish between just changing a source file
      and whether we might need to reconfigure the build system.
    """

    def __init__(self, all_changed=False):
        """Construct a ChangeStatus object.  The default is to create
        a status that indicates no changes, but passing all_changed=True
        will create one that indicates that everything changed"""
        if all_changed:
            self.source_files = 1
            self.make_files = 1
        else:
            self.source_files = 0
            self.make_files = 0

    def record_change(self, file_name):
        """Used by the shipit fetcher to record changes as it updates
        files in the destination.  If the file name might be one used
        in the cmake build system that we use for 1st party code, then
        record that as a "make file" change.  We could broaden this
        to match any file used by various build systems, but it is
        only really useful for our internal cmake stuff at this time.
        If the file isn't a build file and is under the `fbcode_builder`
        dir then we don't class that as an interesting change that we
        might need to rebuild, so we ignore it.
        Otherwise we record the file as a source file change."""

        file_name = file_name.lower()
        if file_name_is_cmake_file(file_name):
            self.make_files += 1
        elif "/fbcode_builder/cmake" in file_name:
            self.source_files += 1
        elif "/fbcode_builder/" not in file_name:
            self.source_files += 1

    def sources_changed(self):
        """Returns true if any source files were changed during
        an update operation.  This will typically be used to decide
        that the build system to be run on the source dir in an
        incremental mode"""
        return self.source_files > 0

    def build_changed(self):
        """Returns true if any build files were changed during
        an update operation.  This will typically be used to decidfe
        that the build system should be reconfigured and re-run
        as a full build"""
        return self.make_files > 0


class Fetcher(object):
    """The Fetcher is responsible for fetching and extracting the
    sources for project.  The Fetcher instance defines where the
    extracted data resides and reports this to the consumer via
    its `get_src_dir` method."""

    def update(self):
        """Brings the src dir up to date, ideally minimizing
        changes so that a subsequent build doesn't over-build.
        Returns a ChangeStatus object that helps the caller to
        understand the nature of the changes required during
        the update."""
        return ChangeStatus()

    def clean(self):
        """Reverts any changes that might have been made to
        the src dir"""
        pass

    def hash(self):
        """Returns a hash that identifies the version of the code in the
        working copy.  For a git repo this is commit hash for the working
        copy.  For other Fetchers this should relate to the version of
        the code in the src dir.  The intent is that if a manifest
        changes the version/rev of a project that the hash be different.
        Importantly, this should be computable without actually fetching
        the code, as we want this to factor into a hash used to download
        a pre-built version of the code, without having to first download
        and extract its sources (eg: boost on windows is pretty painful).
        """
        pass

    def get_src_dir(self):
        """Returns the source directory that the project was
        extracted into"""
        pass


class LocalDirFetcher(object):
    """This class exists to override the normal fetching behavior, and
    use an explicit user-specified directory for the project sources.

    This fetcher cannot update or track changes.  It always reports that the
    project has changed, forcing it to always be built."""

    def __init__(self, path):
        self.path = os.path.realpath(path)

    def update(self):
        return ChangeStatus(all_changed=True)

    def hash(self):
        return "0" * 40

    def get_src_dir(self):
        return self.path


class SystemPackageFetcher(object):
    def __init__(self, build_options, packages):
        self.manager = build_options.host_type.get_package_manager()
        self.packages = packages.get(self.manager)
        if self.packages:
            self.installed = None
        else:
            self.installed = False

    def packages_are_installed(self):
        if self.installed is not None:
            return self.installed

        cmd = None
        if self.manager == "rpm":
            cmd = ["rpm", "-q"] + sorted(self.packages)
        elif self.manager == "deb":
            cmd = ["dpkg", "-s"] + sorted(self.packages)

        if cmd:
            proc = subprocess.run(cmd, capture_output=True)
            if proc.returncode == 0:
                # captured as binary as we will hash this later
                self.installed = proc.stdout
            else:
                # Need all packages to be present to consider us installed
                self.installed = False
        else:
            self.installed = False

        return bool(self.installed)

    def update(self):
        assert self.installed
        return ChangeStatus(all_changed=False)

    def hash(self):
        if self.packages_are_installed():
            return hashlib.sha256(self.installed).hexdigest()
        else:
            return "0" * 40

    def get_src_dir(self):
        return None


class PreinstalledNopFetcher(SystemPackageFetcher):
    def __init__(self):
        self.installed = True


class GitFetcher(Fetcher):
    DEFAULT_DEPTH = 1

    def __init__(self, build_options, manifest, repo_url, rev, depth):
        # Extract the host/path portions of the URL and generate a flattened
        # directory name.  eg:
        # github.com/facebook/folly.git -> github.com-facebook-folly.git
        url = urlparse(repo_url)
        directory = "%s%s" % (url.netloc, url.path)
{'fraction_non_alphanumeric': 0.06038135593220339, 'fraction_numerical': 0.002913135593220339, 'mean_word_length': 2.5250875145857643, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3733323', 'n_tokens_mistral': 4034, 'n_tokens_neox': 3871, 'n_words': 1734}