<commit_before># need a dict to set bloody .name field
from io import BytesIO
import logging
import os
import stat
from typing import List
from unittest import SkipTest
import uuid

import git
from git.cmd import Git
from git.compat import (
    defenc,
    is_win,
)
from git.config import (
    SectionConstraint,
    GitConfigParser,
    cp
)
from git.exc import (
    InvalidGitRepositoryError,
    NoSuchPathError,
    RepositoryDirtyError,
    BadName
)
from git.objects.base import IndexObject, Object
from git.objects.util import Traversable
from git.util import (
    Iterable,
    join_path_native,
    to_native_path_linux,
    RemoteProgress,
    rmtree,
    unbare_repo
)
from git.util import HIDE_WINDOWS_KNOWN_ERRORS

import os.path as osp

from .util import (
    mkhead,
    sm_name,
    sm_section,
    SubmoduleConfigParser,
    find_first_remote_branch
)


__all__ = ["Submodule", "UpdateProgress"]


log = logging.getLogger('git.objects.submodule.base')
log.addHandler(logging.NullHandler())


class UpdateProgress(RemoteProgress):

    """Class providing detailed progress information to the caller who should
    derive from it and implement the ``update(...)`` message"""
    CLONE, FETCH, UPDWKTREE = [1 << x for x in range(RemoteProgress._num_op_codes, RemoteProgress._num_op_codes + 3)]
    _num_op_codes = RemoteProgress._num_op_codes + 3

    __slots__ = ()


BEGIN = UpdateProgress.BEGIN
END = UpdateProgress.END
CLONE = UpdateProgress.CLONE
FETCH = UpdateProgress.FETCH
UPDWKTREE = UpdateProgress.UPDWKTREE


# IndexObject comes via util module, its a 'hacky' fix thanks to pythons import
# mechanism which cause plenty of trouble of the only reason for packages and
# modules is refactoring - subpackages shouldn't depend on parent packages
class Submodule(IndexObject, Iterable, Traversable):

    """Implements access to a git submodule. They are special in that their sha
    represents a commit in the submodule's repository which is to be checked out
    at the path of this instance.
    The submodule type does not have a string type associated with it, as it exists
    solely as a marker in the tree and index.

    All methods work in bare and non-bare repositories."""

    _id_attribute_ = "name"
    k_modules_file = '.gitmodules'
    k_head_option = 'branch'
    k_head_default = 'master'
    k_default_mode = stat.S_IFDIR | stat.S_IFLNK        # submodules are directories with link-status

    # this is a bogus type for base class compatibility
    type = 'submodule'

    __slots__ = ('_parent_commit', '_url', '_branch_path', '_name', '__weakref__')
    _cache_attrs = ('path', '_url', '_branch_path')

    def __init__(self, repo, binsha, mode=None, path=None, name=None, parent_commit=None, url=None, branch_path=None):
        """Initialize this instance with its attributes. We only document the ones
        that differ from ``IndexObject``

        :param repo: Our parent repository
        :param binsha: binary sha referring to a commit in the remote repository, see url parameter
        :param parent_commit: see set_parent_commit()
        :param url: The url to the remote repository which is the submodule
        :param branch_path: full (relative) path to ref to checkout when cloning the remote repository"""
        super(Submodule, self).__init__(repo, binsha, mode, path)
        self.size = 0
        self._parent_commit = parent_commit
        if url is not None:
            self._url = url
        if branch_path is not None:
            assert isinstance(branch_path, str)
            self._branch_path = branch_path
        if name is not None:
            self._name = name

    def _set_cache_(self, attr):
        if attr in ('path', '_url', '_branch_path'):
            reader = self.config_reader()
            # default submodule values
            try:
                self.path = reader.get('path')
            except cp.NoSectionError as e:
                raise ValueError("This submodule instance does not exist anymore in '%s' file"
                                 % osp.join(self.repo.working_tree_dir, '.gitmodules')) from e
            # end
            self._url = reader.get('url')
            # git-python extension values - optional
            self._branch_path = reader.get_value(self.k_head_option, git.Head.to_full_path(self.k_head_default))
        elif attr == '_name':
            raise AttributeError("Cannot retrieve the name of a submodule if it was not set initially")
        else:
            super(Submodule, self)._set_cache_(attr)
        # END handle attribute name

    @classmethod
    def _get_intermediate_items(cls, item: 'Submodule') -> List['Submodule']:  # type: ignore
        """:return: all the submodules of our module repository"""
        try:
            return cls.list_items(item.module())
        except InvalidGitRepositoryError:
            return []
        # END handle intermediate items

    @classmethod
    def _need_gitfile_submodules(cls, git):
        return git.version_info[:3] >= (1, 7, 5)

    def __eq__(self, other):
        """Compare with another submodule"""
        # we may only compare by name as this should be the ID they are hashed with
        # Otherwise this type wouldn't be hashable
        # return self.path == other.path and self.url == other.url and super(Submodule, self).__eq__(other)
        return self._name == other._name

    def __ne__(self, other):
        """Compare with another submodule for inequality"""
        return not (self == other)

    def __hash__(self):
        """Hash this instance using its logical id, not the sha"""
        return hash(self._name)

    def __str__(self):
        return self._name

    def __repr__(self):
        return "git.%s(name=%s, path=%s, url=%s, branch_path=%s)"\
               % (type(self).__name__, self._name, self.path, self.url, self.branch_path)

    @classmethod
    def _config_parser(cls, repo, parent_commit, read_only):
        """:return: Config Parser constrained to our submodule in read or write mode
        :raise IOError: If the .gitmodules file cannot be found, either locally or in the repository
            at the given parent commit. Otherwise the exception would be delayed until the first
            access of the config parser"""
        parent_matches_head = True
        if parent_commit is not None:
            try:
                parent_matches_head = repo.head.commit == parent_commit
            except ValueError:
                # We are most likely in an empty repository, so the HEAD doesn't point to a valid ref
                pass
        # end handle parent_commit

        if not repo.bare and parent_matches_head:
            fp_module = osp.join(repo.working_tree_dir, cls.k_modules_file)
        else:
            assert parent_commit is not None, "need valid parent_commit in bare repositories"
            try:
                fp_module = cls._sio_modules(parent_commit)
            except KeyError as e:
                raise IOError("Could not find %s file in the tree of parent commit %s" %
                              (cls.k_modules_file, parent_commit)) from e
            # END handle exceptions
        # END handle non-bare working tree

        if not read_only and (repo.bare or not parent_matches_head):
            raise ValueError("Cannot write blobs of 'historical' submodule configurations")
        # END handle writes of historical submodules

        return SubmoduleConfigParser(fp_module, read_only=read_only)

    def _clear_cache(self):
        # clear the possibly changed values
        for name in self._cache_attrs:
            try:
                delattr(self, name)
            except AttributeError:
                pass
            # END try attr deletion
        # END for each name to delete

    @classmethod
    def _sio_modules(cls, parent_commit):
        """:return: Configuration file as BytesIO - we only access it through the respective blob's data"""
        sio = BytesIO(parent_commit.tree[cls.k_modules_file].data_stream.read())
        sio.name = cls.k_modules_file
        return sio

    def _config_parser_constrained(self, read_only):
        """:return: Config Parser constrained to our submodule in read or write mode"""
        try:
            pc = self.parent_commit
        except ValueError:
            pc = None
        # end handle empty parent repository
        parser = self._config_parser(self.repo, pc, read_only)
        parser.set_submodule(self)
        return SectionConstraint(parser, sm_section(self.name))

    @classmethod
    def _module_abspath(cls, parent_repo, path, name):
        if cls._need_gitfile_submodules(parent_repo.git):
            return osp.join(parent_repo.git_dir, 'modules', name)
        return osp.join(parent_repo.working_tree_dir, path)
        # end

    @classmethod
    def _clone_repo(cls, repo, url, path, name, **kwargs):
        """:return: Repo instance of newly cloned repository
        :param repo: our parent repository
        :param url: url to clone from
        :param path: repository-relative path to the submodule checkout location
        :param name: canonical of the submodule
        :param kwrags: additinoal arguments given to git.clone"""
        module_abspath = cls._module_abspath(repo, path, name)
        module_checkout_path = module_abspath
        if cls._need_gitfile_submodules(repo.git):
            kwargs['separate_git_dir'] = module_abspath
            module_abspath_dir = osp.dirname(module_abspath)
            if not osp.isdir(module_abspath_dir):
                os.makedirs(module_abspath_dir)
            module_checkout_path = osp.join(repo.working_tree_dir, path)
        # end

        clone = git.Repo.clone_from(url, module_checkout_path, **kwargs)
        if cls._need_gitfile_submodules(repo.git):
            cls._write_git_file_and_module_config(module_checkout_path, module_abspath)
        # end
        return clone

    @classmethod
    def _to_relative_path(cls, parent_repo, path):
        """:return: a path guaranteed  to be relative to the given parent-repository
        :raise ValueError: if path is not contained in the parent repository's working tree"""
        path = to_native_path_linux(path)
        if path.endswith('/'):
            path = path[:-1]
        # END handle trailing slash

        if osp.isabs(path):
            working_tree_linux = to_native_path_linux(parent_repo.working_tree_dir)
            if not path.startswith(working_tree_linux):
                raise ValueError("Submodule checkout path '%s' needs to be within the parents repository at '%s'"
                                 % (working_tree_linux, path))
            path = path[len(working_tree_linux.rstrip('/')) + 1:]
            if not path:
                raise ValueError("Absolute submodule path '%s' didn't yield a valid relative path" % path)
            # end verify converted relative path makes sense
        # end convert to a relative path

        return path

    @classmethod
    def _write_git_file_and_module_config(cls, working_tree_dir, module_abspath):
        """Writes a .git file containing a (preferably) relative path to the actual git module repository.
        It is an error if the module_abspath cannot be made into a relative path, relative to the working_tree_dir
        :note: will overwrite existing files !
        :note: as we rewrite both the git file as well as the module configuration, we might fail on the configuration
            and will not roll back changes done to the git file. This should be a non-issue, but may easily be fixed
            if it becomes one
        :param working_tree_dir: directory to write the .git file into
        :param module_abspath: absolute path to the bare repository
        """
        git_file = osp.join(working_tree_dir, '.git')
        rela_path = osp.relpath(module_abspath, start=working_tree_dir)
        if is_win:
            if osp.isfile(git_file):
                os.remove(git_file)
        with open(git_file, 'wb') as fp:
            fp.write(("gitdir: %s" % rela_path).encode(defenc))

        with GitConfigParser(osp.join(module_abspath, 'config'),
                             read_only=False, merge_includes=False) as writer:
            writer.set_value('core', 'worktree',
                             to_native_path_linux(osp.relpath(working_tree_dir, start=module_abspath)))

    #{ Edit Interface

    @classmethod
    def add(cls, repo, name, path, url=None, branch=None, no_checkout=False, depth=None, env=None):
        """Add a new submodule to the given repository. This will alter the index
        as well as the .gitmodules file, but will not create a new commit.
        If the submodule already exists, no matter if the configuration differs
        from the one provided, the existing submodule will be returned.

        :param repo: Repository instance which should receive the submodule
        :param name: The name/identifier for the submodule
        :param path: repository-relative or absolute path at which the submodule
            should be located
            It will be created as required during the repository initialization.
        :param url: git-clone compatible URL, see git-clone reference for more information
            If None, the repository is assumed to exist, and the url of the first
            remote is taken instead. This is useful if you want to make an existing
            repository a submodule of anotherone.
        :param branch: name of branch at which the submodule should (later) be checked out.
            The given branch must exist in the remote repository, and will be checked
            out locally as a tracking branch.
            It will only be written into the configuration if it not None, which is
            when the checked out branch will be the one the remote HEAD pointed to.
            The result you get in these situation is somewhat fuzzy, and it is recommended
            to specify at least 'master' here.
            Examples are 'master' or 'feature/new'
        :param no_checkout: if True, and if the repository has to be cloned manually,
            no checkout will be performed
        :param depth: Create a shallow clone with a history truncated to the
            specified number of commits.
        :param env: Optional dictionary containing the desired environment variables.
            Note: Provided variables will be used to update the execution
            environment for `git`. If some variable is not specified in `env`
            and is defined in `os.environ`, value from `os.environ` will be used.
            If you want to unset some variable, consider providing empty string
            as its value.
        :return: The newly created submodule instance
        :note: works atomically, such that no change will be done if the repository
            update fails for instance"""
        if repo.bare:
            raise InvalidGitRepositoryError("Cannot add submodules to bare repositories")
        # END handle bare repos

        path = cls._to_relative_path(repo, path)

        # assure we never put backslashes into the url, as some operating systems
        # like it ...
        if url is not None:
            url = to_native_path_linux(url)
        # END assure url correctness

        # INSTANTIATE INTERMEDIATE SM
        sm = cls(repo, cls.NULL_BIN_SHA, cls.k_default_mode, path, name, url='invalid-temporary')
        if sm.exists():
            # reretrieve submodule from tree
            try:
                sm = repo.head.commit.tree[path]
                sm._name = name
                return sm
            except KeyError:
                # could only be in index
                index = repo.index
                entry = index.entries[index.entry_key(path, 0)]
                sm.binsha = entry.binsha
                return sm
            # END handle exceptions
        # END handle existing

        # fake-repo - we only need the functionality on the branch instance
        br = git.Head(repo, git.Head.to_full_path(str(branch) or cls.k_head_default))
        has_module = sm.module_exists()
        branch_is_default = branch is None
        if has_module and url is not None:
            if url not in [r.url for r in sm.module().remotes]:
                raise ValueError(
                    "Specified URL '%s' does not match any remote url of the repository at '%s'" % (url, sm.abspath))
            # END check url
        # END verify urls match

        mrepo = None
        if url is None:
            if not has_module:
                raise ValueError("A URL was not given and existing repository did not exsit at %s" % path)
            # END check url<commit_msg>Fix exsit typo<commit_after># need a dict to set bloody .name field
from io import BytesIO
import logging
import os
import stat
from typing import List
from unittest import SkipTest
import uuid

import git
from git.cmd import Git
from git.compat import (
    defenc,
    is_win,
)
from git.config import (
    SectionConstraint,
    GitConfigParser,
    cp
)
from git.exc import (
    InvalidGitRepositoryError,
    NoSuchPathError,
    RepositoryDirtyError,
    BadName
)
from git.objects.base import IndexObject, Object
from git.objects.util import Traversable
from git.util import (
    Iterable,
    join_path_native,
    to_native_path_linux,
    RemoteProgress,
    rmtree,
    unbare_repo
)
from git.util import HIDE_WINDOWS_KNOWN_ERRORS

import os.path as osp

from .util import (
    mkhead,
    sm_name,
    sm_section,
    SubmoduleConfigParser,
    find_first_remote_branch
)


__all__ = ["Submodule", "UpdateProgress"]


log = logging.getLogger('git.objects.submodule.base')
log.addHandler(logging.NullHandler())


class UpdateProgress(RemoteProgress):

    """Class providing detailed progress information to the caller who should
    derive from it and implement the ``update(...)`` message"""
    CLONE, FETCH, UPDWKTREE = [1 << x for x in range(RemoteProgress._num_op_codes, RemoteProgress._num_op_codes + 3)]
    _num_op_codes = RemoteProgress._num_op_codes + 3

    __slots__ = ()


BEGIN = UpdateProgress.BEGIN
END = UpdateProgress.END
CLONE = UpdateProgress.CLONE
FETCH = UpdateProgress.FETCH
UPDWKTREE = UpdateProgress.UPDWKTREE


# IndexObject comes via util module, its a 'hacky' fix thanks to pythons import
# mechanism which cause plenty of trouble of the only reason for packages and
# modules is refactoring - subpackages shouldn't depend on parent packages
class Submodule(IndexObject, Iterable, Traversable):

    """Implements access to a git submodule. They are special in that their sha
    represents a commit in the submodule's repository which is to be checked out
    at the path of this instance.
    The submodule type does not have a string type associated with it, as it exists
    solely as a marker in the tree and index.

    All methods work in bare and non-bare repositories."""

    _id_attribute_ = "name"
    k_modules_file = '.gitmodules'
    k_head_option = 'branch'
    k_head_default = 'master'
    k_default_mode = stat.S_IFDIR | stat.S_IFLNK        # submodules are directories with link-status

    # this is a bogus type for base class compatibility
    type = 'submodule'

    __slots__ = ('_parent_commit', '_url', '_branch_path', '_name', '__weakref__')
    _cache_attrs = ('path', '_url', '_branch_path')

    def __init__(self, repo, binsha, mode=None, path=None, name=None, parent_commit=None, url=None, branch_path=None):
        """Initialize this instance with its attributes. We only document the ones
        that differ from ``IndexObject``

        :param repo: Our parent repository
        :param binsha: binary sha referring to a commit in the remote repository, see url parameter
        :param parent_commit: see set_parent_commit()
        :param url: The url to the remote repository which is the submodule
        :param branch_path: full (relative) path to ref to checkout when cloning the remote repository"""
        super(Submodule, self).__init__(repo, binsha, mode, path)
        self.size = 0
        self._parent_commit = parent_commit
        if url is not None:
            self._url = url
        if branch_path is not None:
            assert isinstance(branch_path, str)
            self._branch_path = branch_path
        if name is not None:
            self._name = name

    def _set_cache_(self, attr):
        if attr in ('path', '_url', '_branch_path'):
            reader = self.config_reader()
            # default submodule values
            try:
                self.path = reader.get('path')
            except cp.NoSectionError as e:
                raise ValueError("This submodule instance does not exist anymore in '%s' file"
                                 % osp.join(self.repo.working_tree_dir, '.gitmodules')) from e
            # end
            self._url = reader.get('url')
            # git-python extension values - optional
            self._branch_path = reader.get_value(self.k_head_option, git.Head.to_full_path(self.k_head_default))
        elif attr == '_name':
            raise AttributeError("Cannot retrieve the name of a submodule if it was not set initially")
        else:
            super(Submodule, self)._set_cache_(attr)
        # END handle attribute name

    @classmethod
    def _get_intermediate_items(cls, item: 'Submodule') -> List['Submodule']:  # type: ignore
        """:return: all the submodules of our module repository"""
        try:
            return cls.list_items(item.module())
        except InvalidGitRepositoryError:
            return []
        # END handle intermediate items

    @classmethod
    def _need_gitfile_submodules(cls, git):
        return git.version_info[:3] >= (1, 7, 5)

    def __eq__(self, other):
        """Compare with another submodule"""
        # we may only compare by name as this should be the ID they are hashed with
        # Otherwise this type wouldn't be hashable
        # return self.path == other.path and self.url == other.url and super(Submodule, self).__eq__(other)
        return self._name == other._name

    def __ne__(self, other):
        """Compare with another submodule for inequality"""
        return not (self == other)

    def __hash__(self):
        """Hash this instance using its logical id, not the sha"""
        return hash(self._name)

    def __str__(self):
        return self._name

    def __repr__(self):
        return "git.%s(name=%s, path=%s, url=%s, branch_path=%s)"\
               % (type(self).__name__, self._name, self.path, self.url, self.branch_path)

    @classmethod
    def _config_parser(cls, repo, parent_commit, read_only):
        """:return: Config Parser constrained to our submodule in read or write mode
        :raise IOError: If the .gitmodules file cannot be found, either locally or in the repository
            at the given parent commit. Otherwise the exception would be delayed until the first
            access of the config parser"""
        parent_matches_head = True
        if parent_commit is not None:
            try:
                parent_matches_head = repo.head.commit == parent_commit
            except ValueError:
                # We are most likely in an empty repository, so the HEAD doesn't point to a valid ref
                pass
        # end handle parent_commit

        if not repo.bare and parent_matches_head:
            fp_module = osp.join(repo.working_tree_dir, cls.k_modules_file)
        else:
            assert parent_commit is not None, "need valid parent_commit in bare repositories"
            try:
                fp_module = cls._sio_modules(parent_commit)
            except KeyError as e:
                raise IOError("Could not find %s file in the tree of parent commit %s" %
                              (cls.k_modules_file, parent_commit)) from e
            # END handle exceptions
        # END handle non-bare working tree

        if not read_only and (repo.bare or not parent_matches_head):
            raise ValueError("Cannot write blobs of 'historical' submodule configurations")
        # END handle writes of historical submodules

        return SubmoduleConfigParser(fp_module, read_only=read_only)

    def _clear_cache(self):
        # clear the possibly changed values
        for name in self._cache_attrs:
            try:
                delattr(self, name)
            except AttributeError:
                pass
            # END try attr deletion
        # END for each name to delete

    @classmethod
    def _sio_modules(cls, parent_commit):
        """:return: Configuration file as BytesIO - we only access it through the respective blob's data"""
        sio = BytesIO(parent_commit.tree[cls.k_modules_file].data_stream.read())
        sio.name = cls.k_modules_file
        return sio

    def _config_parser_constrained(self, read_only):
        """:return: Config Parser constrained to our submodule in read or write mode"""
        try:
            pc = self.parent_commit
        except ValueError:
            pc = None
        # end handle empty parent repository
        parser = self._config_parser(self.repo, pc, read_only)
        parser.set_submodule(self)
        return SectionConstraint(parser, sm_section(self.name))

    @classmethod
    def _module_abspath(cls, parent_repo, path, name):
        if cls._need_gitfile_submodules(parent_repo.git):
            return osp.join(parent_repo.git_dir, 'modules', name)
        return osp.join(parent_repo.working_tree_dir, path)
        # end

    @classmethod
    def _clone_repo(cls, repo, url, path, name, **kwargs):
        """:return: Repo instance of newly cloned repository
        :param repo: our parent repository
        :param url: url to clone from
        :param path: repository-relative path to the submodule checkout location
        :param name: canonical of the submodule
        :param kwrags: additinoal arguments given to git.clone"""
        module_abspath = cls._module_abspath(repo, path, name)
        module_checkout_path = module_abspath
        if cls._need_gitfile_submodules(repo.git):
            kwargs['separate_git_dir'] = module_abspath
            module_abspath_dir = osp.dirname(module_abspath)
            if not osp.isdir(module_abspath_dir):
                os.makedirs(module_abspath_dir)
            module_checkout_path = osp.join(repo.working_tree_dir, path)
        # end

        clone = git.Repo.clone_from(url, module_checkout_path, **kwargs)
        if cls._need_gitfile_submodules(repo.git):
            cls._write_git_file_and_module_config(module_checkout_path, module_abspath)
        # end
        return clone

    @classmethod
    def _to_relative_path(cls, parent_repo, path):
        """:return: a path guaranteed  to be relative to the given parent-repository
        :raise ValueError: if path is not contained in the parent repository's working tree"""
        path = to_native_path_linux(path)
        if path.endswith('/'):
            path = path[:-1]
        # END handle trailing slash

        if osp.isabs(path):
            working_tree_linux = to_native_path_linux(parent_repo.working_tree_dir)
            if not path.startswith(working_tree_linux):
                raise ValueError("Submodule checkout path '%s' needs to be within the parents repository at '%s'"
                                 % (working_tree_linux, path))
            path = path[len(working_tree_linux.rstrip('/')) + 1:]
            if not path:
                raise ValueError("Absolute submodule path '%s' didn't yield a valid relative path" % path)
            # end verify converted relative path makes sense
        # end convert to a relative path

        return path

    @classmethod
    def _write_git_file_and_module_config(cls, working_tree_dir, module_abspath):
        """Writes a .git file containing a (preferably) relative path to the actual git module repository.
        It is an error if the module_abspath cannot be made into a relative path, relative to the working_tree_dir
        :note: will overwrite existing files !
        :note: as we rewrite both the git file as well as the module configuration, we might fail on the configuration
            and will not roll back changes done to the git file. This should be a non-issue, but may easily be fixed
            if it becomes one
        :param working_tree_dir: directory to write the .git file into
        :param module_abspath: absolute path to the bare repository
        """
        git_file = osp.join(working_tree_dir, '.git')
        rela_path = osp.relpath(module_abspath, start=working_tree_dir)
        if is_win:
            if osp.isfile(git_file):
                os.remove(git_file)
        with open(git_file, 'wb') as fp:
            fp.write(("gitdir: %s" % rela_path).encode(defenc))

        with GitConfigParser(osp.join(module_abspath, 'config'),
                             read_only=False, merge_includes=False) as writer:
            writer.set_value('core', 'worktree',
                             to_native_path_linux(osp.relpath(working_tree_dir, start=module_abspath)))

    #{ Edit Interface

    @classmethod
    def add(cls, repo, name, path, url=None, branch=None, no_checkout=False, depth=None, env=None):
        """Add a new submodule to the given repository. This will alter the index
        as well as the .gitmodules file, but will not create a new commit.
        If the submodule already exists, no matter if the configuration differs
        from the one provided, the existing submodule will be returned.

        :param repo: Repository instance which should receive the submodule
        :param name: The name/identifier for the submodule
        :param path: repository-relative or absolute path at which the submodule
            should be located
            It will be created as required during the repository initialization.
        :param url: git-clone compatible URL, see git-clone reference for more information
            If None, the repository is assumed to exist, and the url of the first
            remote is taken instead. This is useful if you want to make an existing
            repository a submodule of anotherone.
        :param branch: name of branch at which the submodule should (later) be checked out.
            The given branch must exist in the remote repository, and will be checked
            out locally as a tracking branch.
            It will only be written into the configuration if it not None, which is
            when the checked out branch will be the one the remote HEAD pointed to.
            The result you get in these situation is somewhat fuzzy, and it is recommended
            to specify at least 'master' here.
            Examples are 'master' or 'feature/new'
        :param no_checkout: if True, and if the repository has to be cloned manually,
            no checkout will be performed
        :param depth: Create a shallow clone with a history truncated to the
            specified number of commits.
        :param env: Optional dictionary containing the desired environment variables.
            Note: Provided variables will be used to update the execution
            environment for `git`. If some variable is not specified in `env`
            and is defined in `os.environ`, value from `os.environ` will be used.
            If you want to unset some variable, consider providing empty string
            as its value.
        :return: The newly created submodule instance
        :note: works atomically, such that no change will be done if the repository
            update fails for instance"""
        if repo.bare:
            raise InvalidGitRepositoryError("Cannot add submodules to bare repositories")
        # END handle bare repos

        path = cls._to_relative_path(repo, path)

        # assure we never put backslashes into the url, as some operating systems
        # like it ...
        if url is not None:
            url = to_native_path_linux(url)
        # END assure url correctness

        # INSTANTIATE INTERMEDIATE SM
        sm = cls(repo, cls.NULL_BIN_SHA, cls.k_default_mode, path, name, url='invalid-temporary')
        if sm.exists():
            # reretrieve submodule from tree
            try:
                sm = repo.head.commit.tree[path]
                sm._name = name
                return sm
            except KeyError:
                # could only be in index
                index = repo.index
                entry = index.entries[index.entry_key(path, 0)]
                sm.binsha = entry.binsha
                return sm
            # END handle exceptions
        # END handle existing

        # fake-repo - we only need the functionality on the branch instance
        br = git.Head(repo, git.Head.to_full_path(str(branch) or cls.k_head_default))
        has_module = sm.module_exists()
        branch_is_default = branch is None
        if has_module and url is not None:
            if url not in [r.url for r in sm.module().remotes]:
                raise ValueError(
                    "Specified URL '%s' does not match any remote url of the repository at '%s'" % (url, sm.abspath))
            # END check url
        # END verify urls match

        mrepo = None
        if url is None:
            if not has_module:
                raise ValueError("A URL was not given and existing repository did not exist at %s" % path)
            # END check url
{'fraction_non_alphanumeric': 0.06995135635202483, 'fraction_numerical': 0.0006565400340207109, 'mean_word_length': 2.4422187981510017, 'pattern_counts': {'":': 12, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6248388', 'n_tokens_mistral': 9026, 'n_tokens_neox': 8809, 'n_words': 3335}