<commit_before>import os
import subprocess
import sys
import uuid

import dmake.common as common
from dmake.common import DMakeException, SharedVolumeNotFoundException, append_command
from dmake.deepobuild import DMakeFile

tag_push_error_msg = "Unauthorized to push the current state of deployment to git server. If the repository belongs to you, please check that the credentials declared in the DMAKE_JENKINS_SSH_AGENT_CREDENTIALS and DMAKE_JENKINS_HTTP_CREDENTIALS allow you to write to the repository."

###############################################################################

def find_symlinked_directories():
    symlinks = []
    for line in common.run_shell_command("for f in $(dmake_find . -type l); do echo \"$f $(ls -l $f | sed -e 's/.* -> //')\"; done").split('\n'):
        l = line.split(' ')
        if len(l) != 2:
            continue
        link_path = os.path.normpath(l[0])
        if not os.path.isdir(link_path):
            continue
        linked_dir = os.path.normpath(os.path.join(os.path.dirname(link_path), l[1]))
        if not os.path.isdir(linked_dir) or linked_dir[0] == '/':
            continue
        symlinks.append((link_path, linked_dir))
    return symlinks

def look_for_changed_directories():
    if common.force_full_deploy:
        return None

    if not common.target:
        tag = get_tag_name()
        common.logger.info("Looking for changes between HEAD and %s" % tag)
        git_ref = "%s...HEAD" % tag

        try:
            common.run_shell_command2("git fetch origin +refs/tags/{tag}:refs/tags/{tag}".format(tag=tag))
        except common.ShellError as e:
            common.logger.debug("Fetching tag {} failed: {}".format(tag, e))
            common.logger.info("Tag {} not found on remote, assuming everything changed.")
            return None
    else:
        if common.is_local:
            common.logger.info("Looking for changes with {}".format(common.target))
            git_ref = common.target
        else:
            common.logger.info("Looking for changes between HEAD and %s" % common.target)
            git_ref = "%s/%s...HEAD" % (common.remote, common.target)

    try:
        output = common.run_shell_command("git diff --name-only %s" % git_ref)
    except common.ShellError as e:
        common.logger.error("Error: " + str(e))
        return None

    if len(output) == 0:
        return []

    output = [file.strip() for file in output.split('\n')]
    symlinks = find_symlinked_directories()
    to_append = []
    for file in output:
        if len(file) == 0:
            continue
        for sl in symlinks:
            if file.startswith(sl[1]):
                f = file[len(sl[1]):]
                if len(f) == 0:
                    continue
                if f[0] == '/':
                    f = f[1:]
                to_append.append(os.path.join(sl[0], f))
    output += to_append
    common.logger.debug("Changed files: %s", output)

    changed_dirs = set()
    for file in output:
        if len(file) == 0:
            continue
        d = os.path.dirname(file)
        if d in changed_dirs:
            continue

        # Only keep bottom changed directories
        do_add = True
        to_remove = []
        for directory in changed_dirs:
            if directory.startswith(d): # sub directory of d
                do_add = False
            elif d.startswith(directory):
                to_remove = directory

        changed_dirs.difference_update(to_remove)
        if do_add:
            changed_dirs.add(d)
    common.logger.info("Changed directories: %s", changed_dirs)
    return list(changed_dirs)

###############################################################################

def load_dmake_files_list():
    # Ignore permission issues when searching for dmake.yml files, in a portable way
    build_files = common.run_shell_command('dmake_find . -name dmake.yml').split("\n")
    build_files = filter(lambda f: len(f.strip()) > 0, build_files)
    build_files = [file[2:] for file in build_files]
    # Important: for block listed files: we load file in order from root to deepest file
    build_files = sorted(build_files, key = lambda path: len(os.path.dirname(path)))
    return build_files

###############################################################################

def add_service_provider(service_providers, service, file, needs = None, base_variant = None):
    """'service', 'needs' and 'base_variant' are all service names."""
    common.logger.debug("add_service_provider: service: %s, needs: %s, variant: %s" % (service, needs, base_variant))
    if service in service_providers:
        existing_service_provider, _, _ = service_providers[service]
        if existing_service_provider != file:
            raise DMakeException('Service %s re-defined in %s. First defined in %s' % (service, file, existing_service_provider))
    else:
        service_providers[service] = (file, needs, base_variant)

###############################################################################

def activate_file(loaded_files, service_providers, service_dependencies, command, file):
    dmake_file = loaded_files[file]
    if command in ['test', 'run', 'deploy', 'build_docker']:
        nodes = []
        for service in dmake_file.get_services():
            full_service_name = "%s/%s" % (dmake_file.app_name, service.service_name)
            nodes += activate_service(loaded_files, service_providers, service_dependencies, command, full_service_name)
        return nodes
    else:
        raise DMakeException('Unexpected command %s' % command)

###############################################################################

def activate_shared_volumes(shared_volumes):
    children = []
    for shared_volume in shared_volumes:
        shared_volume_service_name = shared_volume.get_service_name()
        children += [('shared_volume', shared_volume_service_name, None)]
    return children

###############################################################################

def activate_link_shared_volumes(loaded_files, service_providers, service):
    file, _, _ = service_providers[service]
    dmake = loaded_files[file]
    link = dmake.get_docker_link(service)

    try:
        shared_volumes = link.get_shared_volumes()
    except SharedVolumeNotFoundException as e:
        raise DMakeException("%s in docker_link '%s' in file '%s'" % (e, link.link_name, file))
    return activate_shared_volumes(shared_volumes)

###############################################################################

def activate_service_shared_volumes(loaded_files, service_providers, service):
    file, _, _ = service_providers[service]
    dmake = loaded_files[file]
    s = dmake._get_service_(service)

    try:
        shared_volumes = s.get_shared_volumes()
    except SharedVolumeNotFoundException as e:
        raise DMakeException("%s in service '%s' in file '%s'" % (e, s.service_name, file))
    return activate_shared_volumes(shared_volumes)

###############################################################################

def activate_base(base_variant):
    if base_variant is None:
        # base_variant is None when no base image is specified,
        # (only root_image is)
        # in which case we do not need to do anything
        return []
    return [('base', base_variant, None)]

###############################################################################

def activate_link(loaded_files, service_providers, service_dependencies, service):
    file, _, _ = service_providers[service]
    dmake = loaded_files[file]
    s = dmake._get_service_(service)

    children = []
    for link in s.needed_links:
        children += activate_service(loaded_files, service_providers, service_dependencies, 'run_link', 'links/%s/%s' % (dmake.get_app_name(), link))

    return children

###############################################################################

def activate_needed_services(loaded_files, service_providers, service_dependencies, needs, command):
    children = []
    for service, service_customization in needs:
        children += activate_service(loaded_files, service_providers, service_dependencies, command, service, service_customization)
    return children

###############################################################################

def activate_service(loaded_files, service_providers, service_dependencies, command, service, service_customization=None):
    node = (command, service, service_customization)
    if command == 'test' and common.skip_tests:
        return []

    if node not in service_dependencies:
        if service not in service_providers:
            raise DMakeException("Cannot find service: %s" % service)
        file, needs, base_variant = service_providers[service]
        children = []
        if command == 'shell':
            children += activate_service_shared_volumes(loaded_files, service_providers, service)
            if common.options.with_dependencies and needs is not None:
                children += activate_needed_services(loaded_files, service_providers, service_dependencies, needs, 'run')
                children += activate_link(loaded_files, service_providers, service_dependencies, service)
            children += activate_base(base_variant)
        elif command == 'test':
            children += activate_service_shared_volumes(loaded_files, service_providers, service)
            if common.options.with_dependencies and needs is not None:
                children += activate_needed_services(loaded_files, service_providers, service_dependencies, needs, 'run')
            children += activate_service(loaded_files, service_providers, service_dependencies, 'build_docker', service)
            if common.options.with_dependencies:
                children += activate_link(loaded_files, service_providers, service_dependencies, service)
        elif command == 'build_docker':
            children += activate_base(base_variant)
        elif command == 'run':
            children += activate_service_shared_volumes(loaded_files, service_providers, service)
            if common.command in ['test', 'deploy']:
                children += activate_service(loaded_files, service_providers, service_dependencies, 'test', service)
            children += activate_service(loaded_files, service_providers, service_dependencies, 'build_docker', service)
            if common.options.with_dependencies and needs is not None:
                children += activate_needed_services(loaded_files, service_providers, service_dependencies, needs, 'run')
                children += activate_link(loaded_files, service_providers, service_dependencies, service)
        elif command == 'run_link':
            children += activate_link_shared_volumes(loaded_files, service_providers, service)
        elif command == 'deploy':
            children += activate_service(loaded_files, service_providers, service_dependencies, 'build_docker', service)
            children += activate_service(loaded_files, service_providers, service_dependencies, 'test', service)
            if common.options.with_dependencies and needs is not None:
                # enforce deployment order by re-using needed_services dependency graph
                # but we don't want to create extra deployments because of customization
                # => deploy recursively using needs dependency, but ignore service customization
                uncustomized_needs = [(child_service, None) for child_service, child_service_customization in needs]
                children += activate_needed_services(loaded_files, service_providers, service_dependencies, uncustomized_needs, 'deploy')

        else:
            raise Exception("Unknown command '%s'" % command)

        service_dependencies[node] = children

    return [node]
<commit_msg>Internal graph dependency bugfix: we didn't properly support services with needed links but no needed services.<commit_after>import os
import subprocess
import sys
import uuid

import dmake.common as common
from dmake.common import DMakeException, SharedVolumeNotFoundException, append_command
from dmake.deepobuild import DMakeFile

tag_push_error_msg = "Unauthorized to push the current state of deployment to git server. If the repository belongs to you, please check that the credentials declared in the DMAKE_JENKINS_SSH_AGENT_CREDENTIALS and DMAKE_JENKINS_HTTP_CREDENTIALS allow you to write to the repository."

###############################################################################

def find_symlinked_directories():
    symlinks = []
    for line in common.run_shell_command("for f in $(dmake_find . -type l); do echo \"$f $(ls -l $f | sed -e 's/.* -> //')\"; done").split('\n'):
        l = line.split(' ')
        if len(l) != 2:
            continue
        link_path = os.path.normpath(l[0])
        if not os.path.isdir(link_path):
            continue
        linked_dir = os.path.normpath(os.path.join(os.path.dirname(link_path), l[1]))
        if not os.path.isdir(linked_dir) or linked_dir[0] == '/':
            continue
        symlinks.append((link_path, linked_dir))
    return symlinks

def look_for_changed_directories():
    if common.force_full_deploy:
        return None

    if not common.target:
        tag = get_tag_name()
        common.logger.info("Looking for changes between HEAD and %s" % tag)
        git_ref = "%s...HEAD" % tag

        try:
            common.run_shell_command2("git fetch origin +refs/tags/{tag}:refs/tags/{tag}".format(tag=tag))
        except common.ShellError as e:
            common.logger.debug("Fetching tag {} failed: {}".format(tag, e))
            common.logger.info("Tag {} not found on remote, assuming everything changed.")
            return None
    else:
        if common.is_local:
            common.logger.info("Looking for changes with {}".format(common.target))
            git_ref = common.target
        else:
            common.logger.info("Looking for changes between HEAD and %s" % common.target)
            git_ref = "%s/%s...HEAD" % (common.remote, common.target)

    try:
        output = common.run_shell_command("git diff --name-only %s" % git_ref)
    except common.ShellError as e:
        common.logger.error("Error: " + str(e))
        return None

    if len(output) == 0:
        return []

    output = [file.strip() for file in output.split('\n')]
    symlinks = find_symlinked_directories()
    to_append = []
    for file in output:
        if len(file) == 0:
            continue
        for sl in symlinks:
            if file.startswith(sl[1]):
                f = file[len(sl[1]):]
                if len(f) == 0:
                    continue
                if f[0] == '/':
                    f = f[1:]
                to_append.append(os.path.join(sl[0], f))
    output += to_append
    common.logger.debug("Changed files: %s", output)

    changed_dirs = set()
    for file in output:
        if len(file) == 0:
            continue
        d = os.path.dirname(file)
        if d in changed_dirs:
            continue

        # Only keep bottom changed directories
        do_add = True
        to_remove = []
        for directory in changed_dirs:
            if directory.startswith(d): # sub directory of d
                do_add = False
            elif d.startswith(directory):
                to_remove = directory

        changed_dirs.difference_update(to_remove)
        if do_add:
            changed_dirs.add(d)
    common.logger.info("Changed directories: %s", changed_dirs)
    return list(changed_dirs)

###############################################################################

def load_dmake_files_list():
    # Ignore permission issues when searching for dmake.yml files, in a portable way
    build_files = common.run_shell_command('dmake_find . -name dmake.yml').split("\n")
    build_files = filter(lambda f: len(f.strip()) > 0, build_files)
    build_files = [file[2:] for file in build_files]
    # Important: for block listed files: we load file in order from root to deepest file
    build_files = sorted(build_files, key = lambda path: len(os.path.dirname(path)))
    return build_files

###############################################################################

def add_service_provider(service_providers, service, file, needs = None, base_variant = None):
    """'service', 'needs' and 'base_variant' are all service names."""
    common.logger.debug("add_service_provider: service: %s, needs: %s, variant: %s" % (service, needs, base_variant))
    if service in service_providers:
        existing_service_provider, _, _ = service_providers[service]
        if existing_service_provider != file:
            raise DMakeException('Service %s re-defined in %s. First defined in %s' % (service, file, existing_service_provider))
    else:
        service_providers[service] = (file, needs, base_variant)

###############################################################################

def activate_file(loaded_files, service_providers, service_dependencies, command, file):
    dmake_file = loaded_files[file]
    if command in ['test', 'run', 'deploy', 'build_docker']:
        nodes = []
        for service in dmake_file.get_services():
            full_service_name = "%s/%s" % (dmake_file.app_name, service.service_name)
            nodes += activate_service(loaded_files, service_providers, service_dependencies, command, full_service_name)
        return nodes
    else:
        raise DMakeException('Unexpected command %s' % command)

###############################################################################

def activate_shared_volumes(shared_volumes):
    children = []
    for shared_volume in shared_volumes:
        shared_volume_service_name = shared_volume.get_service_name()
        children += [('shared_volume', shared_volume_service_name, None)]
    return children

###############################################################################

def activate_link_shared_volumes(loaded_files, service_providers, service):
    file, _, _ = service_providers[service]
    dmake = loaded_files[file]
    link = dmake.get_docker_link(service)

    try:
        shared_volumes = link.get_shared_volumes()
    except SharedVolumeNotFoundException as e:
        raise DMakeException("%s in docker_link '%s' in file '%s'" % (e, link.link_name, file))
    return activate_shared_volumes(shared_volumes)

###############################################################################

def activate_service_shared_volumes(loaded_files, service_providers, service):
    file, _, _ = service_providers[service]
    dmake = loaded_files[file]
    s = dmake._get_service_(service)

    try:
        shared_volumes = s.get_shared_volumes()
    except SharedVolumeNotFoundException as e:
        raise DMakeException("%s in service '%s' in file '%s'" % (e, s.service_name, file))
    return activate_shared_volumes(shared_volumes)

###############################################################################

def activate_base(base_variant):
    if base_variant is None:
        # base_variant is None when no base image is specified,
        # (only root_image is)
        # in which case we do not need to do anything
        return []
    return [('base', base_variant, None)]

###############################################################################

def activate_link(loaded_files, service_providers, service_dependencies, service):
    file, _, _ = service_providers[service]
    dmake = loaded_files[file]
    s = dmake._get_service_(service)

    children = []
    for link in s.needed_links:
        children += activate_service(loaded_files, service_providers, service_dependencies, 'run_link', 'links/%s/%s' % (dmake.get_app_name(), link))

    return children

###############################################################################

def activate_needed_services(loaded_files, service_providers, service_dependencies, needs, command):
    children = []
    for service, service_customization in needs:
        children += activate_service(loaded_files, service_providers, service_dependencies, command, service, service_customization)
    return children

###############################################################################

def activate_service(loaded_files, service_providers, service_dependencies, command, service, service_customization=None):
    node = (command, service, service_customization)
    if command == 'test' and common.skip_tests:
        return []

    if node not in service_dependencies:
        if service not in service_providers:
            raise DMakeException("Cannot find service: %s" % service)
        file, needs, base_variant = service_providers[service]
        children = []
        if command == 'shell':
            children += activate_service_shared_volumes(loaded_files, service_providers, service)
            if common.options.with_dependencies and needs is not None:
                children += activate_needed_services(loaded_files, service_providers, service_dependencies, needs, 'run')
            if common.options.with_dependencies:
                children += activate_link(loaded_files, service_providers, service_dependencies, service)
            children += activate_base(base_variant)
        elif command == 'test':
            children += activate_service_shared_volumes(loaded_files, service_providers, service)
            if common.options.with_dependencies and needs is not None:
                children += activate_needed_services(loaded_files, service_providers, service_dependencies, needs, 'run')
            children += activate_service(loaded_files, service_providers, service_dependencies, 'build_docker', service)
            if common.options.with_dependencies:
                children += activate_link(loaded_files, service_providers, service_dependencies, service)
        elif command == 'build_docker':
            children += activate_base(base_variant)
        elif command == 'run':
            children += activate_service_shared_volumes(loaded_files, service_providers, service)
            if common.command in ['test', 'deploy']:
                children += activate_service(loaded_files, service_providers, service_dependencies, 'test', service)
            children += activate_service(loaded_files, service_providers, service_dependencies, 'build_docker', service)
            if common.options.with_dependencies and needs is not None:
                children += activate_needed_services(loaded_files, service_providers, service_dependencies, needs, 'run')
            if common.options.with_dependencies:
                children += activate_link(loaded_files, service_providers, service_dependencies, service)
        elif command == 'run_link':
            children += activate_link_shared_volumes(loaded_files, service_providers, service)
        elif command == 'deploy':
            children += activate_service(loaded_files, service_providers, service_dependencies, 'build_docker', service)
            children += activate_service(loaded_files, service_providers, service_dependencies, 'test', service)
            if common.options.with_dependencies and needs is not None:
                # enforce deployment order by re-using needed_services dependency graph
                # but we don't want to create extra deployments because of customization
                # => deploy recursively using needs dependency, but ignore service customization
                uncustomized_needs = [(child_service, None) for child_service, child_service_customization in needs]
                children += activate_needed_services(loaded_files, service_providers, service_dependencies, uncustomized_needs, 'deploy')

        else:
            raise Exception("Unknown command '%s'" % command)

        service_dependencies[node] = children

    return [node]

{'fraction_non_alphanumeric': 0.1594160461808751, 'fraction_numerical': 0.001338576089684598, 'mean_word_length': 3.41332841055935, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1011697', 'n_tokens_mistral': 6444, 'n_tokens_neox': 6182, 'n_words': 1639}