<commit_before>#!/usr/bin/env python
"""calicoctl

Override the host:port of the ETCD server by setting the environment variable
ETCD_AUTHORITY [default: 127.0.0.1:4001]

Usage:
  calicoctl node --ip=<IP> [--node-image=<DOCKER_IMAGE_NAME>] [--ip6=<IP6>] [--force-unix-socket]
  calicoctl node stop [--force]
  calicoctl status
  calicoctl shownodes [--detailed]
  calicoctl profile show [--detailed]
  calicoctl profile add <PROFILE>
  calicoctl profile remove <PROFILE>
  calicoctl profile <PROFILE> tag show
  calicoctl profile <PROFILE> tag add <TAG>
  calicoctl profile <PROFILE> tag remove <TAG>
  calicoctl profile <PROFILE> rule show
  calicoctl profile <PROFILE> rule json
  calicoctl profile <PROFILE> rule update
  calicoctl profile <PROFILE> member add <CONTAINER>
  calicoctl ipv4 pool add <CIDR>
  calicoctl ipv4 pool del <CIDR>
  calicoctl ipv4 pool show
  calicoctl ipv6 pool add <CIDR>
  calicoctl ipv6 pool del <CIDR>
  calicoctl ipv6 pool show
  calicoctl container add <CONTAINER> <IP>
  calicoctl container remove <CONTAINER> [--force]
  calicoctl reset
  calicoctl diags

Options:
 --ip=<IP>                The local management address to use.
 --ip6=<IP6>              The local IPv6 management address to use.
 --node-image=<DOCKER_IMAGE_NAME>    Docker image to use for
                          Calico's per-node container
                          [default: calico/node:latest]

"""
import socket
import sys
import time
import os
import re
import json

import netaddr
from docopt import docopt
import sh
import docker
import docker.utils
from netaddr import IPNetwork, IPAddress
from netaddr.core import AddrFormatError
from prettytable import PrettyTable

from node.adapter.datastore import (ETCD_AUTHORITY_ENV,
                                    ETCD_AUTHORITY_DEFAULT,
                                    Rules)
from node.adapter.ipam import IPAMClient
from node.adapter import netns


hostname = socket.gethostname()
client = IPAMClient()
DOCKER_VERSION = "1.16"
docker_client = docker.Client(version=DOCKER_VERSION,
                              base_url=os.getenv("DOCKER_HOST",
                                                 "unix://var/run/docker.sock"))

try:
    modprobe = sh.Command._create('modprobe')
    sysctl = sh.Command._create("sysctl")
    restart = sh.Command._create("restart")
except sh.CommandNotFound as e:
    print "Missing command: %s" % e.message
    
DEFAULT_IPV4_POOL = IPNetwork("192.168.0.0/16")
DEFAULT_IPV6_POOL = IPNetwork("fd80:24e2:f998:72d6::/64")

REAL_SOCK = "/var/run/docker.real.sock"
POWERSTRIP_SOCK = "/var/run/docker.sock"
DOCKER_DEFAULT_FILENAME = "/etc/default/docker"
DOCKER_OPTIONS = 'DOCKER_OPTS="-H unix://%s"' % REAL_SOCK

def get_container_info(container_name):
    """
    Get the full container info array from a partial ID or name.

    :param container_name: The partial ID or name of the container.
    :return: The container info array
    """
    try:
        info = docker_client.inspect_container(container_name)
    except docker.errors.APIError as e:
        if e.response.status_code == 404:
            # Re-raise as a key error for consistency.
            raise KeyError("Container %s was not found." % container_name)
        else:
            raise
    return info

def get_container_id(container_name):
    """
    Get the full container ID from a partial ID or name.

    :param container_name: The partial ID or name of the container.
    :return: The container ID as a string.
    """
    info = get_container_info(container_name)
    return info["Id"]


class ConfigError(Exception):
    pass


def container_add(container_name, ip):
    """
    Add a container (on this host) to Calico networking with the given IP.

    :param container_name: The name or ID of the container.
    :param ip: An IPAddress object with the desired IP to assign.
    """
    # The netns manipulations must be done as root.
    if os.geteuid() != 0:
        print >> sys.stderr, "`calicoctl container add` must be run as root."
        sys.exit(2)
    try:
        info = get_container_info(container_name)
    except KeyError as err:
        print err.message
        sys.exit(1)
    container_id = info["Id"]

    # Check if the container already exists
    try:
        _ = client.get_ep_id_from_cont(hostname, container_id)
    except KeyError:
        # Calico doesn't know about this container.  Continue.
        pass
    else:
        # Calico already set up networking for this container.  Since we got
        # called with an IP address, we shouldn't just silently exit, since
        # that would confuse the user: the container would not be reachable on
        # that IP address.
        print "%s has already been configured with Calico Networking." % \
              container_name
        sys.exit(1)

    # Check the IP is in the allocation pool.  If it isn't, BIRD won't export
    # it.
    ip = IPAddress(ip)
    version = "v%s" % ip.version
    pools = client.get_ip_pools(version)
    pool = None
    for candidate_pool in pools:
        if ip in candidate_pool:
            pool = candidate_pool

    if pool is None:
        print "%s is not in any configured pools" % ip
        sys.exit(1)

    # Check the container is actually running.
    if not info["State"]["Running"]:
        print "%s is not currently running." % container_name
        sys.exit(1)

    # The next hop IPs for this host are stored in etcd.
    next_hops = client.get_default_next_hops(hostname)
    try:
        next_hops[ip.version]
    except KeyError:
        print "This node is not configured for IPv%d." % ip.version
        sys.exit(1)

    # Assign the IP
    if not client.assign_address(pool, ip):
        print "IP address is already assigned in pool %s " % pool
        sys.exit(1)

    # Actually configure the netns.  Use eth1 since eth0 is the docker bridge.
    pid = info["State"]["Pid"]
    endpoint = netns.set_up_endpoint(ip, pid, next_hops,
                                     veth_name="eth1",
                                     proc_alias="proc")

    # Register the endpoint
    client.set_endpoint(hostname, container_id, endpoint)

    print "IP %s added to %s" % (ip, container_name)


def container_remove(container_name):
    """
    Remove a container (on this host) from Calico networking.

    The container may be left in a state without any working networking.
    If there is a network adaptor in the host namespace used by the container
    then it is removed.

    :param container_name: The name or ID of the container.
    """
    # Resolve the name to ID.
    container_id = get_container_id(container_name)

    # Find the endpoint ID. We need this to find any ACL rules
    try:
        endpoint_id = client.get_ep_id_from_cont(hostname, container_id)
    except KeyError:
        print "Container %s doesn't contain any endpoints" % container_name
        return

    # Remove any IP address assignments that this endpoint has
    endpoint = client.get_endpoint(hostname, container_id, endpoint_id)
    for net in endpoint.ipv4_nets | endpoint.ipv6_nets:
        assert(net.size == 1)
        ip = net.ip
        pools = client.get_ip_pools("v%s" % ip.version)
        for pool in pools:
            if ip in pool:
                client.unassign_address(pool, ip)

    # Remove the endpoint
    netns.remove_endpoint(endpoint_id)

    # Remove the container from the datastore.
    client.remove_container(hostname, container_id)

    print "Removed Calico interface from %s" % container_name


def node_stop(force):
    if force or len(client.get_hosts()[hostname]["docker"]) == 0:
        client.remove_host(hostname)
        try:
            docker_client.stop("calico-node")
        except docker.errors.APIError as err:
            if err.response.status_code != 404:
                raise

        print "Node stopped and all configuration removed"
    else:
        print "Current host has active endpoints so can't be stopped." + \
              " Force with --force"


def clean_restart_docker(sock_to_wait_on):
    if os.path.exists(REAL_SOCK):
        os.remove(REAL_SOCK)
    if os.path.exists(POWERSTRIP_SOCK):
        os.remove(POWERSTRIP_SOCK)

    restart("docker")

    # Wait for docker to create the socket
    while not os.path.exists(sock_to_wait_on):
        time.sleep(0.1)


def node(ip, force_unix_socket, node_image, ip6=""):
    # modprobe and sysctl require root privileges.
    if os.geteuid() != 0:
        print >> sys.stderr, "`calicoctl node` must be run as root."
        sys.exit(2)

    modprobe("ip6_tables")
    modprobe("xt_set")

    # Set up etcd
    ipv4_pools = client.get_ip_pools("v4")
    ipv6_pools = client.get_ip_pools("v6")

    # Create default pools if required
    if not ipv4_pools:
        client.add_ip_pool("v4", DEFAULT_IPV4_POOL)
    if not ipv6_pools:
        client.add_ip_pool("v6", DEFAULT_IPV6_POOL)

    client.ensure_global_config()
    client.create_host(hostname, ip, ip6)

    # Enable IP forwarding since all compute hosts are vRouters.
    sysctl("-w", "net.ipv4.ip_forward=1")
    sysctl("-w", "net.ipv6.conf.all.forwarding=1")

    # The docker daemon could be in one of two states:
    # 1) Listening on /var/run/docker.sock - the default
    # 2) listening on /var/run/docker.real.sock - if it's been previously run
    #    with --force-unix-socket
    enable_socket = "NO"

    # We might need to talk to a different docker endpoint, so create some
    # client flexibility.
    node_docker_client = docker_client

    if force_unix_socket:
        # Update docker to use a different unix socket, so powerstrip can run
        # its proxy on the "normal" one. This provides simple access for
        # existing tools to the powerstrip proxy.

        # Set the docker daemon to listen on the docker.real.sock by updating
        # the config, clearing old sockets and restarting.
        socket_config_exists = \
            DOCKER_OPTIONS in open(DOCKER_DEFAULT_FILENAME).read()
        if not socket_config_exists:
            with open(DOCKER_DEFAULT_FILENAME, "a") as docker_config:
                docker_config.write(DOCKER_OPTIONS)
            clean_restart_docker(REAL_SOCK)

        # Always remove the socket that powerstrip will use, as it gets upset
        # otherwise.
        if os.path.exists(POWERSTRIP_SOCK):
            os.remove(POWERSTRIP_SOCK)

        # At this point, docker is listening on a new port but powerstrip isn't
        # running, so docker clients need to talk directly to docker.
        node_docker_client = docker.Client(version=DOCKER_VERSION,
                                           base_url="unix://%s" % REAL_SOCK)
        enable_socket = "YES"
    else:
        # Not using the unix socket.  If there is --force-unix-socket config in
        # place, do some cleanup
        socket_config_exists = \
            DOCKER_OPTIONS in open(DOCKER_DEFAULT_FILENAME).read()
        if socket_config_exists:
            good_lines = [line for line in open(DOCKER_DEFAULT_FILENAME)
                          if DOCKER_OPTIONS not in line]
            open(DOCKER_DEFAULT_FILENAME, 'w').writelines(good_lines)
            clean_restart_docker(POWERSTRIP_SOCK)

    try:
        node_docker_client.remove_container("calico-node", force=True)
    except docker.errors.APIError as err:
        if err.response.status_code != 404:
            raise

    etcd_authority = os.getenv(ETCD_AUTHORITY_ENV, ETCD_AUTHORITY_DEFAULT)

    environment = [<commit_msg>Ignore problems cleaning up --force-unix-socket<commit_after>#!/usr/bin/env python
"""calicoctl

Override the host:port of the ETCD server by setting the environment variable
ETCD_AUTHORITY [default: 127.0.0.1:4001]

Usage:
  calicoctl node --ip=<IP> [--node-image=<DOCKER_IMAGE_NAME>] [--ip6=<IP6>] [--force-unix-socket]
  calicoctl node stop [--force]
  calicoctl status
  calicoctl shownodes [--detailed]
  calicoctl profile show [--detailed]
  calicoctl profile add <PROFILE>
  calicoctl profile remove <PROFILE>
  calicoctl profile <PROFILE> tag show
  calicoctl profile <PROFILE> tag add <TAG>
  calicoctl profile <PROFILE> tag remove <TAG>
  calicoctl profile <PROFILE> rule show
  calicoctl profile <PROFILE> rule json
  calicoctl profile <PROFILE> rule update
  calicoctl profile <PROFILE> member add <CONTAINER>
  calicoctl ipv4 pool add <CIDR>
  calicoctl ipv4 pool del <CIDR>
  calicoctl ipv4 pool show
  calicoctl ipv6 pool add <CIDR>
  calicoctl ipv6 pool del <CIDR>
  calicoctl ipv6 pool show
  calicoctl container add <CONTAINER> <IP>
  calicoctl container remove <CONTAINER> [--force]
  calicoctl reset
  calicoctl diags

Options:
 --ip=<IP>                The local management address to use.
 --ip6=<IP6>              The local IPv6 management address to use.
 --node-image=<DOCKER_IMAGE_NAME>    Docker image to use for
                          Calico's per-node container
                          [default: calico/node:latest]

"""
import socket
import sys
import time
import os
import re
import json

import netaddr
from docopt import docopt
import sh
import docker
import docker.utils
from netaddr import IPNetwork, IPAddress
from netaddr.core import AddrFormatError
from prettytable import PrettyTable

from node.adapter.datastore import (ETCD_AUTHORITY_ENV,
                                    ETCD_AUTHORITY_DEFAULT,
                                    Rules)
from node.adapter.ipam import IPAMClient
from node.adapter import netns


hostname = socket.gethostname()
client = IPAMClient()
DOCKER_VERSION = "1.16"
docker_client = docker.Client(version=DOCKER_VERSION,
                              base_url=os.getenv("DOCKER_HOST",
                                                 "unix://var/run/docker.sock"))

try:
    modprobe = sh.Command._create('modprobe')
    sysctl = sh.Command._create("sysctl")
except sh.CommandNotFound as e:
    print "Missing command: %s" % e.message
    
DEFAULT_IPV4_POOL = IPNetwork("192.168.0.0/16")
DEFAULT_IPV6_POOL = IPNetwork("fd80:24e2:f998:72d6::/64")

REAL_SOCK = "/var/run/docker.real.sock"
POWERSTRIP_SOCK = "/var/run/docker.sock"
DOCKER_DEFAULT_FILENAME = "/etc/default/docker"
DOCKER_OPTIONS = 'DOCKER_OPTS="-H unix://%s"' % REAL_SOCK

def get_container_info(container_name):
    """
    Get the full container info array from a partial ID or name.

    :param container_name: The partial ID or name of the container.
    :return: The container info array
    """
    try:
        info = docker_client.inspect_container(container_name)
    except docker.errors.APIError as e:
        if e.response.status_code == 404:
            # Re-raise as a key error for consistency.
            raise KeyError("Container %s was not found." % container_name)
        else:
            raise
    return info

def get_container_id(container_name):
    """
    Get the full container ID from a partial ID or name.

    :param container_name: The partial ID or name of the container.
    :return: The container ID as a string.
    """
    info = get_container_info(container_name)
    return info["Id"]


class ConfigError(Exception):
    pass


def container_add(container_name, ip):
    """
    Add a container (on this host) to Calico networking with the given IP.

    :param container_name: The name or ID of the container.
    :param ip: An IPAddress object with the desired IP to assign.
    """
    # The netns manipulations must be done as root.
    if os.geteuid() != 0:
        print >> sys.stderr, "`calicoctl container add` must be run as root."
        sys.exit(2)
    try:
        info = get_container_info(container_name)
    except KeyError as err:
        print err.message
        sys.exit(1)
    container_id = info["Id"]

    # Check if the container already exists
    try:
        _ = client.get_ep_id_from_cont(hostname, container_id)
    except KeyError:
        # Calico doesn't know about this container.  Continue.
        pass
    else:
        # Calico already set up networking for this container.  Since we got
        # called with an IP address, we shouldn't just silently exit, since
        # that would confuse the user: the container would not be reachable on
        # that IP address.
        print "%s has already been configured with Calico Networking." % \
              container_name
        sys.exit(1)

    # Check the IP is in the allocation pool.  If it isn't, BIRD won't export
    # it.
    ip = IPAddress(ip)
    version = "v%s" % ip.version
    pools = client.get_ip_pools(version)
    pool = None
    for candidate_pool in pools:
        if ip in candidate_pool:
            pool = candidate_pool

    if pool is None:
        print "%s is not in any configured pools" % ip
        sys.exit(1)

    # Check the container is actually running.
    if not info["State"]["Running"]:
        print "%s is not currently running." % container_name
        sys.exit(1)

    # The next hop IPs for this host are stored in etcd.
    next_hops = client.get_default_next_hops(hostname)
    try:
        next_hops[ip.version]
    except KeyError:
        print "This node is not configured for IPv%d." % ip.version
        sys.exit(1)

    # Assign the IP
    if not client.assign_address(pool, ip):
        print "IP address is already assigned in pool %s " % pool
        sys.exit(1)

    # Actually configure the netns.  Use eth1 since eth0 is the docker bridge.
    pid = info["State"]["Pid"]
    endpoint = netns.set_up_endpoint(ip, pid, next_hops,
                                     veth_name="eth1",
                                     proc_alias="proc")

    # Register the endpoint
    client.set_endpoint(hostname, container_id, endpoint)

    print "IP %s added to %s" % (ip, container_name)


def container_remove(container_name):
    """
    Remove a container (on this host) from Calico networking.

    The container may be left in a state without any working networking.
    If there is a network adaptor in the host namespace used by the container
    then it is removed.

    :param container_name: The name or ID of the container.
    """
    # Resolve the name to ID.
    container_id = get_container_id(container_name)

    # Find the endpoint ID. We need this to find any ACL rules
    try:
        endpoint_id = client.get_ep_id_from_cont(hostname, container_id)
    except KeyError:
        print "Container %s doesn't contain any endpoints" % container_name
        return

    # Remove any IP address assignments that this endpoint has
    endpoint = client.get_endpoint(hostname, container_id, endpoint_id)
    for net in endpoint.ipv4_nets | endpoint.ipv6_nets:
        assert(net.size == 1)
        ip = net.ip
        pools = client.get_ip_pools("v%s" % ip.version)
        for pool in pools:
            if ip in pool:
                client.unassign_address(pool, ip)

    # Remove the endpoint
    netns.remove_endpoint(endpoint_id)

    # Remove the container from the datastore.
    client.remove_container(hostname, container_id)

    print "Removed Calico interface from %s" % container_name


def node_stop(force):
    if force or len(client.get_hosts()[hostname]["docker"]) == 0:
        client.remove_host(hostname)
        try:
            docker_client.stop("calico-node")
        except docker.errors.APIError as err:
            if err.response.status_code != 404:
                raise

        print "Node stopped and all configuration removed"
    else:
        print "Current host has active endpoints so can't be stopped." + \
              " Force with --force"


def clean_restart_docker(sock_to_wait_on):
    restart = sh.Command._create("restart")

    if os.path.exists(REAL_SOCK):
        os.remove(REAL_SOCK)
    if os.path.exists(POWERSTRIP_SOCK):
        os.remove(POWERSTRIP_SOCK)

    restart("docker")

    # Wait for docker to create the socket
    while not os.path.exists(sock_to_wait_on):
        time.sleep(0.1)


def node(ip, force_unix_socket, node_image, ip6=""):
    # modprobe and sysctl require root privileges.
    if os.geteuid() != 0:
        print >> sys.stderr, "`calicoctl node` must be run as root."
        sys.exit(2)

    modprobe("ip6_tables")
    modprobe("xt_set")

    # Set up etcd
    ipv4_pools = client.get_ip_pools("v4")
    ipv6_pools = client.get_ip_pools("v6")

    # Create default pools if required
    if not ipv4_pools:
        client.add_ip_pool("v4", DEFAULT_IPV4_POOL)
    if not ipv6_pools:
        client.add_ip_pool("v6", DEFAULT_IPV6_POOL)

    client.ensure_global_config()
    client.create_host(hostname, ip, ip6)

    # Enable IP forwarding since all compute hosts are vRouters.
    sysctl("-w", "net.ipv4.ip_forward=1")
    sysctl("-w", "net.ipv6.conf.all.forwarding=1")

    # The docker daemon could be in one of two states:
    # 1) Listening on /var/run/docker.sock - the default
    # 2) listening on /var/run/docker.real.sock - if it's been previously run
    #    with --force-unix-socket
    enable_socket = "NO"

    # We might need to talk to a different docker endpoint, so create some
    # client flexibility.
    node_docker_client = docker_client

    if force_unix_socket:
        # Update docker to use a different unix socket, so powerstrip can run
        # its proxy on the "normal" one. This provides simple access for
        # existing tools to the powerstrip proxy.

        # Set the docker daemon to listen on the docker.real.sock by updating
        # the config, clearing old sockets and restarting.
        socket_config_exists = \
            DOCKER_OPTIONS in open(DOCKER_DEFAULT_FILENAME).read()
        if not socket_config_exists:
            with open(DOCKER_DEFAULT_FILENAME, "a") as docker_config:
                docker_config.write(DOCKER_OPTIONS)
            clean_restart_docker(REAL_SOCK)

        # Always remove the socket that powerstrip will use, as it gets upset
        # otherwise.
        if os.path.exists(POWERSTRIP_SOCK):
            os.remove(POWERSTRIP_SOCK)

        # At this point, docker is listening on a new port but powerstrip isn't
        # running, so docker clients need to talk directly to docker.
        node_docker_client = docker.Client(version=DOCKER_VERSION,
                                           base_url="unix://%s" % REAL_SOCK)
        enable_socket = "YES"
    else:
        # Not using the unix socket.  If there is --force-unix-socket config in
        # place, try some cleanup
        try:
            socket_config_exists = \
                DOCKER_OPTIONS in open(DOCKER_DEFAULT_FILENAME).read()
        except IOError:
            # Can't find Docker config file.  Don't attempt to change anything.
            # Just carry on.
            pass
        else:
            if socket_config_exists:
                good_lines = [line for line in open(DOCKER_DEFAULT_FILENAME)
                              if DOCKER_OPTIONS not in line]
                open(DOCKER_DEFAULT_FILENAME, 'w').writelines(good_lines)
                clean_restart_docker(POWERSTRIP_SOCK)

    try:
        node_docker_client.remove_container("calico-node", force=True)
    except docker.errors.APIError as err:
        if err.response.status_code != 404:
            raise

    etcd_authority = os.getenv(ETCD_AUTHORITY_ENV, ETCD_AUTHORITY_DEFAULT)

    environment = [
{'fraction_non_alphanumeric': 0.08225424822542482, 'fraction_numerical': 0.008259840825984082, 'mean_word_length': 2.567526089625537, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 17, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6759959', 'n_tokens_mistral': 7257, 'n_tokens_neox': 6935, 'n_words': 2196}