<commit_before>#!/bin/env python
from __future__ import print_function

import boto3
import botocore
import consul
import datetime
import inspect
import os
import re
import sys

from docopt import docopt
from collections import OrderedDict
from threading import Thread, Lock
from time import sleep


class Logger(object):
    def __init__(self):
        self._out = sys.stdout
        self.level = "info"

    def info(self, event, **data):
        self.log("\033[32mINFO    \033[0m", event, **data)

    def warn(self, event, **data):
        self.log("\033[33mWARNING \033[0m", event, **data)

    def debug(self, event, **data):
        if self.level == "debug":
            self.log("\033[35mDEBUG   \033[0m", event, **data)

    def error(self, event, **data):
        self.log("\033[31mERROR   \033[0m", event, **data)

    def log(self, level, event, **data):
        formatted_data = " ".join(
            "{}={!r}".format(k, v) for k, v in data.iteritems()
        )
        self._out.write("{} {}{}\033[36m[{}]\033[0m {}\n".format(
            datetime.datetime.utcnow().replace(microsecond=0),
            level,
            "%s.%s" % (inspect.stack()[2][0].f_locals['self'].__class__.__name__, inspect.stack()[2][3]) if self.level == "debug" else "",
            event,
            formatted_data
        ))


class Unify411(object):
    """Class to generate dns records"""
    def __init__(self, consul_address, datacenter=None, external_hosts=[], zones=None, output=[], domains=[]):
        super(Unify411, self).__init__()
        self.lock = Lock()
        self.log = Logger()
        self.consul = consul.Consul(host=consul_address)
        self.external_hosts = external_hosts if external_hosts else []
        self.domains = domains
        self.output = output
        if datacenter:
            if datacenter in self.consul.catalog.datacenters():
                self.datacenter = datacenter
            else:
                self.log.error("Error %s is not valid datacenter (valid dc: %s)" % (datacenter, ",".join(self.consul.catalog.datacenters())))
                sys.exit(1)
        else:
            self.datacenter = self.consul.agent.self()['Config']['Datacenter']
        self.log.info("Connected to datacenter %s" % self.datacenter)

    def refresh(self):
        self.__get_wanted_records()
        self.__get_current_records()

    def __get_wanted_records(self):
        self.wanted_dns_records = {}
        self.__retrieve_consul_records()
        self.__retrieve_external_records(self.external_hosts)

    def __get_current_records(self):
        self.current_dns_records = {}
        self.__retrieve_aws_records()

    def generate_output_file(self, file_path=None, slave=False, domains=[]):
        out_a = ""
        out_cname = ""
        records = self.wanted_dns_records if not slave else self.current_dns_records
        for domain, entry in records.iteritems():
            if domains and domain not in domains:
                continue
            for host in sorted(entry):
                if re.match(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$', host[0]):
                    out_a += "%s %s %s\n" % (host[0], host[1], host[1].split(".")[0])
                else:
                    out_cname += "cname=%s,%s\n" % (host[1], host[0])
        if not file_path:
            return (out_a, out_cname)
        else:
            if not os.path.isfile("%s.conf" % file_path) or out_cname != file.read(open("%s.conf" % file_path)):
                with open("%s.conf" % file_path,'w') as output:
                    if out_a:
                        out_cname = "addn-hosts=%s.hosts\n" % file_path + out_cname
                    print(out_cname, end='', file=output)
            if not os.path.isfile("%s.hosts" % file_path) or out_a != file.read(open("%s.hosts" % file_path)):
                with open("%s.hosts" % file_path,'w') as output:
                    print(out_a, end='', file=output)

    def update_route53(self, commit=True):
        for profile, actions in self.__compute_aws_actions().iteritems():
            for name, action in actions.iteritems():
                if len(action['values']) == 1 and not re.match(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$', action['values'][0]):
                    c_type = "CNAME"
                else:
                    c_type = "A"
                self.log.info("Changing Record Set",  action=action['action'], type=c_type, name=name, values=action['values'])
                self._change_rr_sets(self.__aws_profiles[profile]['connection'], action['action'], self.__aws_profiles[profile]['zoneID'], name, c_type, action['values'], commit)

    def _change_rr_sets(self, route53_client, action, zone_id, name, _type, values, commit, ttl=30):
        change = {"Changes": [{"Action": action,
                               "ResourceRecordSet": {"Name": name,
                                                     "Type": _type,
                                                     "TTL": ttl,
                                                     "ResourceRecords": [{"Value": "{}".format(value)} for value in values],
                                                     }
                               }
                              ]
                  }

        if commit:
            try:
                route53_client.change_resource_record_sets(HostedZoneId=zone_id, ChangeBatch=change)
            except Exception as ex:
                self.log.error("Failed to commit the action for the resource record sets", action=action, name=name, type=_type, values=values, change=change, ex=ex)
                if ttl == 30 and action == "DELETE":
                    self.log.info("Trying the same with default boto TTL...")
                    self._change_rr_sets(route53_client, action, zone_id, name, _type, values, commit, ttl=600)

    def __find_parent_zone_for(self, zone):
        """ Helper to find if we have parent zone we can use"""
        parent_zones = []
        for domain in self.wanted_dns_records.keys():
            if domain in zone:
                parent_zones.append(domain)
        if parent_zones:
            parent_zones = sorted(parent_zones, key=lambda entry: len(entry))
            return parent_zones[0]
        else:
            return zone

    def __get_aws_profiles(self):
        """ This function will iterate over the potential profiles and try to create a connection. If it fails it will remove it."""
        self.__aws_profiles = {}
        for domain in self.wanted_dns_records.keys():
            # if the current domain is a sub zone of an existing domain try the parent domain instead.<commit_msg>Fix an issue when we specify the domains<commit_after>#!/bin/env python
from __future__ import print_function

import boto3
import botocore
import consul
import datetime
import inspect
import os
import re
import sys

from docopt import docopt
from collections import OrderedDict
from threading import Thread, Lock
from time import sleep


class Logger(object):
    def __init__(self):
        self._out = sys.stdout
        self.level = "info"

    def info(self, event, **data):
        self.log("\033[32mINFO    \033[0m", event, **data)

    def warn(self, event, **data):
        self.log("\033[33mWARNING \033[0m", event, **data)

    def debug(self, event, **data):
        if self.level == "debug":
            self.log("\033[35mDEBUG   \033[0m", event, **data)

    def error(self, event, **data):
        self.log("\033[31mERROR   \033[0m", event, **data)

    def log(self, level, event, **data):
        formatted_data = " ".join(
            "{}={!r}".format(k, v) for k, v in data.iteritems()
        )
        self._out.write("{} {}{}\033[36m[{}]\033[0m {}\n".format(
            datetime.datetime.utcnow().replace(microsecond=0),
            level,
            "%s.%s" % (inspect.stack()[2][0].f_locals['self'].__class__.__name__, inspect.stack()[2][3]) if self.level == "debug" else "",
            event,
            formatted_data
        ))


class Unify411(object):
    """Class to generate dns records"""
    def __init__(self, consul_address, datacenter=None, external_hosts=[], zones=None, output=[], domains=[]):
        super(Unify411, self).__init__()
        self.lock = Lock()
        self.log = Logger()
        self.consul = consul.Consul(host=consul_address)
        self.external_hosts = external_hosts if external_hosts else []
        self.domains = domains
        self.output = output
        if datacenter:
            if datacenter in self.consul.catalog.datacenters():
                self.datacenter = datacenter
            else:
                self.log.error("Error %s is not valid datacenter (valid dc: %s)" % (datacenter, ",".join(self.consul.catalog.datacenters())))
                sys.exit(1)
        else:
            self.datacenter = self.consul.agent.self()['Config']['Datacenter']
        self.log.info("Connected to datacenter %s" % self.datacenter)

    def refresh(self):
        self.__get_wanted_records()
        self.__get_current_records()

    def __get_wanted_records(self):
        self.wanted_dns_records = {}
        self.__retrieve_consul_records()
        self.__retrieve_external_records(self.external_hosts)

    def __get_current_records(self):
        self.current_dns_records = {}
        self.__retrieve_aws_records()

    def generate_output_file(self, file_path=None, slave=False, domains=[]):
        out_a = ""
        out_cname = ""
        records = self.wanted_dns_records if not slave else self.current_dns_records
        for domain, entry in records.iteritems():
            if domains and domain not in domains:
                continue
            for host in sorted(entry):
                if re.match(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$', host[0]):
                    out_a += "%s %s %s\n" % (host[0], host[1], host[1].split(".")[0])
                else:
                    out_cname += "cname=%s,%s\n" % (host[1], host[0])
        if not file_path:
            return (out_a, out_cname)
        else:
            if not os.path.isfile("%s.conf" % file_path) or out_cname != file.read(open("%s.conf" % file_path)):
                with open("%s.conf" % file_path,'w') as output:
                    if out_a:
                        out_cname = "addn-hosts=%s.hosts\n" % file_path + out_cname
                    print(out_cname, end='', file=output)
            if not os.path.isfile("%s.hosts" % file_path) or out_a != file.read(open("%s.hosts" % file_path)):
                with open("%s.hosts" % file_path,'w') as output:
                    print(out_a, end='', file=output)

    def update_route53(self, commit=True):
        for profile, actions in self.__compute_aws_actions().iteritems():
            for name, action in actions.iteritems():
                if len(action['values']) == 1 and not re.match(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$', action['values'][0]):
                    c_type = "CNAME"
                else:
                    c_type = "A"
                self.log.info("Changing Record Set",  action=action['action'], type=c_type, name=name, values=action['values'])
                self._change_rr_sets(self.__aws_profiles[profile]['connection'], action['action'], self.__aws_profiles[profile]['zoneID'], name, c_type, action['values'], commit)

    def _change_rr_sets(self, route53_client, action, zone_id, name, _type, values, commit, ttl=30):
        change = {"Changes": [{"Action": action,
                               "ResourceRecordSet": {"Name": name,
                                                     "Type": _type,
                                                     "TTL": ttl,
                                                     "ResourceRecords": [{"Value": "{}".format(value)} for value in values],
                                                     }
                               }
                              ]
                  }

        if commit:
            try:
                route53_client.change_resource_record_sets(HostedZoneId=zone_id, ChangeBatch=change)
            except Exception as ex:
                self.log.error("Failed to commit the action for the resource record sets", action=action, name=name, type=_type, values=values, change=change, ex=ex)
                if ttl == 30 and action == "DELETE":
                    self.log.info("Trying the same with default boto TTL...")
                    self._change_rr_sets(route53_client, action, zone_id, name, _type, values, commit, ttl=600)

    def __find_parent_zone_for(self, zone):
        """ Helper to find if we have parent zone we can use"""
        parent_zones = []
        for domain in self.wanted_dns_records.keys():
            if domain in zone:
                parent_zones.append(domain)
        if parent_zones:
            parent_zones = sorted(parent_zones, key=lambda entry: len(entry))
            return parent_zones[0]
        else:
            return zone

    def __get_aws_profiles(self):
        """ This function will iterate over the potential profiles and try to create a connection. If it fails it will remove it."""
        self.__aws_profiles = {}
        if self.domains:
            domains = self.domains
        else:
            domains = self.wanted_dns_records.keys()
        for domain in domains:
            # if the current domain is a sub zone of an existing domain try the parent domain instead.
{'fraction_non_alphanumeric': 0.12946528451040346, 'fraction_numerical': 0.014766201804757998, 'mean_word_length': 2.041506010433205, 'pattern_counts': {'":': 20, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1001563', 'n_tokens_mistral': 4091, 'n_tokens_neox': 3884, 'n_words': 951}