<commit_before>
from time import time

from sqlobject import connectionForURI, sqlhub, SQLObjectNotFound
import boto
from ConfigParser import SafeConfigParser
from os.path import expanduser

from pdnsmodels import record, domain, CNAME, MASTER
from functools import wraps


def memoize(fctn):
    memory = {}
    @wraps(fctn)
    def memo(*args):
        haxh = tuple(args)

        if haxh not in memory:
            memory[haxh] = fctn(*args)

        return memory[haxh]
    return memo


#dnsdb = '/root/pdns.sqlite'
dnsdb = '/var/spool/powerdns/pdns.sqlite'
conn_str = 'sqlite://%s' % dnsdb
connection = connectionForURI(conn_str)
sqlhub.processConnection = connection

def get_credentials():
    confp = SafeConfigParser()
    confp.read(expanduser('~/.s3cfg'))
    return (confp.get('default', 'access_key'),
                     confp.get('default', 'secret_key'))


@memoize
def get_rootdn_record(name):
    try:
        return domain.selectBy(name=name).getOne()
    except SQLObjectNotFound:
        return domain(name=name, type=MASTER)

def gatherinstances():
    conn = boto.connect_ec2(*get_credentials())
    instances = conn.get_all_instances()
    return instances

def process_all(instances):
    started_at = int(time())

    instances = map(lambda x: x.instances[0], instances)
    instances = filter(lambda x: x.tags, instances)

    def _process(fqdn, domain_base, public_dns_name):
        try:
            sr = record.selectBy(name=fqdn).getOne()
            if sr.content != public_dns_name:
                sr.update(content=public_dns_name)
            else:
                return sr #record already existed, and didn't get updated, so
                          # change_date will still be older than started_at.
                          # return it for exclusion in stale record processing.
        except SQLObjectNotFound:
            sr = record(domain=get_rootdn_record(domain_base),<commit_msg>Iterate over every region looking for servers that we need to create DNS records for.<commit_after>
from time import time

from sqlobject import connectionForURI, sqlhub, SQLObjectNotFound
import boto
import boto.ec2
from ConfigParser import SafeConfigParser
from os.path import expanduser

from pdnsmodels import record, domain, CNAME, MASTER
from functools import wraps


def memoize(fctn):
    memory = {}
    @wraps(fctn)
    def memo(*args):
        haxh = tuple(args)

        if haxh not in memory:
            memory[haxh] = fctn(*args)

        return memory[haxh]
    return memo


#dnsdb = '/root/pdns.sqlite'
dnsdb = '/var/spool/powerdns/pdns.sqlite'
conn_str = 'sqlite://%s' % dnsdb
connection = connectionForURI(conn_str)
sqlhub.processConnection = connection

def get_credentials():
    confp = SafeConfigParser()
    confp.read(expanduser('~/.s3cfg'))
    return {'aws_access_key_id': confp.get('default', 'access_key'),
            'aws_secret_access_key': confp.get('default', 'secret_key')}


def trampoline(*instance_lists):
    instance_lists = iter(instance_lists)
    while 1:
        instances = iter(instance_lists.next())
        while 1:
            try:
                yield instances.next()
            except StopIteration:
                break



@memoize
def get_rootdn_record(name):
    try:
        return domain.selectBy(name=name).getOne()
    except SQLObjectNotFound:
        return domain(name=name, type=MASTER)


def gatherinstances():
    regions = boto.ec2.regions(**get_credentials())
    regions = (region.connect(**get_credentials()) for region in regions)

    instances = (region.get_all_instances() for region in regions)

    return trampoline(*instances)


def process_all(instances):
    started_at = int(time())

    instances = map(lambda x: x.instances[0], instances)
    instances = filter(lambda x: x.tags, instances)

    def _process(fqdn, domain_base, public_dns_name):
        try:
            sr = record.selectBy(name=fqdn).getOne()
            if sr.content != public_dns_name:
                sr.update(content=public_dns_name)
            else:
                return sr #record already existed, and didn't get updated, so
                          # change_date will still be older than started_at.
                          # return it for exclusion in stale record processing.
        except SQLObjectNotFound:
            sr = record(domain=get_rootdn_record(domain_base),
{'fraction_non_alphanumeric': 0.09670781893004116, 'fraction_numerical': 0.00205761316872428, 'mean_word_length': 2.8854351687388986, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '642841', 'n_tokens_mistral': 1359, 'n_tokens_neox': 1314, 'n_words': 335}