<commit_before>import os,sys,time
import subprocess
import tarfile
import datetime

import boto
import boto.ec2
import boto.s3
import boto.s3.connection

import utils

class Adslogging:
  
  def __init__(self):
    self.c = utils.connect(boto.s3.connection.S3Connection)

  def discoverLogstash(self):
    c = utils.connect(boto.ec2.connection.EC2Connection)
    instances = [i for i in c.get_only_instances() if i.tags.get('Name',None) == 'adslogging-asg']

    if instances:
      latest = sorted(instances,key=lambda i: datetime.datetime.strptime(i.launch_time[:i.lanuch_time.rfind('.')],'%Y-%m-%dT%H:%M:%s'))[-1]
      return latest.private_ip_address

    return None
    

  def localProvision(self,bucket='s3-adsabs-adslogging',key='adslogging_certs.tar.gz',certpath='/adslogging/dockerfiles/logstash/certs/'):
    utils.mkdir_p(certpath)
    b = self.c.get_bucket(bucket)
    k = b.get_key(key)
    f = os.path.join(certpath,key)
    k.get_contents_to_filename(f)
    with tarfile.open(f) as tf:
      tf.extractall(path=certpath)

    logstash_ip = discoverLogstash()
    with open('/SET_LOGSTASH_SERVER.bash','w') as fp:
      fp.write('export LOGSTASH_SERVER="%s:"' % logstash_ip)<commit_msg>Syntax fix<commit_after>import os,sys,time
import subprocess
import tarfile
import datetime

import boto
import boto.ec2
import boto.s3
import boto.s3.connection

import utils

class Adslogging:
  
  def __init__(self):
    self.c = utils.connect(boto.s3.connection.S3Connection)

  def discoverLogstash(self):
    c = utils.connect(boto.ec2.connection.EC2Connection)
    instances = [i for i in c.get_only_instances() if i.tags.get('Name',None) == 'adslogging-asg']

    if instances:
      latest = sorted(instances,key=lambda i: datetime.datetime.strptime(i.launch_time[:i.lanuch_time.rfind('.')],'%Y-%m-%dT%H:%M:%s'))[-1]
      return latest.private_ip_address

    return None
    

  def localProvision(self,bucket='s3-adsabs-adslogging',key='adslogging_certs.tar.gz',certpath='/adslogging/dockerfiles/logstash/certs/'):
    utils.mkdir_p(certpath)
    b = self.c.get_bucket(bucket)
    k = b.get_key(key)
    f = os.path.join(certpath,key)
    k.get_contents_to_filename(f)
    with tarfile.open(f) as tf:
      tf.extractall(path=certpath)

    logstash_ip = self.discoverLogstash()
    with open('/SET_LOGSTASH_SERVER.bash','w') as fp:
      fp.write('export LOGSTASH_SERVER="%s:"' % logstash_ip)
{'fraction_non_alphanumeric': 0.13745847176079734, 'fraction_numerical': 0.007475083056478406, 'mean_word_length': 4.294505494505494, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1171121', 'n_tokens_mistral': 903, 'n_tokens_neox': 891, 'n_words': 142}