<commit_before>#!/usr/bin/env python

import os,sys
import time
import re
import subprocess

import json
import requests
import logging
import logging.handlers

import boto
import boto.ec2

sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
from components import utils

IP_REGEX = r'\b(?:\d{1,3}\.){3}\d{1,3}\b'
logger = logging.getLogger(__name__)

class AppFilter(logging.Filter):
  def filter(self, record):
    record.app_name = 'backup-daily'
    return True

def getDockerContainerIP():
  cmd = ['docker','inspect','solr']
  P = subprocess.Popen(cmd,stdout=subprocess.PIPE)
  P.wait()
  lines = json.loads(P.stdout.read())
  ip = lines[0]['NetworkSettings']['IPAddress']
  return ip

def isLeader(URL='http://localhost:8983/solr/admin/collections?action=CLUSTERSTATUS&wt=json'):
  r = requests.get(URL)
  res = json.loads(r.content)

  leaders = []
  for shard in res['cluster']['collections']['collection1']['shards'].values():
    for replica in shard['replicas'].values():
      if 'leader' in replica and replica['leader']:
        ip = re.search(IP_REGEX,replica['base_url']).group()
        leaders.append(ip)

  if getDockerContainerIP() in leaders:
    return True
  return False

def createSnapshot():
  c = utils.connect(boto.ec2.connection.EC2Connection)
  this_instance = utils.get_this_instance()
  volume = [v for v in c.get_all_volumes() if v.attach_data.instance_id == this_instance.id and 'solr-data-volume' in v.tags.get('Name',None)]
  if len(volume) != 1:
    logger.critical('Wrong number of EBS volumes attached to instance %s' % this_instance.id)
    sys.exit(1)
  volume[0].create_snapshot(description='%s-%s-snapshot' % (volume.tags['Name'],volume.tags['shardId']) )

def isActive(URL='http://localhost:8983/solr/collection1/admin/mbeans?stats=true&wt=json'):
  r = requests.get(URL)
  res = json.loads(r.content)

  for i in res['solr-mbeans']:
    if i.upper() == 'UPDATEHANDLER':
      index = res['solr-mbeans'].index(i)+1
      if res['solr-mbeans'][index]['updateHandler']['stats']['docsPending']:
        return True
  return False


def forceCommit(URL='http://localhost:8983/solr/update?commit=true'):
  r = requests.get(URL)

def main():
  #1. Check if leader: if not, return
  #2. Check if docsPending: if yes, call commit and poll until docsPending=0
  #3. Call boto.ec2 to creat_snapshot()

  logfmt = '%(levelname)s\t%(app_name)s: %(message)s'
  datefmt= '%m/%d/%Y %H:%M:%S'
  logger.addFilter(AppFilter())
  formatter = logging.Formatter(fmt=logfmt,datefmt=datefmt)
  logging.root.setLevel(logging.INFO)
  ch = logging.StreamHandler() #console handler
  ch.setFormatter(formatter)<commit_msg>Add shardId snapshot tagging<commit_after>#!/usr/bin/env python

import os,sys
import time
import re
import subprocess

import json
import requests
import logging
import logging.handlers

import boto
import boto.ec2

sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
from components import utils

IP_REGEX = r'\b(?:\d{1,3}\.){3}\d{1,3}\b'
logger = logging.getLogger(__name__)

class AppFilter(logging.Filter):
  def filter(self, record):
    record.app_name = 'backup-daily'
    return True

def getDockerContainerIP():
  cmd = ['docker','inspect','solr']
  P = subprocess.Popen(cmd,stdout=subprocess.PIPE)
  P.wait()
  lines = json.loads(P.stdout.read())
  ip = lines[0]['NetworkSettings']['IPAddress']
  return ip

def isLeader(URL='http://localhost:8983/solr/admin/collections?action=CLUSTERSTATUS&wt=json'):
  r = requests.get(URL)
  res = json.loads(r.content)

  leaders = []
  for shard in res['cluster']['collections']['collection1']['shards'].values():
    for replica in shard['replicas'].values():
      if 'leader' in replica and replica['leader']:
        ip = re.search(IP_REGEX,replica['base_url']).group()
        leaders.append(ip)

  if getDockerContainerIP() in leaders:
    return True
  return False

def createSnapshot():
  c = utils.connect(boto.ec2.connection.EC2Connection)
  this_instance = utils.get_this_instance()
  volume = [v for v in c.get_all_volumes() if v.attach_data.instance_id == this_instance.id and 'solr-data-volume' in v.tags.get('Name','']
  if len(volume) != 1:
    logger.critical('Wrong number of EBS volumes attached to instance %s' % this_instance.id)
    sys.exit(1)
  s = volume[0].create_snapshot(description='%s-%s-snapshot' % (volume[0].tags['Name'],volume[0].tags['shardId']) )
  s.add_tag(key='shardId',value=volume[0].tags['shardId'])


def isActive(URL='http://localhost:8983/solr/collection1/admin/mbeans?stats=true&wt=json'):
  r = requests.get(URL)
  res = json.loads(r.content)

  for i in res['solr-mbeans']:
    if i.upper() == 'UPDATEHANDLER':
      index = res['solr-mbeans'].index(i)+1
      if res['solr-mbeans'][index]['updateHandler']['stats']['docsPending']:
        return True
  return False


def forceCommit(URL='http://localhost:8983/solr/update?commit=true'):
  r = requests.get(URL)

def main():
  #1. Check if leader: if not, return
  #2. Check if docsPending: if yes, call commit and poll until docsPending=0
  #3. Call boto.ec2 to creat_snapshot()

  logfmt = '%(levelname)s\t%(app_name)s: %(message)s'
  datefmt= '%m/%d/%Y %H:%M:%S'
  logger.addFilter(AppFilter())
  formatter = logging.Formatter(fmt=logfmt,datefmt=datefmt)
  logging.root.setLevel(logging.INFO)
  ch = logging.StreamHandler() #console handler
  ch.setFormatter(formatter)
{'fraction_non_alphanumeric': 0.16257896692679302, 'fraction_numerical': 0.01244890375325158, 'mean_word_length': 4.7510683760683765, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7277303', 'n_tokens_mistral': 1913, 'n_tokens_neox': 1891, 'n_words': 376}