<commit_before>#===================================================================================================
#  C L A S S
#===================================================================================================
import os, subprocess, re, signal, sys, MySQLdb, json
from datetime import datetime
import siteReadiness

class Alarm(Exception):
    pass

def alarm_handler(signum, frame):
    raise Alarm

class SiteReadinessHandler:
    def __init__(self):
        self.siteReadiness = {}
    
    def extractReadinessData(self):
        self.getWaitingRoomData()
        self.getDeadSitesData()

    def getDeadSitesData(self):
        webServer = 'http://dashb-ssb.cern.ch/dashboard/request.py/getplotdata?'
        args = 'columnid=199&time=2184&dateFrom=&dateTo=&sites=all&clouds=undefined&batch=1'

        url = '"' + webServer + args + '"'
        cmd = 'curl -k -H "Accept: application/json" ' + url
        print ' Access DeadSites: ' + cmd

        process = subprocess.Popen(cmd,stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE,shell=True)
        signal.signal(signal.SIGALRM, alarm_handler)
        signal.alarm(10*60)  # 10 minutes
        try:
            mystring, error = process.communicate()
            signal.alarm(0)
        except Alarm:
            print " Oops, taking too long!"
            raise Exception(" FATAL -- Call to DeadSites timed out, stopping")

        if process.returncode != 0:
            print " Received non-zero exit status: " + str(process.returncode)
            raise Exception(" FATAL -- Call to DeadSites failed, stopping")

        dataJson = json.loads(mystring)
        data = dataJson["csvdata"]
        for item in data:
            site = item['VOName']
            status = item['Status']
            timest = datetime.strptime(item['Time'],"%Y-%m-%dT%H:%M:%S").date()
            
            if site not in self.siteReadiness:
                self.siteReadiness[site] = siteReadiness.SiteReadiness(site)
            self.siteReadiness[site].update(0,status,timest)

        forced = 'T2_TW_Taiwan'
        if forced not in self.siteReadiness:
            self.siteReadiness[forced] = siteReadiness.SiteReadiness(forced)
        self.siteReadiness[forced].update(0,'in',datetime.strptime('2014-11-03',"%Y-%m-%d").date())
        
    def getWaitingRoomData(self):
        webServer = 'http://dashb-ssb.cern.ch/dashboard/request.py/getplotdata?'
        args = 'columnid=153&time=2184&dateFrom=&dateTo=&sites=all&clouds=undefined&batch=1'

        url = '"' + webServer + args + '"'
        cmd = 'curl -k -H "Accept: application/json" ' + url
        print ' Access SiteWaitingRoom: ' + cmd

        process = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE,shell=True)
        signal.signal(signal.SIGALRM, alarm_handler)
        signal.alarm(10*60)  # 10 minutes
        try:
            mystring, error = process.communicate()
            signal.alarm(0)
        except Alarm:
            print " Oops, taking too long!"
            raise Exception(" FATAL -- Call to SiteWaitingRoom timed out, stopping")
<commit_msg>commented out special part for taiwan site<commit_after>#===================================================================================================
#  C L A S S
#===================================================================================================
import os, subprocess, re, signal, sys, MySQLdb, json
from datetime import datetime
import siteReadiness

class Alarm(Exception):
    pass

def alarm_handler(signum, frame):
    raise Alarm

class SiteReadinessHandler:
    def __init__(self):
        self.siteReadiness = {}
    
    def extractReadinessData(self):
        self.getWaitingRoomData()
        self.getDeadSitesData()

    def getDeadSitesData(self):
        webServer = 'http://dashb-ssb.cern.ch/dashboard/request.py/getplotdata?'
        args = 'columnid=199&time=2184&dateFrom=&dateTo=&sites=all&clouds=undefined&batch=1'

        url = '"' + webServer + args + '"'
        cmd = 'curl -k -H "Accept: application/json" ' + url
        print ' Access DeadSites: ' + cmd

        process = subprocess.Popen(cmd,stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE,shell=True)
        signal.signal(signal.SIGALRM, alarm_handler)
        signal.alarm(10*60)  # 10 minutes
        try:
            mystring, error = process.communicate()
            signal.alarm(0)
        except Alarm:
            print " Oops, taking too long!"
            raise Exception(" FATAL -- Call to DeadSites timed out, stopping")

        if process.returncode != 0:
            print " Received non-zero exit status: " + str(process.returncode)
            raise Exception(" FATAL -- Call to DeadSites failed, stopping")

        dataJson = json.loads(mystring)
        data = dataJson["csvdata"]
        for item in data:
            site = item['VOName']
            status = item['Status']
            timest = datetime.strptime(item['Time'],"%Y-%m-%dT%H:%M:%S").date()
            
            if site not in self.siteReadiness:
                self.siteReadiness[site] = siteReadiness.SiteReadiness(site)
            self.siteReadiness[site].update(0,status,timest)

        #forced = 'T2_TW_Taiwan'
        #if forced not in self.siteReadiness:
        #    self.siteReadiness[forced] = siteReadiness.SiteReadiness(forced)
        #self.siteReadiness[forced].update(0,'in',datetime.strptime('2014-11-03',"%Y-%m-%d").date())
        
    def getWaitingRoomData(self):
        webServer = 'http://dashb-ssb.cern.ch/dashboard/request.py/getplotdata?'
        args = 'columnid=153&time=2184&dateFrom=&dateTo=&sites=all&clouds=undefined&batch=1'

        url = '"' + webServer + args + '"'
        cmd = 'curl -k -H "Accept: application/json" ' + url
        print ' Access SiteWaitingRoom: ' + cmd

        process = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE,shell=True)
        signal.signal(signal.SIGALRM, alarm_handler)
        signal.alarm(10*60)  # 10 minutes
        try:
            mystring, error = process.communicate()
            signal.alarm(0)
        except Alarm:
            print " Oops, taking too long!"
            raise Exception(" FATAL -- Call to SiteWaitingRoom timed out, stopping")

{'fraction_non_alphanumeric': 0.17564365818985941, 'fraction_numerical': 0.013268046122255567, 'mean_word_length': 2.6707246376811593, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3304222', 'n_tokens_mistral': 1921, 'n_tokens_neox': 1809, 'n_words': 372}