<commit_before>#===================================================================================================
#  C L A S S
#===================================================================================================
import os, re, sys, MySQLdb
import datetime
import siteStatus

class DbInfoHandler:
    def __init__(self):
        self.allSites = {}
        self.datasetIds = {}
        self.datasetSizes = {}
        self.datasetRanks = {}
        
        self.extractAllSites()

    def setDatasetRanks(self,dsetRanks):
        for dset in dsetRanks:
            self.datasetRanks[dset] = dsetRanks[dset][0]

    def extractAllSites(self):
        print ' Access site storage database to find all sites.'
        
        # AnalysisOps is GroupId=1
        sql = "select SiteName,SizeTb,Sites.Status,Sites.SiteId from Quotas,Sites "
        sql = sql + "where GroupId=1 and Quotas.SiteId=Sites.SiteId"
        
        results = self.dbExecSql(sql)
        for row in results:
            siteName = row[0]
            if "T1_" in siteName:
                continue
            siteSizeGb = float(row[1])*1000
            willBeUsed = int(row[2])
            siteId = int(row[3])
            self.allSites[siteName] = siteStatus.SiteStatus(siteName)
            self.allSites[siteName].setStatus(willBeUsed)
            self.allSites[siteName].setSize(siteSizeGb)
            self.allSites[siteName].setId(siteId)
            
        for site in sorted(self.allSites):
            if self.allSites[site].getStatus() == 0:
                print ' Site not active, status=%d  - %s'%(self.allSites[site].getStatus(),site)
            else:
                print ' Site --- active, status=%d  - %s'%(self.allSites[site].getStatus(),site)
  
    def enableSite(self,site,targetQuota):
        siteId = self.allSites[site].getId()
        
        print "\n !! Activating site " + site + " !!" 
        
            # Activate site
        if targetQuota  > self.allSites[site].getSize():
            print "  - set quota to " + str(targetQuota) + " -" 
            sql = 'update Quotas set SizeTb=' + str(targetQuota)
            sql = sql + ' where SiteId=' + str(siteId) + ' limit 1'
            self.dbExecSql(sql)<commit_msg>commented out special part for taiwan site<commit_after>#===================================================================================================
#  C L A S S
#===================================================================================================
import os, re, sys, MySQLdb
import datetime
import siteStatus

class DbInfoHandler:
    def __init__(self):
        self.allSites = {}
        self.datasetIds = {}
        self.datasetSizes = {}
        self.datasetRanks = {}
        
        self.extractAllSites()

    def setDatasetRanks(self,dsetRanks):
        for dset in dsetRanks:
            self.datasetRanks[dset] = dsetRanks[dset][0]

    def extractAllSites(self):
        print ' Access site storage database to find all sites.'
        
        # AnalysisOps is GroupId=1
        sql = "select SiteName,SizeTb,Sites.Status,Sites.SiteId from Quotas,Sites "
        sql = sql + "where GroupId=1 and Quotas.SiteId=Sites.SiteId"
        
        results = self.dbExecSql(sql)
        for row in results:
            siteName = row[0]
            if "T1_" in siteName:
                continue
            siteSizeGb = float(row[1])*1000
            willBeUsed = int(row[2])
            siteId = int(row[3])
            self.allSites[siteName] = siteStatus.SiteStatus(siteName)
            self.allSites[siteName].setStatus(willBeUsed)
            self.allSites[siteName].setSize(siteSizeGb)
            self.allSites[siteName].setId(siteId)
            
        for site in sorted(self.allSites):
            if self.allSites[site].getStatus() == 0:
                print ' Site not active, status=%d  - %s'%(self.allSites[site].getStatus(),site)
            else:
                print ' Site --- active, status=%d  - %s'%(self.allSites[site].getStatus(),site)
  
    def enableSite(self,site,targetQuota):
        if site == 'T2_TW_Taiwan':
            return

        siteId = self.allSites[site].getId()
        
        print "\n !! Activating site " + site + " !!" 
        
            # Activate site
        if targetQuota  > self.allSites[site].getSize():
            print "  - set quota to " + str(targetQuota) + " -" 
            sql = 'update Quotas set SizeTb=' + str(targetQuota)
            sql = sql + ' where SiteId=' + str(siteId) + ' limit 1'
            self.dbExecSql(sql)
{'fraction_non_alphanumeric': 0.19102844638949673, 'fraction_numerical': 0.006345733041575492, 'mean_word_length': 2.2258292166549047, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1673180', 'n_tokens_mistral': 1372, 'n_tokens_neox': 1321, 'n_words': 275}