<commit_before>#!/usr/bin/env python

import os
import ROOT as root

class site(object):
  def __init__(self,l):
    self.name=l[0]
    self.quota=float(l[1])
    self.used=float(l[2])
    self.toDelete=float(l[3])
    self.lastCp=float(l[4])

fileName = os.environ.get('SITE_MONITOR_FILE')
monitorDB = os.environ.get('MONITOR_DB')

siteInfos=[]

with open(fileName) as siteMonitorFile:
  for line in siteMonitorFile:
    l = line.strip().split(' ')
    siteInfos.append(site(l))

nSites = len(siteInfos)

# last cp fraction
hLow = root.TH1F("hLow","hLow",nSites+1,-1.5,nSites-0.5)
hMed = root.TH1F("hMed","hMed",nSites+1,-1.5,nSites-0.5)
hHigh = root.TH1F("hHigh","hHigh",nSites+1,-1.5,nSites-0.5)
medThreshold=0.5
highThreshold=0.7
xaxis = hHigh.GetXaxis()
hHigh.GetYaxis().SetTitle('last copy fraction')
hHigh.SetTitle('')
xaxis.SetBinLabel(1,' ')
hHigh.SetMaximum(1)

for i in range(nSites):
  s = siteInfos[i]
  if s.quota==0:
    # misread line
    continue
  xaxis.SetBinLabel(i+2,s.name)
  cpFr = s.lastCp/s.quota
  if cpFr<highThreshold:
    if cpFr<medThreshold:
      hLow.Fill(i,cpFr)
    else:
      hMed.Fill(i,cpFr)
  else:
    hHigh.Fill(i,cpFr)

c = root.TCanvas('c','c',1000,600)
c.SetBottomMargin(.3)
for hist,color in zip([hLow,hMed,hHigh],[8,5,2]):
  hist.SetFillColor(color)
  hist.SetLineColor(color)
hHigh.SetStats(0)
hHigh.Draw("hist")
hMed.Draw("hist same")
hLow.Draw("hist same")

c.SaveAs(monitorDB+'/lastCpFractionSites.png')


hLow.Reset()
hMed.Reset()
hHigh.Reset()
medThreshold=0.5
highThreshold=0.7
superHighThreshold=0.9
hHigh.GetYaxis().SetTitle('used fraction')
hHigh.SetMaximum(1)

for i in range(nSites):
  s = siteInfos[i]
  usedFr = s.used/s.quota
  if usedFr>superHighThreshold:
    hLow.Fill(i,usedFr)
  else:
    if usedFr<highThreshold:
      if usedFr<medThreshold:
        hLow.Fill(i,usedFr)
      else:
        hMed.Fill(i,usedFr)
    else:
      hHigh.Fill(i,usedFr)
<commit_msg>previous issue was in two places; fix applied<commit_after>#!/usr/bin/env python

import os
import ROOT as root

class site(object):
  def __init__(self,l):
    self.name=l[0]
    self.quota=float(l[1])
    self.used=float(l[2])
    self.toDelete=float(l[3])
    self.lastCp=float(l[4])

fileName = os.environ.get('SITE_MONITOR_FILE')
monitorDB = os.environ.get('MONITOR_DB')

siteInfos=[]

with open(fileName) as siteMonitorFile:
  for line in siteMonitorFile:
    l = line.strip().split(' ')
    siteInfos.append(site(l))

nSites = len(siteInfos)

# last cp fraction
hLow = root.TH1F("hLow","hLow",nSites+1,-1.5,nSites-0.5)
hMed = root.TH1F("hMed","hMed",nSites+1,-1.5,nSites-0.5)
hHigh = root.TH1F("hHigh","hHigh",nSites+1,-1.5,nSites-0.5)
medThreshold=0.5
highThreshold=0.7
xaxis = hHigh.GetXaxis()
hHigh.GetYaxis().SetTitle('last copy fraction')
hHigh.SetTitle('')
xaxis.SetBinLabel(1,' ')
hHigh.SetMaximum(1)

for i in range(nSites):
  s = siteInfos[i]
  if s.quota==0:
    # misread line
    continue
  xaxis.SetBinLabel(i+2,s.name)
  cpFr = s.lastCp/s.quota
  if cpFr<highThreshold:
    if cpFr<medThreshold:
      hLow.Fill(i,cpFr)
    else:
      hMed.Fill(i,cpFr)
  else:
    hHigh.Fill(i,cpFr)

c = root.TCanvas('c','c',1000,600)
c.SetBottomMargin(.3)
for hist,color in zip([hLow,hMed,hHigh],[8,5,2]):
  hist.SetFillColor(color)
  hist.SetLineColor(color)
hHigh.SetStats(0)
hHigh.Draw("hist")
hMed.Draw("hist same")
hLow.Draw("hist same")

c.SaveAs(monitorDB+'/lastCpFractionSites.png')


hLow.Reset()
hMed.Reset()
hHigh.Reset()
medThreshold=0.5
highThreshold=0.7
superHighThreshold=0.9
hHigh.GetYaxis().SetTitle('used fraction')
hHigh.SetMaximum(1)

for i in range(nSites):
  s = siteInfos[i]
  if s.quota==0:
    # misread line
    continue
  usedFr = s.used/s.quota
  if usedFr>superHighThreshold:
    hLow.Fill(i,usedFr)
  else:
    if usedFr<highThreshold:
      if usedFr<medThreshold:
        hLow.Fill(i,usedFr)
      else:
        hMed.Fill(i,usedFr)
    else:
      hHigh.Fill(i,usedFr)

{'fraction_non_alphanumeric': 0.16560833121666244, 'fraction_numerical': 0.025654051308102617, 'mean_word_length': 4.257676902536716, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5245402', 'n_tokens_mistral': 1702, 'n_tokens_neox': 1687, 'n_words': 258}