<commit_before>#!/usr/bin/python

import argparse
import pickle
import re
import socket
import struct
import sqlite3
import sys
import time
from calendar import timegm

def get_data_publish_stats(last_timestamp, db_file):
    # check in server.conf if CVMFS_STATISTICS_DB variable is set
    conn = sqlite3.connect(db_file) # change path
    c = conn.cursor()

    # order DESC by id ---> first element of the list has the last start_time
    c.execute('SELECT start_time, files_added, files_removed, files_changed \
                FROM publish_statistics \
                WHERE start_time > ("%s") \
                ORDER BY publish_id DESC' % last_timestamp)
    return c.fetchall() 


def get_data_gc_stats(last_timestamp, db_file):
    # check in server.conf if CVMFS_STATISTICS_DB variable is set
    conn = sqlite3.connect(db_file) # change path
    c = conn.cursor()

    # order DESC by id ---> first element of the list has the last start_time
    c.execute('SELECT start_time, n_preserved_catalogs, n_condemned_catalogs, n_condemned_objects, sz_condemned_bytes \
                FROM gc_statistics \
                WHERE start_time > ("%s") \
                ORDER BY gc_id DESC' % last_timestamp )
    return c.fetchall() 


def run(sock, db_file):
    tuples = ([])
    lines = []

    # the following file must have two timestamps (one per line):
    # last_start_time_sent_for_publish_statistics
    # last_start_time_sent_for_gc_statistics
    f=open("aux.txt", "r+") # todo: change path to `/var/spool/cvmfs/<REPO>/last_timestamp_sent`

    # read first two lines
    publish_timestamp = f.readline()[:-1] # delete the newline character
    gc_timestamp = f.readline()
    
    f.truncate() # delete file content
    f.seek(0)    # move file cursor

    # # for debug
    # print publish_timestamp
    # print gc_timestamp

    publish_stats = get_data_publish_stats(publish_timestamp, db_file)
    gc_stats = get_data_gc_stats(gc_timestamp, db_file)

    # # for debug
    # print "length stats publish=%d " % len(publish_stats)
    # print "length   stats    gc=%d " % len(gc_stats)

    if len(publish_stats) > 0:
        publish_timestamp = publish_stats[0][0]
        for x in xrange(0,len(publish_stats)):
            time_obj = time.strptime(publish_stats[x][0], "%Y-%m-%d %H:%M:%S")
            timestamp_epoch = timegm(time_obj)
            tuples.append(('cvmfs.publish.files_added', (timestamp_epoch, publish_stats[x][1])))
            tuples.append(('cvmfs.publish.files_removed', (timestamp_epoch, publish_stats[x][2])))
            tuples.append(('cvmfs.publish.files_changed', (timestamp_epoch, publish_stats[x][3])))
            # #just for DBG
            # lines.append("cvmfs.publish.files_added %s %s" % (publish_stats[x][1], publish_stats[x][0]))
            # lines.append("cvmfs.publish.files_removed %s %s" % (publish_stats[x][2], publish_stats[x][0]))
            # lines.append("cvmfs.publish.files_changed %s %s" % (publish_stats[x][3], publish_stats[x][0]))

    if len(gc_stats) > 0:
        gc_timestamp=gc_stats[0][0]
        for x in xrange(0,len(gc_stats)):
            time_obj = time.strptime(gc_stats[x][0], "%Y-%m-%d %H:%M:%S")
            timestamp_epoch = timegm(time_obj)
            tuples.append(('cvmfs.gc.n_preserved_catalogs', (timestamp_epoch, gc_stats[x][1])))
            tuples.append(('cvmfs.gc.n_condemned_catalogs', (timestamp_epoch, gc_stats[x][2])))
            tuples.append(('cvmfs.gc.n_condemned_objects', (timestamp_epoch, gc_stats[x][3])))
            tuples.append(('cvmfs.gc.sz_condemned_bytes', (timestamp_epoch, gc_stats[x][4])))
            # #just for DBG
            # lines.append("cvmfs.gc.n_preserved_catalogs %s %s" % (gc_stats[x][1], gc_stats[x][0]))
            # lines.append("cvmfs.gc.n_condemned_catalogs %s %s" % (gc_stats[x][2], gc_stats[x][0]))
            # lines.append("cvmfs.gc.n_condemned_objects %s %s" % (gc_stats[x][3], gc_stats[x][0]))<commit_msg>Send all publish basic metrics.<commit_after>#!/usr/bin/python

import argparse
import pickle
import re
import socket
import struct
import sqlite3
import sys
import time
from calendar import timegm

def get_data_publish_stats(last_timestamp, db_file):
    conn = sqlite3.connect(db_file)
    c = conn.cursor()

    # order DESC by id ---> first element of the list has the last start_time
    c.execute('SELECT start_time, files_added, files_removed, files_changed, duplicated_files, directories_added, \
                      directories_removed, directories_changed, sz_bytes_added, sz_bytes_removed, sz_bytes_uploaded \
                FROM publish_statistics \
                WHERE start_time > ("%s") \
                ORDER BY publish_id DESC' % last_timestamp)
    return c.fetchall() 


def get_data_gc_stats(last_timestamp, db_file):
    conn = sqlite3.connect(db_file)
    c = conn.cursor()
    c.execute('SELECT start_time, n_preserved_catalogs, n_condemned_catalogs, n_condemned_objects, sz_condemned_bytes \
                FROM gc_statistics \
                WHERE start_time > ("%s") \
                ORDER BY gc_id DESC' % last_timestamp )
    return c.fetchall() 


def run(sock, db_file):
    tuples = ([])
    lines = []

    # the following file must have two timestamps (one per line):
    # last_start_time_sent_for_publish_statistics
    # last_start_time_sent_for_gc_statistics
    f=open("aux.txt", "r+") # todo: change path to `/var/spool/cvmfs/<REPO>/last_timestamp_sent`

    # read first two lines
    publish_timestamp = f.readline()[:-1] # delete the newline character
    gc_timestamp = f.readline()
    
    f.truncate() # delete file content
    f.seek(0)    # move file cursor

    publish_stats = get_data_publish_stats(publish_timestamp, db_file)
    gc_stats = get_data_gc_stats(gc_timestamp, db_file)

    if len(publish_stats) > 0:
        publish_timestamp = publish_stats[0][0]
        for x in xrange(0,len(publish_stats)):
            time_obj = time.strptime(publish_stats[x][0], "%Y-%m-%d %H:%M:%S")
            timestamp_epoch = timegm(time_obj)
            tuples.append(('cvmfs.publish.files_added', (timestamp_epoch, publish_stats[x][1])))
            tuples.append(('cvmfs.publish.files_removed', (timestamp_epoch, publish_stats[x][2])))
            tuples.append(('cvmfs.publish.files_changed', (timestamp_epoch, publish_stats[x][3])))
            tuples.append(('cvmfs.publish.duplicated_files', (timestamp_epoch, publish_stats[x][4])))
            tuples.append(('cvmfs.publish.directories_added', (timestamp_epoch, publish_stats[x][5])))
            tuples.append(('cvmfs.publish.directories_removed', (timestamp_epoch, publish_stats[x][6])))
            tuples.append(('cvmfs.publish.directories_changed', (timestamp_epoch, publish_stats[x][7])))
            tuples.append(('cvmfs.publish.sz_bytes_added', (timestamp_epoch, publish_stats[x][8])))
            tuples.append(('cvmfs.publish.sz_bytes_removed', (timestamp_epoch, publish_stats[x][9])))
            tuples.append(('cvmfs.publish.sz_bytes_uploaded', (timestamp_epoch, publish_stats[x][10])))
            # #just for DBG
            # lines.append("cvmfs.publish.files_added %s %s" % (publish_stats[x][1], publish_stats[x][0]))
            # lines.append("cvmfs.publish.files_removed %s %s" % (publish_stats[x][2], publish_stats[x][0]))
            # lines.append("cvmfs.publish.files_changed %s %s" % (publish_stats[x][3], publish_stats[x][0]))
            # lines.append("cvmfs.publish.duplicated_files %s %s" % (publish_stats[x][4], publish_stats[x][0]))
            # lines.append("cvmfs.publish.directories_added %s %s" % (publish_stats[x][5], publish_stats[x][0]))
            # lines.append("cvmfs.publish.directories_removed %s %s" % (publish_stats[x][6], publish_stats[x][0]))
            # lines.append("cvmfs.publish.directories_changed %s %s" % (publish_stats[x][7], publish_stats[x][0]))
            # lines.append("cvmfs.publish.sz_bytes_added %s %s" % (publish_stats[x][8], publish_stats[x][0]))
            # lines.append("cvmfs.publish.sz_bytes_removed %s %s" % (publish_stats[x][9], publish_stats[x][0]))
            # lines.append("cvmfs.publish.sz_bytes_uploaded %s %s" % (publish_stats[x][10], publish_stats[x][0]))

    if len(gc_stats) > 0:
        gc_timestamp=gc_stats[0][0]
        for x in xrange(0,len(gc_stats)):
            time_obj = time.strptime(gc_stats[x][0], "%Y-%m-%d %H:%M:%S")
            timestamp_epoch = timegm(time_obj)
            tuples.append(('cvmfs.gc.n_preserved_catalogs', (timestamp_epoch, gc_stats[x][1])))
            tuples.append(('cvmfs.gc.n_condemned_catalogs', (timestamp_epoch, gc_stats[x][2])))
            tuples.append(('cvmfs.gc.n_condemned_objects', (timestamp_epoch, gc_stats[x][3])))
            tuples.append(('cvmfs.gc.sz_condemned_bytes', (timestamp_epoch, gc_stats[x][4])))
            # #just for DBG
            # lines.append("cvmfs.gc.n_preserved_catalogs %s %s" % (gc_stats[x][1], gc_stats[x][0]))
            # lines.append("cvmfs.gc.n_condemned_catalogs %s %s" % (gc_stats[x][2], gc_stats[x][0]))
            # lines.append("cvmfs.gc.n_condemned_objects %s %s" % (gc_stats[x][3], gc_stats[x][0]))
{'fraction_non_alphanumeric': 0.13985784581738656, 'fraction_numerical': 0.009950792782941498, 'mean_word_length': 3.399230399230399, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6507553', 'n_tokens_mistral': 3201, 'n_tokens_neox': 3165, 'n_words': 555}