<commit_before>#!/usr/bin/python3

'''
Monitor for Docklet
Description:Monitor system for docklet will collect data on resources usages and status of vnode 
            and phyiscal machines. And master can fetch these data and then show them on the web page.
            Besides, Monitor will also bill the vnodes according to their resources usage amount.

Design:Monitor mainly consists of three parts: Collectors, Master_Collector and Fetchers.
       1.Collectors will collect data every two seconds on each worker. And 'Container_Collector' will 
       collect data of containers(vnodes), while 'Collector' will collect data of physical machines.
       2.'Master_Collector' only runs on Master. It fetches the data on workers every two seconds by rpc 
       and stores them in the memory of Master.
       3.Fetchers are classes that Master will use them to fetch specific data in the memory and then show 
       them on the web. 'Container_Fetcher' is the class to fetch the containers data in 'monitor_vnodes', 
       while 'Fetcher' is the class to fetch the data of physical machines in 'monitor_hosts'.
'''


import subprocess,re,os,etcdlib,psutil,math,sys
import time,threading,json,traceback,platform
import env
from datetime import datetime

from model import db,VNode,History,User
from log import logger
from httplib2 import Http
from urllib.parse import urlencode

# billing parameters
a_cpu = 500         # seconds
b_mem = 1000000     # MB
c_disk = 4000       # MB

# major dict to store the monitoring data
# only use on Master
# monitor_hosts: use workers' ip addresses as first key.
# second key: cpuinfo,diskinfo,meminfo,osinfo,cpuconfig,running,containers,containerslist
# 1.cpuinfo stores the cpu usages data, and it has keys: user,system,idle,iowait
# 2.diskinfo stores the disks usages data, and it has keys: device,mountpoint,total,used,free,percent
# 3.meminfo stores the memory usages data, and it has keys: total,used,free,buffers,cached,percent
# 4.osinfo stores the information of operating system, 
# and it has keys: platform,system,node,release,version,machine,processor              
# 5.cpuconfig stores the information of processors, and it is a list, each element of list is a dict
# which stores the information of a processor, each element has key: processor,model name,<commit_msg>Change billing parameters<commit_after>#!/usr/bin/python3

'''
Monitor for Docklet
Description:Monitor system for docklet will collect data on resources usages and status of vnode 
            and phyiscal machines. And master can fetch these data and then show them on the web page.
            Besides, Monitor will also bill the vnodes according to their resources usage amount.

Design:Monitor mainly consists of three parts: Collectors, Master_Collector and Fetchers.
       1.Collectors will collect data every two seconds on each worker. And 'Container_Collector' will 
       collect data of containers(vnodes), while 'Collector' will collect data of physical machines.
       2.'Master_Collector' only runs on Master. It fetches the data on workers every two seconds by rpc 
       and stores them in the memory of Master.
       3.Fetchers are classes that Master will use them to fetch specific data in the memory and then show 
       them on the web. 'Container_Fetcher' is the class to fetch the containers data in 'monitor_vnodes', 
       while 'Fetcher' is the class to fetch the data of physical machines in 'monitor_hosts'.
'''


import subprocess,re,os,etcdlib,psutil,math,sys
import time,threading,json,traceback,platform
import env
from datetime import datetime

from model import db,VNode,History,User
from log import logger
from httplib2 import Http
from urllib.parse import urlencode

# billing parameters
a_cpu = 500         # seconds
b_mem = 2000000     # MB
c_disk = 4000       # MB

# major dict to store the monitoring data
# only use on Master
# monitor_hosts: use workers' ip addresses as first key.
# second key: cpuinfo,diskinfo,meminfo,osinfo,cpuconfig,running,containers,containerslist
# 1.cpuinfo stores the cpu usages data, and it has keys: user,system,idle,iowait
# 2.diskinfo stores the disks usages data, and it has keys: device,mountpoint,total,used,free,percent
# 3.meminfo stores the memory usages data, and it has keys: total,used,free,buffers,cached,percent
# 4.osinfo stores the information of operating system, 
# and it has keys: platform,system,node,release,version,machine,processor              
# 5.cpuconfig stores the information of processors, and it is a list, each element of list is a dict
# which stores the information of a processor, each element has key: processor,model name,
{'fraction_non_alphanumeric': 0.05615087869695671, 'fraction_numerical': 0.010287183883411917, 'mean_word_length': 4.179800221975583, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1403487', 'n_tokens_mistral': 1276, 'n_tokens_neox': 1217, 'n_words': 579}