<commit_before>#!/usr/bin/python -u
###
# This file is part of Arduino Ciao
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
# Copyright 2015 Arduino Srl (http://www.arduino.org/)
#
# authors:
# _giuseppe[at]arduino[dot]org
#
# edited 06 Apr 2016 by <EMAIL>
#
###

import io, os, sys, signal, re
import logging, json
from threading import Thread
import time
import atexit

import settings
from utils import *
from ciaoconnector import CiaoConnector
import ciaoserver

#function to handle OS signals
def signal_handler(signum, frame):
	global logger
	global keepcycling
	logger.info("Received signal %d" % signum)
	keepcycling = False

def check_version(core_req_ver_num, core_ver_num):
	global logger
	operator = re.sub("[0-9.]", "", core_req_ver_num) #get operators
	core_req_ver_num = re.sub("[^0-9.]", "", core_req_ver_num) #get numbers
	#core_ver_num > core_req_ver_num => 1
	#core_ver_num = core_req_ver_num => 0
	#core_ver_num < core_req_ver_num => -1
	comp = __compare(core_ver_num, core_req_ver_num)
	#logger.info("VERSION CORE REQUIRED: %s - OPERATOR: %s - VERSION CORE CURRENT %s" % (core_req_ver_num, operator, core_ver_num))

	if operator == "=":
		if comp == 0:
			return True
		else:
			return False
	if operator == ">":
		if comp == 1:
			return True
		else:
			return False
	if operator == "<":
		if comp == -1:
			return True
		else:
			return False
	if operator == ">=":
		if comp >= 0:
			return True
		else:
			return False
	if operator == "<=":
		if comp <= 0:
			return True
		else:
			return False

def __compare(v1, v2):
	def normalize(v):
		return [int(x) for x in re.sub(r'(\.0+)*$','', v).split(".")]
	return cmp(normalize(v1), normalize(v2))

#opening logfile
logger = get_logger("ciao")

#loading configuration for connectors
settings.load_connectors(logger)

#check if connectors have been actually loaded
if not "connectors" in settings.conf or len(settings.conf["connectors"]) == 0:
	logger.critical("No connector enabled, exiting.")
	sys.exit(1)

#creating shared dictionary
shd = {}

#start ciaoserver thread (to interact with connectors)
server = Thread(name="server", target=ciaoserver.init, args=(settings.conf,shd,))
server.daemon = True
server.start()

#we start MANAGED connectors after ciaoserver (so they can register properly)
for connector, connector_conf in settings.conf['connectors'].items():
	core_version = settings.conf["core"]
	required_version = connector_conf['core'] if "core" in connector_conf else ">=0.0.0"

	if not ( check_version(required_version, core_version) ):
		logger.error("Required version of Ciao Core [%s] for the connector %s is not compatible with the working Core version [%s]" %(required_version, connector, core_version ))
	else:
		shd[connector] = CiaoConnector(connector, connector_conf)
		# connector must start after it has been added to shd,
		# it can register only if listed in shd
		shd[connector].start()

#TODO: maybe we can start another thread to control Ciao Core status
logger.warning(shd)
#variable to "mantain control" over while loop
keepcycling = True

#adding signals management
signal.signal(signal.SIGINT, signal_handler) #ctrl+c
signal.signal(signal.SIGHUP, signal_handler) #SIGHUP - 1
signal.signal(signal.SIGTERM, signal_handler) #SIGTERM - 15

#acquiring stream for receiving/sending command from MCU
if settings.use_fakestdin:
	print "Starting Ciao in DEBUG MODE"
	logger.debug("Starting Ciao in DEBUG MODE")

	handle = io.open(settings.basepath + "fake.stdin", "r+b")
else:
	#disable echo on terminal
	enable_echo(sys.stdin, False)

	#allow terminal echo to be enabled back when ciao exits
	atexit.register(enable_echo, sys.stdin.fileno(), True)
	handle = io.open(sys.stdin.fileno(), "rb")
	flush_terminal(sys.stdin)

while keepcycling:
	try:
		#reading from input device
		cmd = clean_command(handle.readline())
	except KeyboardInterrupt, e:
		logger.warning("SIGINT received")
	except IOError, e:
		logger.warning("Interrupted system call")
	else:
		if cmd:
			logger.debug("%s" % cmd)
			connector, action = is_valid_command(cmd)
			if connector == False:
				logger.warning("unknown command: %s" % cmd)
				out(-1, "unknown_command")
			elif connector == "ciao": #internal commands
				params = cmd.split(";",2)
				if len(params) != 3:
					out(-1, "unknown_command")
					continue
				if action == "r" and params[2] == "status": #read status
					out(1, "running")
				elif action == "w" and params[2] == "quit": #stop ciao
					out(1, "done")
					keepcycling = False
			elif not connector in settings.conf['connectors']:
				logger.warning("unknown connector: %s" % cmd)
				out(-1, "unknown_connector")
			elif not connector in shd:
				logger.warning("connector not runnable: %s" % cmd)
				out(-1, "connector_not_runnable")
			else:
				shd[connector].run(action, cmd)

		# the sleep is really useful to prevent ciao to "cap" all CPU
		# this could be increased/decreased (keep an eye on CPU usage)
		time.sleep(0.01)

#stopping connectors (managed)
for name, connector in shd.items():
	logger.info("Sending stop signal to %s" % name)
	connector.stop()

logger.info("Exiting")
sys.exit(0)
<commit_msg>better coded<commit_after>#!/usr/bin/python -u
###
# This file is part of Arduino Ciao
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
# Copyright 2015 Arduino Srl (http://www.arduino.org/)
#
# authors:
# _giuseppe[at]arduino[dot]org
#
# edited 06 Apr 2016 by <EMAIL>
#
###

import io, os, sys, signal, re
import logging, json
from threading import Thread
import time
import atexit

import settings
from utils import *
from ciaoconnector import CiaoConnector
import ciaoserver

#function to handle OS signals
def signal_handler(signum, frame):
	global logger
	global keepcycling
	logger.info("Received signal %d" % signum)
	keepcycling = False

def check_version(core_req_ver_num, core_ver_num):
	global logger
	operator = re.sub("[0-9.]", "", core_req_ver_num) #get operators
	core_req_ver_num = re.sub("[^0-9.]", "", core_req_ver_num) #get numbers
	#core_ver_num > core_req_ver_num => 1
	#core_ver_num = core_req_ver_num => 0
	#core_ver_num < core_req_ver_num => -1
	comp = __compare(core_ver_num, core_req_ver_num)
	#logger.info("VERSION CORE REQUIRED: %s - OPERATOR: %s - VERSION CORE CURRENT %s" % (core_req_ver_num, operator, core_ver_num))

	if operator == "=":
		if comp == 0:
			return True
		else:
			return False
	elif operator == ">":
		if comp == 1:
			return True
		else:
			return False
	elif operator == "<":
		if comp == -1:
			return True
		else:
			return False
	elif operator == ">=":
		if comp >= 0:
			return True
		else:
			return False
	elif operator == "<=":
		if comp <= 0:
			return True
		else:
			return False

def __compare(v1, v2):
	def normalize(v):
		return [int(x) for x in re.sub(r'(\.0+)*$','', v).split(".")]
	return cmp(normalize(v1), normalize(v2))

#opening logfile
logger = get_logger("ciao")

#loading configuration for connectors
settings.load_connectors(logger)

#check if connectors have been actually loaded
if not "connectors" in settings.conf or len(settings.conf["connectors"]) == 0:
	logger.critical("No connector enabled, exiting.")
	sys.exit(1)

#creating shared dictionary
shd = {}

#start ciaoserver thread (to interact with connectors)
server = Thread(name="server", target=ciaoserver.init, args=(settings.conf,shd,))
server.daemon = True
server.start()

#we start MANAGED connectors after ciaoserver (so they can register properly)
for connector, connector_conf in settings.conf['connectors'].items():
	core_version = settings.conf["core"]
	required_version = connector_conf['core'] if "core" in connector_conf else ">=0.0.0"

	if not ( check_version(required_version, core_version) ):
		logger.error("Required version of Ciao Core [%s] for the connector %s is not compatible with the working Core version [%s]" %(required_version, connector, core_version ))
	else:
		shd[connector] = CiaoConnector(connector, connector_conf)
		# connector must start after it has been added to shd,
		# it can register only if listed in shd
		shd[connector].start()

#TODO: maybe we can start another thread to control Ciao Core status
logger.warning(shd)
#variable to "mantain control" over while loop
keepcycling = True

#adding signals management
signal.signal(signal.SIGINT, signal_handler) #ctrl+c
signal.signal(signal.SIGHUP, signal_handler) #SIGHUP - 1
signal.signal(signal.SIGTERM, signal_handler) #SIGTERM - 15

#acquiring stream for receiving/sending command from MCU
if settings.use_fakestdin:
	print "Starting Ciao in DEBUG MODE"
	logger.debug("Starting Ciao in DEBUG MODE")

	handle = io.open(settings.basepath + "fake.stdin", "r+b")
else:
	#disable echo on terminal
	enable_echo(sys.stdin, False)

	#allow terminal echo to be enabled back when ciao exits
	atexit.register(enable_echo, sys.stdin.fileno(), True)
	handle = io.open(sys.stdin.fileno(), "rb")
	flush_terminal(sys.stdin)

while keepcycling:
	try:
		#reading from input device
		cmd = clean_command(handle.readline())
	except KeyboardInterrupt, e:
		logger.warning("SIGINT received")
	except IOError, e:
		logger.warning("Interrupted system call")
	else:
		if cmd:
			logger.debug("%s" % cmd)
			connector, action = is_valid_command(cmd)
			if connector == False:
				logger.warning("unknown command: %s" % cmd)
				out(-1, "unknown_command")
			elif connector == "ciao": #internal commands
				params = cmd.split(";",2)
				if len(params) != 3:
					out(-1, "unknown_command")
					continue
				if action == "r" and params[2] == "status": #read status
					out(1, "running")
				elif action == "w" and params[2] == "quit": #stop ciao
					out(1, "done")
					keepcycling = False
			elif not connector in settings.conf['connectors']:
				logger.warning("unknown connector: %s" % cmd)
				out(-1, "unknown_connector")
			elif not connector in shd:
				logger.warning("connector not runnable: %s" % cmd)
				out(-1, "connector_not_runnable")
			else:
				shd[connector].run(action, cmd)

		# the sleep is really useful to prevent ciao to "cap" all CPU
		# this could be increased/decreased (keep an eye on CPU usage)
		time.sleep(0.01)

#stopping connectors (managed)
for name, connector in shd.items():
	logger.info("Sending stop signal to %s" % name)
	connector.stop()

logger.info("Exiting")
sys.exit(0)

{'fraction_non_alphanumeric': 0.1110294717936158, 'fraction_numerical': 0.008000653114539963, 'mean_word_length': 4.568181818181818, 'pattern_counts': {'":': 16, '<': 13, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1941290', 'n_tokens_mistral': 4250, 'n_tokens_neox': 3863, 'n_words': 1359}