<commit_before>import paramiko, logging, socket, time, re, datetime
import sys

python3_usage=True
if sys.version_info[0] < 3:
    python3_usage=False

#==================some variables
line_break = "\r"
ios_any_cli_length = "terminal length 0"
vrp_cli_length = "screen-length 0 temporary"
junos_cli_length = "set cli screen-length 0"
nokia_sr_os_cli_length ="environment no more"
cli_prompt = ("#", ">")
initial_wait_time = 2 
#==================================
def get_command_results(channel, hostname):
	## http://joelinoff.com/blog/?p=905
	maxseconds = 30
	bufsize = 65500
	# Poll until completion or timeout
	# Note that we cannot directly use the stdout file descriptor
	# because it stalls at 64K bytes (65536).
	input_idx = 0
	timeout_flag = False
	start = datetime.datetime.now()
	start_secs = time.mktime(start.timetuple())
	output = ''
	channel.setblocking(0)
	while not channel.exit_status_ready():
		if channel.recv_ready():
			data = channel.recv(bufsize).decode('ascii')
			output += data
		if channel.exit_status_ready():
			break
		# Timeout check
		now = datetime.datetime.now()
		now_secs = time.mktime(now.timetuple())
		et_secs = now_secs - start_secs
		if et_secs > maxseconds:
			timeout_flag = True
			break
		rbuffer = output.rstrip(' ')
		if rbuffer.endswith(hostname+">") or rbuffer.endswith(hostname+"#") or rbuffer.endswith(hostname):
			break
		time.sleep(0.100)
	return output
 
def send_command_and_get_response(channel, cmd, hostname):
	if not cmd.endswith("\n"):
		channel.send(cmd+"\n")
	else:
		channel.send(cmd)
	results = get_command_results(channel, hostname)
	try:
		results = results.split(hostname)[-0] # at the end of the output, an empty line with router name comes, remove it 
		results = results.split(cmd)[-1] #router returns the first command that is send, so split and do not display the command that is sent
		if results[-1:] in ["<","["]: #if router is huawei before router name there might be a string  < or [  e.g. <nw_rt_...>
			results = results[:-1]
		return results
	except:
		logging.warning("something went wrong when trying to remove hostname from the output get with command %s" % cmd)<commit_msg>Add files via upload<commit_after>import paramiko, logging, socket, time, re, datetime
import sys

python3_usage=True
if sys.version_info[0] < 3:
    python3_usage=False

#==================some variables
line_break = "\r"
ios_any_cli_length = "terminal length 0"
vrp_cli_length = "screen-length 0 temporary"
junos_cli_length = "set cli screen-length 0"
nokia_sr_os_cli_length ="environment no more"
cli_prompt = ("#", ">")
initial_wait_time = 2 
MAX_BUFFER = 65535
#==================================
def get_command_results(channel, hostname):
	## http://joelinoff.com/blog/?p=905
	maxseconds = 30
	# Poll until completion or timeout
	# Note that we cannot directly use the stdout file descriptor
	# because it stalls at 64K bytes (65536).
	start = datetime.datetime.now()
	start_secs = time.mktime(start.timetuple())
	output = ''
	channel.setblocking(0)
	while not channel.exit_status_ready():
		if channel.recv_ready():
			data = channel.recv(MAX_BUFFER).decode('ascii')
			output += data
		if channel.exit_status_ready():
			break
		# Timeout check
		now = datetime.datetime.now()
		now_secs = time.mktime(now.timetuple())
		et_secs = now_secs - start_secs
		if et_secs > maxseconds:
			break
		rbuffer = output.rstrip(' ')
		if rbuffer.endswith(hostname+">") or rbuffer.endswith(hostname+"#") or rbuffer.endswith(hostname):
			break
		time.sleep(0.100)
	return output
 
def send_command_and_get_response(channel, cmd, hostname):
	if not cmd.endswith("\n"):
		channel.send(cmd+"\n")
	else:
		channel.send(cmd)
	results = get_command_results(channel, hostname)
	try:
		results = results.split(hostname)[-0] # at the end of the output, an empty line with router name comes, remove it 
		results = results.split(cmd)[-1] #router returns the first command that is send, so split and do not display the command that is sent
		if results[-1:] in ["<","["]: #if router is huawei before router name there might be a string  < or [  e.g. <nw_rt_...>
			results = results[:-1]
		return results
	except:
		logging.warning("something went wrong when trying to remove hostname from the output get with command %s" % cmd)
{'fraction_non_alphanumeric': 0.1364700374531835, 'fraction_numerical': 0.016151685393258428, 'mean_word_length': 4.659602649006622, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2607952', 'n_tokens_mistral': 1521, 'n_tokens_neox': 1394, 'n_words': 414}