<commit_before>#!/usr/bin/env python

# tile-generator
#
# Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import, division, print_function
import sys
import yaml
import json
import requests
import time
try:
	# Python 3
	from urllib.parse import urlparse
except ImportError:
	# Python 2
	from urlparse import urlparse
import subprocess
import pty
import os
import select

requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning)

# This function assumes that we are executing from within a concourse
# pool-resource repository, where the claimed PCF instance metadata
# is available in a file named './metadata'
#
def get_credentials():
	try:
		with open('metadata') as cred_file:
			creds = yaml.safe_load(cred_file)
			creds['opsmgr']
			creds['opsmgr']['url']
			creds['opsmgr']['username']
			creds['opsmgr']['password']
	except KeyError as e:
		print('metadata file is missing a value:', e.message, file=sys.stderr)
		sys.exit(1)
	except IOError as e:
		print('Not a Concourse PCF pool resource.', file=sys.stderr)
		print('Execute this from within the pool repository root, after a successful claim/acquire.', file=sys.stderr)
		sys.exit(1)
	return creds

class auth(requests.auth.AuthBase):

	def __init__(self, creds):
		self.creds = creds

	def __call__(self, request):
		url = self.creds.get('opsmgr').get('url') + '/uaa/oauth/token'
		username = self.creds.get('opsmgr').get('username')
		password = self.creds.get('opsmgr').get('password')
		headers = { 'Accept': 'application/json' }
		data = {
			'grant_type': 'password',
			'client_id': 'opsman',
			'client_secret': '',
			'username': username,
			'password': password,
			'response_type': 'token',
		}
		response = requests.post(url, data=data, verify=False, headers=headers)
		if response.status_code != requests.codes.ok:
			return requests.auth.HTTPBasicAuth(username, password)(request)
		response = response.json()
		access_token = response.get('access_token')
		token_type = response.get('token_type')
		request.headers['Authorization'] = token_type + ' ' + access_token
		return request

def get(url, stream=False, check=True):
	creds = get_credentials()
	url = creds.get('opsmgr').get('url') + url
	headers = { 'Accept': 'application/json' }
	response = requests.get(url, auth=auth(creds), verify=False, headers=headers, stream=stream)
	check_response(response, check=check)
	return response

def put(url, payload):
	creds = get_credentials()
	url = creds.get('opsmgr').get('url') + url
	response = requests.put(url, auth=auth(creds), verify=False, data=payload)
	check_response(response)
	return response

def put_json(url, payload):
	creds = get_credentials()
	url = creds.get('opsmgr').get('url') + url
	response = requests.put(url, auth=auth(creds), verify=False, json=payload)
	check_response(response)
	return response

def post(url, payload, check=True):
	creds = get_credentials()
	url = creds.get('opsmgr').get('url') + url
	response = requests.post(url, auth=auth(creds), verify=False, data=payload)
	check_response(response, check)
	return response

def post_yaml(url, filename, payload):
	creds = get_credentials()
	url = creds.get('opsmgr').get('url') + url
	files = { filename: yaml.safe_dump(payload) }
	response = requests.post(url, auth=auth(creds), verify=False, files=files)
	check_response(response)
	return response

def upload(url, filename):
	creds = get_credentials()
	url = creds.get('opsmgr').get('url') + url
	files = { 'product[file]': open(filename, 'rb') }
	response = requests.post(url, auth=auth(creds), verify=False, files=files)
	check_response(response)
	return response

def delete(url, check=True):
	creds = get_credentials()
	url = creds.get('opsmgr').get('url') + url
	response = requests.delete(url, auth=auth(creds), verify=False)
	check_response(response, check=check)
	return response

def check_response(response, check=True):
	if check and response.status_code != requests.codes.ok:
		print('-', response.status_code, response.request.url, file=sys.stderr)
		try:
			errors = response.json()["errors"]
			print('- '+('\n- '.join(json.dumps(errors, indent=4).splitlines())), file=sys.stderr)
		except:
			print(response.text, file=sys.stderr)
		sys.exit(1)

def ssh(commands = []):
	creds = get_credentials()
	url = creds.get('opsmgr').get('url')
	host = urlparse(url).hostname
	command = [
		'ssh',
		'-q',
		'-o', 'UserKnownHostsFile=/dev/null',
		'-o', 'StrictHostKeyChecking=no',
		'ubuntu@' + host
	]
	commands += [ 'stty -echo' if len(commands) == 0 else 'exit' ]
	commands = [ creds.get('opsmgr').get('password') ] + commands
	pid, tty = pty.fork()
	if pid == 0:
		try:
			subprocess.check_call(command)
			sys.exit(0)
		except subprocess.CalledProcessError as error:
			print('Command failed with exit code', error.returncode)
			print(error.output)
			sys.exit(error.returncode)
	else:
		output = os.read(tty, 4096)
		sys.stdout.write(output)
		sys.stdout.flush()
		while True:
			if len(commands) > 0:
				wlist = [ tty ]
			else:
				wlist = []
			rlist, wlist, xlist = select.select([sys.stdin.fileno(), tty], wlist, [])
			if sys.stdin.fileno() in rlist:
				input = os.read(sys.stdin.fileno(), 1024)
				if len(input) == 0:
					os.close(tty)
					break
				else:
					os.write(tty, input)
					os.fsync(tty)
			elif tty in rlist:
				output = os.read(tty, 1024)
				if len(output) == 0:
					break
				sys.stdout.write(output)
				sys.stdout.flush()
			elif tty in wlist:
				os.write(tty, commands[0] + '\n')
				commands = commands[1:]

def get_products():
	available_products = get('/api/products').json()
	installed_products = get('/api/installation_settings').json()['products']
	for product in available_products:
		installed = [ p for p in installed_products if p['identifier'] == product['name'] and p['product_version'] == product['product_version'] ]
		product['installed'] = len(installed) > 0
		if product['installed']:
			product['guid'] = installed[0]['guid']
	return available_products

def get_version():
	# 1.7 and 1.8 have version in the diagnostic report.
	response = get('/api/v0/diagnostic_report', check=False)
	if response.status_code == requests.codes.ok:
		diag = response.json()
		version = diag['versions']['release_version']
		return [ int(x) for x in version.split('.') ]
	# 1.6 (and maybe earlier?) has the version in the p-bosh (Ops Manager Director) product.
	products = get('/api/products').json()
	for product in products:
		if product['name'] == 'p-bosh':
			version = product['product_version']
			return [ int(x) for x in version.split('.') ]
	print('Error: could not determine Ops Manager version.', file=sys.stderr)
	sys.exit(1)<commit_msg>Standardize prompt, make it indicate the opsmgr host we are in<commit_after>#!/usr/bin/env python

# tile-generator
#
# Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import, division, print_function
import sys
import yaml
import json
import requests
import time
try:
	# Python 3
	from urllib.parse import urlparse
except ImportError:
	# Python 2
	from urlparse import urlparse
import subprocess
import pty
import os
import select

requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning)

# This function assumes that we are executing from within a concourse
# pool-resource repository, where the claimed PCF instance metadata
# is available in a file named './metadata'
#
def get_credentials():
	try:
		with open('metadata') as cred_file:
			creds = yaml.safe_load(cred_file)
			creds['opsmgr']
			creds['opsmgr']['url']
			creds['opsmgr']['username']
			creds['opsmgr']['password']
	except KeyError as e:
		print('metadata file is missing a value:', e.message, file=sys.stderr)
		sys.exit(1)
	except IOError as e:
		print('Not a Concourse PCF pool resource.', file=sys.stderr)
		print('Execute this from within the pool repository root, after a successful claim/acquire.', file=sys.stderr)
		sys.exit(1)
	return creds

class auth(requests.auth.AuthBase):

	def __init__(self, creds):
		self.creds = creds

	def __call__(self, request):
		url = self.creds.get('opsmgr').get('url') + '/uaa/oauth/token'
		username = self.creds.get('opsmgr').get('username')
		password = self.creds.get('opsmgr').get('password')
		headers = { 'Accept': 'application/json' }
		data = {
			'grant_type': 'password',
			'client_id': 'opsman',
			'client_secret': '',
			'username': username,
			'password': password,
			'response_type': 'token',
		}
		response = requests.post(url, data=data, verify=False, headers=headers)
		if response.status_code != requests.codes.ok:
			return requests.auth.HTTPBasicAuth(username, password)(request)
		response = response.json()
		access_token = response.get('access_token')
		token_type = response.get('token_type')
		request.headers['Authorization'] = token_type + ' ' + access_token
		return request

def get(url, stream=False, check=True):
	creds = get_credentials()
	url = creds.get('opsmgr').get('url') + url
	headers = { 'Accept': 'application/json' }
	response = requests.get(url, auth=auth(creds), verify=False, headers=headers, stream=stream)
	check_response(response, check=check)
	return response

def put(url, payload):
	creds = get_credentials()
	url = creds.get('opsmgr').get('url') + url
	response = requests.put(url, auth=auth(creds), verify=False, data=payload)
	check_response(response)
	return response

def put_json(url, payload):
	creds = get_credentials()
	url = creds.get('opsmgr').get('url') + url
	response = requests.put(url, auth=auth(creds), verify=False, json=payload)
	check_response(response)
	return response

def post(url, payload, check=True):
	creds = get_credentials()
	url = creds.get('opsmgr').get('url') + url
	response = requests.post(url, auth=auth(creds), verify=False, data=payload)
	check_response(response, check)
	return response

def post_yaml(url, filename, payload):
	creds = get_credentials()
	url = creds.get('opsmgr').get('url') + url
	files = { filename: yaml.safe_dump(payload) }
	response = requests.post(url, auth=auth(creds), verify=False, files=files)
	check_response(response)
	return response

def upload(url, filename):
	creds = get_credentials()
	url = creds.get('opsmgr').get('url') + url
	files = { 'product[file]': open(filename, 'rb') }
	response = requests.post(url, auth=auth(creds), verify=False, files=files)
	check_response(response)
	return response

def delete(url, check=True):
	creds = get_credentials()
	url = creds.get('opsmgr').get('url') + url
	response = requests.delete(url, auth=auth(creds), verify=False)
	check_response(response, check=check)
	return response

def check_response(response, check=True):
	if check and response.status_code != requests.codes.ok:
		print('-', response.status_code, response.request.url, file=sys.stderr)
		try:
			errors = response.json()["errors"]
			print('- '+('\n- '.join(json.dumps(errors, indent=4).splitlines())), file=sys.stderr)
		except:
			print(response.text, file=sys.stderr)
		sys.exit(1)

def ssh_eat_output(output):
	sys.stdout.write(output)
	sys.stdout.flush()

def ssh(commands = []):
	creds = get_credentials()
	url = creds.get('opsmgr').get('url')
	host = urlparse(url).hostname
	command = [
		'ssh',
		'-q',
		'-o', 'UserKnownHostsFile=/dev/null',
		'-o', 'StrictHostKeyChecking=no',
		'ubuntu@' + host
	]
	prompt = host.split('.',1)[0]
	commands += [ 'stty -echo' if len(commands) == 0 else 'exit' ]
	commands = [ 'export PS1="' + prompt + '$ "' ] + commands
	commands = [ creds.get('opsmgr').get('password') ] + commands
	pid, tty = pty.fork()
	if pid == 0:
		try:
			subprocess.check_call(command)
			sys.exit(0)
		except subprocess.CalledProcessError as error:
			print('Command failed with exit code', error.returncode)
			print(error.output)
			sys.exit(error.returncode)
	else:
		output = os.read(tty, 4096)
		sys.stdout.write(output)
		sys.stdout.flush()
		while True:
			if len(commands) > 0:
				wlist = [ tty ]
			else:
				wlist = []
			rlist, wlist, xlist = select.select([sys.stdin.fileno(), tty], wlist, [])
			if sys.stdin.fileno() in rlist:
				input = os.read(sys.stdin.fileno(), 1024)
				if len(input) == 0:
					os.close(tty)
					break
				else:
					os.write(tty, input)
					os.fsync(tty)
			elif tty in rlist:
				output = os.read(tty, 1024)
				if len(output) == 0:
					break
				ssh_eat_output(output)
			elif tty in wlist:
				os.write(tty, commands[0] + '\n')
				commands = commands[1:]

def get_products():
	available_products = get('/api/products').json()
	installed_products = get('/api/installation_settings').json()['products']
	for product in available_products:
		installed = [ p for p in installed_products if p['identifier'] == product['name'] and p['product_version'] == product['product_version'] ]
		product['installed'] = len(installed) > 0
		if product['installed']:
			product['guid'] = installed[0]['guid']
	return available_products

def get_version():
	# 1.7 and 1.8 have version in the diagnostic report.
	response = get('/api/v0/diagnostic_report', check=False)
	if response.status_code == requests.codes.ok:
		diag = response.json()
		version = diag['versions']['release_version']
		return [ int(x) for x in version.split('.') ]
	# 1.6 (and maybe earlier?) has the version in the p-bosh (Ops Manager Director) product.
	products = get('/api/products').json()
	for product in products:
		if product['name'] == 'p-bosh':
			version = product['product_version']
			return [ int(x) for x in version.split('.') ]
	print('Error: could not determine Ops Manager version.', file=sys.stderr)
	sys.exit(1)
{'fraction_non_alphanumeric': 0.14422619452185143, 'fraction_numerical': 0.0064568748725616805, 'mean_word_length': 4.709740007760963, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '206535', 'n_tokens_mistral': 5207, 'n_tokens_neox': 4837, 'n_words': 1261}