<commit_before>#!/usr/bin/env python

# tile-generator
#
# Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import, division, print_function
import os
import sys
import yaml
import json
import time
import click
import subprocess

PATH = os.path.dirname(os.path.realpath(__file__))
sys.path.append(os.path.join(PATH, os.path.join('..', 'lib')))
from . import opsmgr
from . import erb

@click.group()
@click.option('-t', '--target')
@click.option('-n', '--non-interactive', is_flag=True)
def cli(target, non_interactive):
	opsmgr.get_credentials(target, non_interactive)

@cli.command('ssh')
@click.argument('argv', nargs=-1)
@click.option('--debug', '-d', is_flag=True)
def ssh_cmd(argv, debug=False):
	opsmgr.ssh(argv, debug=debug)

@cli.command('reboot')
@click.option('--yes-i-am-sure', '-y', is_flag=True)
def reboot_cmd(yes_i_am_sure=False):
	if not yes_i_am_sure:
		print('Rebooting in', end="")
		for i in range(10, 0, -1):
			sys.stdout.write(' ' +str(i))
			sys.stdout.flush()
			time.sleep(1)
		print()
	opsmgr.ssh(['sudo reboot now'], silent=True)
	time.sleep(10) # Allow system time to go down before we ping the API
	opsmgr.unlock()

@cli.command('unlock')
def unlock_cmd():
	opsmgr.unlock()

@cli.command('products')
def products_cmd():
	products = opsmgr.get_products()
	for product in products:
		print("-", product["name"], product["product_version"], "(installed)" if product["installed"] else "")

@cli.command('changes')
def changes_cmd():
	changes = opsmgr.get_changes()
	if changes is None:
		print('This command is only available for PCF 1.7 and beyond.', file=sys.stderr)
		sys.exit(1)
	for p in changes['product_changes']:
		print(p['action'], p['guid'])
		for e in p['errands']:
			print('-', e['name'])

@cli.command('is-available')
@click.argument('product')
@click.argument('version', None, required=False)
def is_available_cmd(product, version):
	products = opsmgr.get_products()
	matches = [ p for p in products if p['name'] == product and (version is None or p['product_version'] == version) ]
	if len(matches) < 1:
		print('No match found for product', product, 'version', version, file=sys.stderr)
		sys.exit(1)

@cli.command('is-installed')
@click.argument('product')
@click.argument('version', None, required=False)
def is_installed_cmd(product, version):
	products = opsmgr.get_products()
	matches = [ p for p in products if p['name'] == product and (version is None or p['product_version'] == version) and p['installed'] ]
	if len(matches) < 1:
		print('Product', product, 'version', version, 'is not installed', file=sys.stderr)
		sys.exit(1)

@cli.command('configure')
@click.argument('product')
@click.argument('properties_file', None, required=False)
@click.option('--strict', is_flag=True)
@click.option('--skip-validation', is_flag=True)
@click.option('-n', '--network')
def configure_cmd(product, properties_file, strict=False, skip_validation=False, network=None):
	properties = {}
	if properties_file is not None:
		with open(properties_file) as f:
			properties = yaml.safe_load(f)
	opsmgr.configure(product, properties, strict, skip_validation, network)

@cli.command('settings')
@click.argument('product', None, required=False)
def settings_cmd(product):
	settings = opsmgr.get('/api/installation_settings').json()
	if product is not None:
		settings = [ p for p in settings['products'] if p['identifier'] == product ]
		if len(settings) < 1:
			print('No settings found for product', product, file=sys.stderr)
			sys.exit(1)
		settings = settings[0]
	print(json.dumps(settings, indent=4))

@cli.command('cf-info')
def cf_info_cmd():
	cfinfo = opsmgr.get_cfinfo()
	for key in sorted(cfinfo):
		print('-', key + ':', cfinfo[key])

@cli.command('import')
@click.argument('tile')
def import_cmd(tile):
	opsmgr.upload('/api/products', tile)
	print('-', 'Tile', tile, 'imported successfully. You can run `pcf install` to load it.')

@cli.command('install')
@click.argument('product')
@click.argument('version')
def install_cmd(product, version):
	products = opsmgr.get_products()
	matches = [ p for p in products if p['name'] == product and p['installed'] ]
	if len(matches) < 1:
		payload = {
			'name': product,
			'product_version': version,
		}
		opsmgr.post('/api/installation_settings/products', payload)
	else:
		payload = {
			'to_version': version
		}
		response = opsmgr.put('/api/installation_settings/products/' + matches[0]['guid'], payload, check=False)
		if response.status_code == 422:
			errors = response.json()["errors"]
			for error in errors:
				if error.endswith(' is already in use.'):
					print('-','version already installed')
					return
		opsmgr.check_response(response)

@cli.command('uninstall')
@click.argument('product')
@click.argument('version', None, required=False)
def uninstall_cmd(product, version):
	products = opsmgr.get('/api/installation_settings/products').json()
	matches = [ p for p in products if p['type'] == product ]
	for match in matches:
		if version and 'product_version' in match:
			if match['product_version'] == version:
				opsmgr.delete('/api/installation_settings/products/' + match['guid'])
		else:
			opsmgr.delete('/api/installation_settings/products/' + match['guid'])

@cli.command('delete-unused-products')
def delete_unused_products_cmd():
	opsmgr.delete('/api/products')

@cli.command('backup')
@click.argument('backup_file')
def backup_cmd(backup_file):
	response = opsmgr.get('/api/installation_asset_collection', stream=True)
	with open(backup_file, 'wb') as f:
		for chunk in response.iter_content(1024):
			f.write(chunk)

@cli.command('restore')
@click.argument('backup_file')
def restore_cmd(backup_file):
	creds = get_credentials()
	with open(backup_file, 'rb') as f:
		payload = { 'installation[file]': f, 'password': creds['opsmgr']['password'] }
		opsmgr.post('/api/installation_asset_collection', f)

@cli.command('cleanup')
@click.argument('product')
def cleanup_cmd(product):
	#
	# Attempt 1 - Delete any uninstalled versions
	#
	products = opsmgr.get('/api/installation_settings/products').json()
	matches = [ p for p in products if p['type'] == product ]
	for match in matches:
		print('- attempting to delete', match['name'], file=sys.stderr)
		opsmgr.delete('/api/installation_settings/products/' + match['guid'])
	products = opsmgr.get('/api/installation_settings/products').json()
	matches = [ p for p in products if p['type'] == product ]
	if len(matches) < 1:
		sys.exit(0)
	if len(matches) > 1:
		print('- more than one match remains installed', file=sys.stderr)
		sys.exit(1)
	#
	# Attempt 2 - Uninstall deployed version
	#
	match = matches[0]
	print('- product was deployed, applying changes to uninstall it', file=sys.stderr)
	apply_changes_cmd()
	opsmgr.delete('/api/products')
	products = opsmgr.get('/api/installation_settings/products').json()
	matches = [ p for p in products if p['type'] == product ]
	if len(matches) < 1:
		sys.exit(0)
	#
	# Attempt 3 - Re-deploy with errands disabled, then uninstall
	#
	match = matches[0]
	print('- uninstall appears to have failed', file=sys.stderr)
	print('- re-deploying with disabled errands', file=sys.stderr)
	opsmgr.disable_errands(product)
	apply_changes_cmd()
	print('- uninstalling with disabled errands', file=sys.stderr)
	opsmgr.delete('/api/installation_settings/products/' + match['guid'])
	apply_changes_cmd()
	opsmgr.delete('/api/products')
	products = opsmgr.get('/api/installation_settings/products').json()
	matches = [ p for p in products if p['type'] == product ]
	if len(matches) > 0:
		print('- failed to uninstall', file=sys.stderr)
		sys.exit(1)

@cli.command('apply-changes')
@click.option('--product', help='product to select errands from. Only valid in combination with -deploy-errands or --delete-errands.')
@click.option('--deploy-errands', help='Comma separated list of errands to run after install/update. For example: "deploy-all,configure-broker"')
@click.option('--delete-errands', help='Comma separated list of errands to run before delete. For example: "pre_delete"')
def apply_changes_cmd(product, deploy_errands, delete_errands):
	body = None
	version = opsmgr.get_version()
	pre_1_10 = version[0] == 1 and version[1] < 10
	if pre_1_10:
		enabled_errands = []
		deploy_errand_list = None if deploy_errands is None else deploy_errands.split(',')
		delete_errand_list = None if delete_errands is None else delete_errands.split(',')
		changes = opsmgr.get_changes(product, deploy_errand_list, delete_errand_list)
		if changes is not None:
			if len(changes['product_changes']) == 0:
				print('Nothing to do', file=sys.stderr)
				return
			for p in changes['product_changes']:
				post_deploy = serialize_errands(p, 'post_deploy', 'post_deploy_errands')
				enabled_errands.extend(post_deploy)
				pre_delete = serialize_errands(p, 'pre_delete', 'pre_delete_errands')
				enabled_errands.extend(pre_delete)
		body = '&'.join(enabled_errands)
	in_progress = None
	install_id = None
	while install_id is None:
		if pre_1_10:
			response = opsmgr.post('/api/installation?ignore_warnings=true', body, check=False)
		else:
			response = opsmgr.post('/api/v0/installations?ignore_warnings=true', body, check=False)
		if response.status_code == 422 and "Install in progress" in response.json()["errors"]:
			if in_progress is None:
				print('Waiting for in-progress installation to complete', file=sys.stderr)
				in_progress = opsmgr.last_install()
			time.sleep(10)
			if opsmgr.install_exists(in_progress + 1):
				install_id = in_progress + 1
				break
			continue
		opsmgr.check_response(response)
		install = response.json()['install']<commit_msg>Distinguish 422 in exit status to enable conditional behavior in CI<commit_after>#!/usr/bin/env python

# tile-generator
#
# Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import, division, print_function
import os
import sys
import yaml
import json
import time
import click
import subprocess

PATH = os.path.dirname(os.path.realpath(__file__))
sys.path.append(os.path.join(PATH, os.path.join('..', 'lib')))
from . import opsmgr
from . import erb

@click.group()
@click.option('-t', '--target')
@click.option('-n', '--non-interactive', is_flag=True)
def cli(target, non_interactive):
	opsmgr.get_credentials(target, non_interactive)

@cli.command('ssh')
@click.argument('argv', nargs=-1)
@click.option('--debug', '-d', is_flag=True)
def ssh_cmd(argv, debug=False):
	opsmgr.ssh(argv, debug=debug)

@cli.command('reboot')
@click.option('--yes-i-am-sure', '-y', is_flag=True)
def reboot_cmd(yes_i_am_sure=False):
	if not yes_i_am_sure:
		print('Rebooting in', end="")
		for i in range(10, 0, -1):
			sys.stdout.write(' ' +str(i))
			sys.stdout.flush()
			time.sleep(1)
		print()
	opsmgr.ssh(['sudo reboot now'], silent=True)
	time.sleep(10) # Allow system time to go down before we ping the API
	opsmgr.unlock()

@cli.command('unlock')
def unlock_cmd():
	opsmgr.unlock()

@cli.command('products')
def products_cmd():
	products = opsmgr.get_products()
	for product in products:
		print("-", product["name"], product["product_version"], "(installed)" if product["installed"] else "")

@cli.command('changes')
def changes_cmd():
	changes = opsmgr.get_changes()
	if changes is None:
		print('This command is only available for PCF 1.7 and beyond.', file=sys.stderr)
		sys.exit(1)
	for p in changes['product_changes']:
		print(p['action'], p['guid'])
		for e in p['errands']:
			print('-', e['name'])

@cli.command('is-available')
@click.argument('product')
@click.argument('version', None, required=False)
def is_available_cmd(product, version):
	products = opsmgr.get_products()
	matches = [ p for p in products if p['name'] == product and (version is None or p['product_version'] == version) ]
	if len(matches) < 1:
		print('No match found for product', product, 'version', version, file=sys.stderr)
		sys.exit(1)

@cli.command('is-installed')
@click.argument('product')
@click.argument('version', None, required=False)
def is_installed_cmd(product, version):
	products = opsmgr.get_products()
	matches = [ p for p in products if p['name'] == product and (version is None or p['product_version'] == version) and p['installed'] ]
	if len(matches) < 1:
		print('Product', product, 'version', version, 'is not installed', file=sys.stderr)
		sys.exit(1)

@cli.command('configure')
@click.argument('product')
@click.argument('properties_file', None, required=False)
@click.option('--strict', is_flag=True)
@click.option('--skip-validation', is_flag=True)
@click.option('-n', '--network')
def configure_cmd(product, properties_file, strict=False, skip_validation=False, network=None):
	properties = {}
	if properties_file is not None:
		with open(properties_file) as f:
			properties = yaml.safe_load(f)
	opsmgr.configure(product, properties, strict, skip_validation, network)

@cli.command('settings')
@click.argument('product', None, required=False)
def settings_cmd(product):
	settings = opsmgr.get('/api/installation_settings').json()
	if product is not None:
		settings = [ p for p in settings['products'] if p['identifier'] == product ]
		if len(settings) < 1:
			print('No settings found for product', product, file=sys.stderr)
			sys.exit(1)
		settings = settings[0]
	print(json.dumps(settings, indent=4))

@cli.command('cf-info')
def cf_info_cmd():
	cfinfo = opsmgr.get_cfinfo()
	for key in sorted(cfinfo):
		print('-', key + ':', cfinfo[key])

@cli.command('import')
@click.argument('tile')
def import_cmd(tile):
	opsmgr.upload('/api/products', tile)
	print('-', 'Tile', tile, 'imported successfully. You can run `pcf install` to load it.')

@cli.command('install')
@click.argument('product')
@click.argument('version')
def install_cmd(product, version):
	products = opsmgr.get_products()
	matches = [ p for p in products if p['name'] == product and p['installed'] ]
	if len(matches) < 1:
		payload = {
			'name': product,
			'product_version': version,
		}
		opsmgr.post('/api/installation_settings/products', payload)
	else:
		payload = {
			'to_version': version
		}
		response = opsmgr.put('/api/installation_settings/products/' + matches[0]['guid'], payload, check=False)
		if response.status_code == 422:
			errors = response.json()["errors"]
			for error in errors:
				if error.endswith(' is already in use.'):
					print('-','version already installed')
					return
		opsmgr.check_response(response)

@cli.command('uninstall')
@click.argument('product')
@click.argument('version', None, required=False)
def uninstall_cmd(product, version):
	products = opsmgr.get('/api/installation_settings/products').json()
	matches = [ p for p in products if p['type'] == product ]
	for match in matches:
		if version and 'product_version' in match:
			if match['product_version'] == version:
				opsmgr.delete('/api/installation_settings/products/' + match['guid'])
		else:
			opsmgr.delete('/api/installation_settings/products/' + match['guid'])

@cli.command('delete-unused-products')
def delete_unused_products_cmd():
	opsmgr.delete('/api/products')

@cli.command('backup')
@click.argument('backup_file')
def backup_cmd(backup_file):
	response = opsmgr.get('/api/installation_asset_collection', stream=True)
	with open(backup_file, 'wb') as f:
		for chunk in response.iter_content(1024):
			f.write(chunk)

@cli.command('restore')
@click.argument('backup_file')
def restore_cmd(backup_file):
	creds = get_credentials()
	with open(backup_file, 'rb') as f:
		payload = { 'installation[file]': f, 'password': creds['opsmgr']['password'] }
		opsmgr.post('/api/installation_asset_collection', f)

@cli.command('cleanup')
@click.argument('product')
def cleanup_cmd(product):
	#
	# Attempt 1 - Delete any uninstalled versions
	#
	products = opsmgr.get('/api/installation_settings/products').json()
	matches = [ p for p in products if p['type'] == product ]
	for match in matches:
		print('- attempting to delete', match['name'], file=sys.stderr)
		opsmgr.delete('/api/installation_settings/products/' + match['guid'])
	products = opsmgr.get('/api/installation_settings/products').json()
	matches = [ p for p in products if p['type'] == product ]
	if len(matches) < 1:
		sys.exit(0)
	if len(matches) > 1:
		print('- more than one match remains installed', file=sys.stderr)
		sys.exit(1)
	#
	# Attempt 2 - Uninstall deployed version
	#
	match = matches[0]
	print('- product was deployed, applying changes to uninstall it', file=sys.stderr)
	apply_changes_cmd()
	opsmgr.delete('/api/products')
	products = opsmgr.get('/api/installation_settings/products').json()
	matches = [ p for p in products if p['type'] == product ]
	if len(matches) < 1:
		sys.exit(0)
	#
	# Attempt 3 - Re-deploy with errands disabled, then uninstall
	#
	match = matches[0]
	print('- uninstall appears to have failed', file=sys.stderr)
	print('- re-deploying with disabled errands', file=sys.stderr)
	opsmgr.disable_errands(product)
	apply_changes_cmd()
	print('- uninstalling with disabled errands', file=sys.stderr)
	opsmgr.delete('/api/installation_settings/products/' + match['guid'])
	apply_changes_cmd()
	opsmgr.delete('/api/products')
	products = opsmgr.get('/api/installation_settings/products').json()
	matches = [ p for p in products if p['type'] == product ]
	if len(matches) > 0:
		print('- failed to uninstall', file=sys.stderr)
		sys.exit(1)

@cli.command('apply-changes')
@click.option('--product', help='product to select errands from. Only valid in combination with -deploy-errands or --delete-errands.')
@click.option('--deploy-errands', help='Comma separated list of errands to run after install/update. For example: "deploy-all,configure-broker"')
@click.option('--delete-errands', help='Comma separated list of errands to run before delete. For example: "pre_delete"')
def apply_changes_cmd(product, deploy_errands, delete_errands):
	body = None
	version = opsmgr.get_version()
	pre_1_10 = version[0] == 1 and version[1] < 10
	if pre_1_10:
		enabled_errands = []
		deploy_errand_list = None if deploy_errands is None else deploy_errands.split(',')
		delete_errand_list = None if delete_errands is None else delete_errands.split(',')
		changes = opsmgr.get_changes(product, deploy_errand_list, delete_errand_list)
		if changes is not None:
			if len(changes['product_changes']) == 0:
				print('Nothing to do', file=sys.stderr)
				return
			for p in changes['product_changes']:
				post_deploy = serialize_errands(p, 'post_deploy', 'post_deploy_errands')
				enabled_errands.extend(post_deploy)
				pre_delete = serialize_errands(p, 'pre_delete', 'pre_delete_errands')
				enabled_errands.extend(pre_delete)
		body = '&'.join(enabled_errands)
	in_progress = None
	install_id = None
	while install_id is None:
		if pre_1_10:
			response = opsmgr.post('/api/installation?ignore_warnings=true', body, check=False)
		else:
			response = opsmgr.post('/api/v0/installations?ignore_warnings=true', body, check=False)
		if response.status_code == 422 and "Install in progress" in response.json()["errors"]:
			if in_progress is None:
				print('Waiting for in-progress installation to complete', file=sys.stderr)
				in_progress = opsmgr.last_install()
			time.sleep(10)
			if opsmgr.install_exists(in_progress + 1):
				install_id = in_progress + 1
				break
			continue
		elif response.status_code == 422:
			print('-', response.status_code, response.request.url, file=sys.stderr)
			try:
				errors = response.json()["errors"]
				print('- '+('\n- '.join(json.dumps(errors, indent=4).splitlines())), file=sys.stderr)
			except:
				print(response.text, file=sys.stderr)
			sys.exit(2)
		opsmgr.check_response(response)
		install = response.json()['install']
{'fraction_non_alphanumeric': 0.15013042218143174, 'fraction_numerical': 0.007342285769490871, 'mean_word_length': 5.200359389038634, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2604212', 'n_tokens_mistral': 7121, 'n_tokens_neox': 6759, 'n_words': 1748}