<jupyter_start><jupyter_text>Destroy AWS Resources<jupyter_code>%load_ext lab_black
%load_ext autoreload
%autoreload 2
import os
from glob import glob
from time import sleep

import boto3
from dotenv import find_dotenv, load_dotenv
%aimport src.s3.buckets
import src.s3.buckets as s3b

%aimport src.cw.cloudwatch_logs
import src.cw.cloudwatch_logs as cwlogs

%aimport src.iam.iam_roles
import src.iam.iam_roles as iamr

%aimport src.firehose.kinesis_firehose
import src.firehose.kinesis_firehose as knsfire

%aimport src.ec2.ec2_instances_sec_groups
import src.ec2.ec2_instances_sec_groups as ec2h

%aimport src.keypairs.ssh_keypairs
import src.keypairs.ssh_keypairs as ssh_keys

%aimport src.ansible.playbook_utils
import src.ansible.playbook_utils as pbu
load_dotenv(find_dotenv())
aws_region = os.getenv("AWS_REGION")<jupyter_output><empty_output><jupyter_text>About In this notebook, the following AWS resources will be destroyed- EC2 instance- S3 bucket- CloudWatch Logging group (automatically deletes CloudWatch Logging stream)- IAM role- Kinesis Firehose Delivery Stream Pre-Requisites1. As mentioned in `README.md`, the following environment variables should be set with the user's AWS credendials ([1](https://docs.aws.amazon.com/sdk-for-php/v3/developer-guide/guide_credentials_environment.html), [2](https://docs.aws.amazon.com/sdk-for-php/v3/developer-guide/guide_credentials_profiles.html))   - `AWS_ACCESS_KEY_ID`   - `AWS_SECRET_KEY`   - `AWS_REGION`   These credentials must be associated to a user group whose users have been granted programmatic access to AWS resources. In order to configure this for an IAM user group, see the documentation [here](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users_create.htmlid_users_create_console). Notes1. All resources must exist in the same AWS region (specified by the environment variable `AWS_REGION`).  User Inputs<jupyter_code># S3
s3_bucket_name = ""

# IAM Role
iam_role_name = "kinesis-firehose-role"
iam_policy_name = "mypolicy"

# Kinesis Firehose Stream
firehose_stream_name = "twitter_delivery_stream"

# CloudWatch Logging
cw_logs_group_name = f"kinesisfirehose_{firehose_stream_name}"

# EC2 Security Groups
sg_group_tags = [{"Key": "Name", "Value": "allow-inbound-ssh"}]

# SSH Key Pairs
key_fname = "aws_ec2_key"
keypair_name = "ec2-key-pair"

# EC2 Instances
ec2_instance_tags_list = [{"Key": "Name", "Value": "my-ec2-instance"}]
ansible_inventory_host_vars_fpath = "inventories/production/host_vars/ec2host"<jupyter_output><empty_output><jupyter_text>Reset Host Target EC2 Attributes in Ansible Inventory Reset the target IP address<jupyter_code>%%time
pbu.replace_inventory_host_ip(ansible_inventory_host_vars_fpath, "...")<jupyter_output>Changed to use specified IP address.
ansible_host: ...
ansible_connection: ssh
ansible_user: ec2-user
ansible_python_interpreter: /usr/bin/python3
ansible_ssh_private_key_file: /tmp/aws_ec2_key.pem
CPU times: user 1.13 ms, sys: 148 µs, total: 1.28 ms
Wall time: 755 µs<jupyter_text>Reset the target Python version<jupyter_code>%%time
pbu.replace_inventory_python_version(ansible_inventory_host_vars_fpath, "2.7")<jupyter_output>Changed to use Python 2.7.
ansible_host: ...
ansible_connection: ssh
ansible_user: ec2-user
ansible_python_interpreter: /usr/bin/python2.7
ansible_ssh_private_key_file: /tmp/aws_ec2_key.pem
CPU times: user 724 µs, sys: 94 µs, total: 818 µs
Wall time: 507 µs<jupyter_text>Stop EC2 Instance<jupyter_code>%%time
ec2_instance_filter = dict(Filters=[{"Name": "tag:Name", "Values": ["my-ec2-instance"]}])
# Filter instances group by tag
filtered_instances_list = ec2h.list_ec2_instances_by_filter(aws_region, ec2_instance_filter)
# Slice filtered instances by index, to get first matching instance_id
first_filtered_instance_id = filtered_instances_list[0]["id"]
stop_instance_response = ec2h.stop_instance(first_filtered_instance_id, aws_region)
stop_instance_response<jupyter_output><empty_output><jupyter_text>Terminate EC2 Instance<jupyter_code>%%time
ec2_instance_filter = dict(Filters=[{"Name": "tag:Name", "Values": ["my-ec2-instance"]}])
# Filter instances group by tag
filtered_instances_list = ec2h.list_ec2_instances_by_filter(aws_region, ec2_instance_filter)
# Slice filtered instances by index, to get first matching instance_id
first_filtered_instance_id = filtered_instances_list[0]["id"]
terminated_instance_response = ec2h.terminate_instance(first_filtered_instance_id, aws_region)
terminated_instance_response
%%time
ec2_instance_filter = dict(
    Filters=[{"Name": "tag:Name", "Values": ["my-ec2-instance"]}]
)
ec2h.check_ec2_instance_termination(ec2_instance_filter, aws_region)<jupyter_output>CPU times: user 31.3 ms, sys: 0 ns, total: 31.3 ms
Wall time: 218 ms<jupyter_text>Delete Local SSH Key Pair<jupyter_code>%%time
key_deletion_response = ssh_keys.delete_key_pair(keypair_name, aws_region)
key_deletion_response
%%time
key_filter = dict(Filters=[{"Name": "tag:Name", "Values": ["my-ssh-key-pair"]}])
ssh_keys.check_deletion_key_pair(key_filter, aws_region)<jupyter_output>CPU times: user 19 ms, sys: 0 ns, total: 19 ms
Wall time: 174 ms<jupyter_text>Delete EC2 Security Groups<jupyter_code>sleep(45)
%%time
sg_filter = dict(Filters=[{"Name": "tag:Name", "Values": ["allow-inbound-ssh"]}])
sg_id_list = ec2h.get_security_group_ids(aws_region, sg_filter)
ec2h.delete_sg(sg_id_list[0], aws_region)
%%time
sg_filter = dict(Filters=[{"Name": "tag:Name", "Values": ["allow-inbound-ssh"]}])
ec2h.check_ec2_security_group_deletion(sg_filter, aws_region)<jupyter_output>CPU times: user 28.8 ms, sys: 0 ns, total: 28.8 ms
Wall time: 323 ms<jupyter_text>Delete the Kinesis Firehose Delivery Stream<jupyter_code>%%time
firehose_deletion_response = knsfire.delete_kinesis_firehose_stream(
    firehose_stream_name, aws_region
)
knsfire.check_kinesis_firehose_stream_seletion(firehose_stream_name, aws_region)<jupyter_output>CPU times: user 37.7 ms, sys: 0 ns, total: 37.7 ms
Wall time: 336 ms<jupyter_text>IAM  Delete IAM Firehose-S3 Policy Get Attached IAM Policies<jupyter_code>%%time
iam_firehose_s3_policy_list = iamr.get_iam_policies(aws_region, attached=True)
iam_policy_dict = [
    iam_firehose_s3_policy
    for iam_firehose_s3_policy in iam_firehose_s3_policy_list
    if iam_firehose_s3_policy["PolicyName"] == iam_policy_name
][0]
iam_policy_dict<jupyter_output><empty_output><jupyter_text>Delete IAM Firehose-S3 Policy<jupyter_code>%%time
policy_detachment_response, policy_deletion_response = iamr.delete_iam_policy(
    iam_policy_dict["Arn"], iam_role_name, aws_region
)
print(policy_detachment_response)
print(policy_deletion_response)<jupyter_output><empty_output><jupyter_text>Delete IAM Role<jupyter_code>%%time
iam_role_deletion_response = iamr.delete_iam_role(iam_role_name, aws_region)
iamr.check_iam_role_deletion(iam_role_name, aws_region)<jupyter_output>CPU times: user 41.6 ms, sys: 0 ns, total: 41.6 ms
Wall time: 279 ms<jupyter_text>Delete CloudWatch Log Group and Stream<jupyter_code>%%time
cwlogs.delete_cw_log_group_stream(cw_logs_group_name, firehose_stream_name, aws_region)
cwlogs.check_cw_log_group_deletion(cw_logs_group_name, aws_region)
cwlogs.check_cw_log_stream_deletion(cw_logs_group_name, firehose_stream_name, aws_region)<jupyter_output>CPU times: user 89 ms, sys: 5.22 ms, total: 94.2 ms
Wall time: 561 ms<jupyter_text>(Optional) Delete the S3 Bucket<jupyter_code>%%time
if s3_bucket_name:
    s3_bucket_deletion_response = s3b.delete_s3_bucket(s3_bucket_name, aws_region)
    s3b.check_s3_bucket_deletion(s3_bucket_name, aws_region)<jupyter_output>CPU times: user 12.6 ms, sys: 181 µs, total: 12.8 ms
Wall time: 161 ms
{'fraction_non_alphanumeric': 0.09820374983610856, 'fraction_numerical': 0.023469253966172807, 'mean_word_length': 7.984687868080094, 'pattern_counts': {'":': 16, '<': 48, '<?xml version=': 0, '>': 48, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '492448', 'license': 'MIT', 'n_tokens_mistral': 2973, 'n_tokens_neox': 2841, 'n_words': 571, 'path': '6_delete_aws_resources.ipynb', 'repo_name': 'elsdes3/big-data-ml', 'size': '7627'}