<jupyter_start><jupyter_text>Verify S3_BUCKET Bucket Creation<jupyter_code>import boto3
import time

session = boto3.session.Session()
region = session.region_name
account_id = boto3.client("sts").get_caller_identity().get("Account")

s3 = boto3.Session().client(service_name="s3", region_name=region)
setup_s3_bucket_passed = False
%store -r airflow_bucket_name
%store -r s3_mwaa_private_path
%store -r s3_mwaa_dags_private_path
%store -r airflow_env_name
%store -r airflow_vpc_name
%store -r team_role_arn
%store -r airflow_sg_id
%store -r airflow_subnet_ids
!aws s3 ls $s3_mwaa_private_path
from botocore.client import ClientError

response = None

try:
    response = s3.head_bucket(Bucket=airflow_bucket_name)
    print(response)
    setup_s3_bucket_passed = True
except ClientError as e:
    print("[ERROR] Cannot find bucket {} in {} due to {}.".format(airflow_bucket_name, response, e))
# %store setup_s3_bucket_passed<jupyter_output><empty_output><jupyter_text>Create Managed Apache Airflow Environment<jupyter_code>mwaa = boto3.client("mwaa")

s3_mwaa_bucket_arn = "arn:aws:s3:::{}".format(airflow_bucket_name)

airflow_env_arn = mwaa.create_environment(
    DagS3Path="dags",
    ExecutionRoleArn=team_role_arn,
    AirflowVersion="1.10.12",
    WebserverAccessMode="PUBLIC_ONLY",
    LoggingConfiguration={
        "DagProcessingLogs": {"Enabled": True, "LogLevel": "ERROR"},
        "SchedulerLogs": {"Enabled": True, "LogLevel": "ERROR"},
        "TaskLogs": {"Enabled": True, "LogLevel": "INFO"},
        "WebserverLogs": {"Enabled": True, "LogLevel": "ERROR"},
        "WorkerLogs": {"Enabled": True, "LogLevel": "ERROR"},
    },
    MaxWorkers=3,
    Name=airflow_env_name,
    NetworkConfiguration={
        "SecurityGroupIds": [
            airflow_sg_id,
        ],
        "SubnetIds": airflow_subnet_ids,
    },
    RequirementsS3ObjectVersion="latest",
    RequirementsS3Path="requirements.txt",
    SourceBucketArn=s3_mwaa_bucket_arn,
    EnvironmentClass="mw1.small",
)

%store airflow_env_arn<jupyter_output><empty_output><jupyter_text>Please be patient this can take around 15 Minutes.<jupyter_code>def get_airflow_check():
    response = mwaa.get_environment(Name=airflow_env_name)
    mwaa_status = response["Environment"]["Status"]
    return mwaa_status


mwaa_status = "CREATING"

print("Checking to see if MWAA Env: {} is ready.".format(airflow_env_name))

while get_airflow_check() != "AVAILABLE":
    mwaa_status
    time.sleep(60)
    print("Still waiting for MWAA Environment...")

print("Sucess! MWAA Env: {} is ready!".format(airflow_env_name))<jupyter_output><empty_output><jupyter_text>PLEASE MAKE SURE THAT THE ABOVE COMMAND RAN SUCESSFULLY BEFORE CONTINUING<jupyter_code>response = mwaa.create_web_login_token(
    Name=airflow_env_name
)

webServerHostName = response["WebServerHostname"]
webToken = response["WebToken"]
airflowUIUrl = 'https://'+webServerHostName+'/aws_mwaa/aws-console-sso?login=true#'+webToken
print("Here is your AirflowUI Url:"\n)
airflowUIUrl<jupyter_output><empty_output><jupyter_text>Release Resources<jupyter_code>%%html

<p><b>Shutting down your kernel for this notebook to release resources.</b></p>
<button class="sm-command-button" data-commandlinker-command="kernelmenu:shutdown" style="display:none;">Shutdown Kernel</button>
        
<script>
try {
    els = document.getElementsByClassName("sm-command-button");
    els[0].click();
}
catch(err) {
    // NoOp
}    
</script>
%%javascript

try {
    Jupyter.notebook.save_checkpoint();
    Jupyter.notebook.session.delete();
}
catch(err) {
    // NoOp
}<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12825484764542935, 'fraction_numerical': 0.009418282548476454, 'mean_word_length': 5.506306306306306, 'pattern_counts': {'":': 18, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '545075', 'license': 'Apache-2.0', 'n_tokens_mistral': 1311, 'n_tokens_neox': 1272, 'n_words': 222, 'path': '10_pipeline/airflow/02_Create_Airflow_Environment.ipynb', 'repo_name': 'MarcusFra/workshop', 'size': '3610'}