<jupyter_start><jupyter_text>Notebook: Working with AWS SQS Queues  **Setup:**<jupyter_code># Import libraries we will use:
import boto3  # boto3 is AWS's Python SDK, by AWS
from botocore.exceptions import ClientError
from dotenv import load_dotenv
import json
import os
# Get access info from .env file:
load_dotenv()

AWS_ACCESS_KEY_ID =os.getenv("AWS_ACCESS_KEY_ID")
AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY")

SQS_QUEUE_NAME = os.getenv("SQS_QUEUE_NAME")
# Create an SQS Service Resource:
sqs = boto3.resource('sqs',
                     aws_access_key_id=AWS_ACCESS_KEY_ID,
                     aws_secret_access_key=AWS_SECRET_ACCESS_KEY
                    )

# Create a client object for SQS, using the access keys in our .env file:
sqs_client = boto3.client('sqs',
                          aws_access_key_id=AWS_ACCESS_KEY_ID,
                          aws_secret_access_key=AWS_SECRET_ACCESS_KEY
                         )

# Get our queue from SQS:
queue = sqs.get_queue_by_name(QueueName=SQS_QUEUE_NAME)
queue.url<jupyter_output><empty_output><jupyter_text>Functions: For Working with Our TeamReel SQS Queues:<jupyter_code>def sqs_queue_get_next_item():
    """
    Gets the next message from our AWS SQS (Simple Queue Service) queue, and returns it.
    """

    # Get next message from SQS queue
    # (messages are auto-added by our S3 bucket each time a new video is uploaded to our S3 bucket):
    message = sqs_client.receive_message(QueueUrl=queue.url,
                                         MaxNumberOfMessages=1,
                                         MessageAttributeNames=['All']
                                        )

    # Check to see if queue is empty:
    if message == []:
        return "No messages in queue."
    elif (type(message) is dict) and ('Messages' not in message.keys()):
        return "No messages in queue."

    # If queue has messages (is not empty):
    else:
        message = message['Messages'][0]
        return message
def sqs_delete_message_from_queue(receipt_handle:str):
    """
    Deletes a just-received message from the queue. (Note: Valid only within
    30 seconds of receiving the message, because that is our SQS queue's
    visibility timeout. So it is ideal to delete the message in the same run
    or same code cell during which you received the message).
    """
    
    # Delete message received from SQS queue:
    sqs_client.delete_message(
        QueueUrl=queue.url,
        ReceiptHandle=receipt_handle
    )

    return True<jupyter_output><empty_output><jupyter_text>Testing Our Functions:<jupyter_code>sqs_queue_get_next_item()<jupyter_output><empty_output><jupyter_text>-------------  For Reference Only: Other SQS Operations<jupyter_code># # FOR REFERENCE ONLY (Not Used Here):
# # Resource objects: Other Relevant Functions for Service *RESOURCE* Objects:

# # -------------------------------------------------------

# # Create an SQS Resource object (already done above):
# sqs = boto3.resource('sqs', 
#                      aws_access_key_id=AWS_ACCESS_KEY_ID, 
#                      aws_secret_access_key=AWS_SECRET_ACCESS_KEY
#                     )

# # -------------------------------------------------------

# # List all queues we have in SQS:
# for queue in sqs.queues.all():
#     print(queue)

# # -------------------------------------------------------

# # Create a new SQS queue: Standard queue:
# queue = sqs.create_queue(QueueName='queue_development_ds_123')

# # -------------------------------------------------------

# # OR create a new SQS queue: FIFO queue:
# queue_test_ds = sqs.create_queue(QueueName='queue_test_ds_123.fifo', 
#                                  Attributes={
#                                      'FifoQueue': 'true', 
#                                      'ContentBasedDeduplication': 'true'
#                                  }
#                                 )
# # FOR REFERENCE ONLY (Not Used Here):
# # Client objects: Other Relevant Functions for Service *CLIENT* Objects:

# # -------------------------------------------------------

# # Most notable functions for SQS Client objects:

# # 'create_queue':
# # Similar syntax and usage to create_queue() for an SQS Resource (see above).

# # -------------------------------------------------------

# # Add message to SQS queue: 
# # METHOD 1: Message content in MessageBody as a stringified JSON:
# sent_message = sqs_client.send_message(QueueUrl=queue.url, 
#                                        MessageBody=str(message_json), 
#                                        MessageAttributes=None
#                                       )

# # Add message to SQS queue: 
# # METHOD 2: Message content in MessageAttributes as keys, AWS SQS style:
# sent_message = sqs_client.send_message(QueueUrl=queue.url, 
#                                        MessageBody="New video uploaded", 
#                                        # FifoQueue ONLY: MessageGroupId="MessageGroupTeamReel", 
#                                        # FifoQueue ONLY: MessageDeduplicationId="MessageDeduplicationId1234567892", 
#                                        MessageAttributes={
#                                            "video_info": {
#                                                "DataType": "String", 
#                                                "StringValue": "user6739820"
#                                            }, 
#                                            "prompt_id": {
#                                                "DataType": "String", 
#                                                "StringValue": "prompt24850295"
#                                            }, 
#                                            "video_id": {
#                                                "DataType": "String", 
#                                                "StringValue": "video2548560"
#                                            }, 
#                                            "video_s3_key": {
#                                                "DataType": "String", 
#                                                "StringValue": "videos/ALPACAVID-l3Vlyt5vJ.webm"
#                                            }, 
#                                            "video_s3_filename": {
#                                                "DataType": "String", 
#                                                "StringValue": "ALPACAVID-l3Vlyt5vJ.webm"
#                                            }
#                                        }
#                                       )

# # -------------------------------------------------------

# # 'delete_message'

# # 'delete_message_batch'

# # 'delete_queue'

# # 'get_queue_attributes'

# # 'get_queue_url'

# # 'get_waiter'

# # 'list_dead_letter_source_queues'

# # 'list_queues'

# # 'meta'

# # 'receive_message'

# # 'send_message'

# # 'send_message_batch'

# # 'waiter_names'
# # FOR REFERENCE ONLY (Not Used Here):
# # Queue objects: Other Relevant Functions at for *QUEUE* objects:

# # -------------------------------------------------------

# # Get our queue from an SQS service Resource 
# # (here, "sqs" is the service Resource we established above with the above sqs = boto3.resource(____) code):
# queue = sqs.get_queue_by_name(QueueName=SQS_QUEUE_NAME)
# queue

# # Get URL of queue:
# queue.url

# # Get other attributes of queue:
# queue.attributes

# queue.attributes.get('Policy')
# # Alternative syntax: queue.attributes['Policy']

# queue.attributes.get('DelaySeconds')
# # Alternative syntax: queue.attributes['DelaySeconds']

# queue.attributes.get('QueueArn').split(':')[-1]
# # Alternate syntax for above: queue.attributes['QueueArn'].split(':')[-1]

# # Get messages from the queue (up to 10 messages, as per the 
# # MaxNumberOfMessages param below):
# messages = queue.receive_messages(QueueUrl=queue.url, 
#                                   MaxNumberOfMessages=10, 
#                                   MessageAttributeNames=['All']
#                                  )
# Get all messages from SQS queue (which are auto-added by our S3 bucket 
# each time a new video is uploaded to our S3 bucket):
messages_all = queue.receive_messages(QueueUrl=queue.url, 
                                      MaxNumberOfMessages=10, 
                                      MessageAttributeNames=['All']
                                     )

# Get the content of the first message 
# (assuming the message is from an S3 bucket notification -> content is 
# stored in MessageBody and not in MessageAttributes):
message_01_json = json.loads(messages_all[0].body)
message_01_json
# Getting specific info from the raw message above:
video_s3_key = message_01_json['Records'][0]['s3']['object']['key']
video_s3_key
# # FOR REFERENCE ONLY (Not Used Here):
# # Message objects: Other Relevant Functions at for *MESSAGE* objects:

# # -------------------------------------------------------

# # Start with any message (here, we get the first message from the list our 
# # above queue.receive_messages() call returned):
# message = messages[0]

# # Get the message's SQS id:
# message.message_id

# # Get the message body (usually the JSON with the message's main info is stored here):
# message.body
# # Turn the message body string into a JSON (Python dictionary) we can work with more easily:
# json.loads(message.body)

# # Get the message's attributes (the content of the message can also be stored as key:value pairs in 
# # MessageAttributes, so if it's not stored in MessageBody as a str(JSON) then check here):
# message.message_attributes

# # Which SQS queue does the message come from, and what is that queue's URL?:
# message.Queue()
# message.queue_url

# # Get the message's receipt handle (this is what you need to delete the message from the queue):
# message.receipt_handle

# # Delete the message from the queue:
# # (Note: You may need to input the message's receipt_handle, in which case you need to 
# # both get the message and delete it in the same code cell or .py file, because a 
# # receipt_handle is only valid for 1-10 ms or similar):
# message.delete<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1336929867582148, 'fraction_numerical': 0.008239333006375674, 'mean_word_length': 1.6831578947368422, 'pattern_counts': {'":': 15, '<': 17, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 64, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '445264', 'license': 'MIT', 'n_tokens_mistral': 2744, 'n_tokens_neox': 2605, 'n_words': 763, 'path': 'data_infra/notebooks/notebook_aws_sqs_workbook.ipynb', 'repo_name': 'tatianaportsova/TeamReel_DS', 'size': '10195'}