<commit_before>#!/usr/bin/env python
# -*- encoding: utf-8

import datetime as dt
import gzip
import os
import subprocess
import sys

import boto3

from service_utils import service


def get_message(sqs_client, sqs_queue_url):
    resp = sqs_client.receive_message(
        QueueUrl=sqs_queue_url,
        MaxNumberOfMessages=1
    )

    try:
        message = resp['Messages'][0]
    except (KeyError, IndexError):
        print('*** No messages received from SQS, aborting!')
        sys.exit(0)
    else:
        print(f'*** Received message {message!r} from SQS')
        return message


def build_elasticsearch_url(index):
    # We read the Elasticsearch auth information from the environment variables
    username = os.environ['es_username']
    password = <PASSWORD>['<PASSWORD>']
    hostname = os.environ['es_hostname']
    port = os.environ['es_port']
    scheme = os.environ.get('es_scheme', 'http://')

    return f'{scheme}{username}:{password}@{hostname}:{port}/{index}'


@service
def run():
    print(os.environ)
    sqs_queue_url = os.environ['sqs_queue_url']
<commit_msg>elasticdump: default to HTTPS, not HTTP<commit_after>#!/usr/bin/env python
# -*- encoding: utf-8

import datetime as dt
import gzip
import os
import subprocess
import sys

import boto3

from service_utils import service


def get_message(sqs_client, sqs_queue_url):
    resp = sqs_client.receive_message(
        QueueUrl=sqs_queue_url,
        MaxNumberOfMessages=1
    )

    try:
        message = resp['Messages'][0]
    except (KeyError, IndexError):
        print('*** No messages received from SQS, aborting!')
        sys.exit(0)
    else:
        print(f'*** Received message {message!r} from SQS')
        return message


def build_elasticsearch_url(index):
    # We read the Elasticsearch auth information from the environment variables
    username = os.environ['es_username']
    password = os.environ['es_password']
    hostname = os.environ['es_hostname']
    port = os.environ['es_port']
    scheme = os.environ.get('es_scheme', 'https://')

    return f'{scheme}{username}:{password}@{hostname}:{port}/{index}'


@service
def run():
    print(os.environ)
    sqs_queue_url = os.environ['sqs_queue_url']

{'fraction_non_alphanumeric': 0.12149954832881663, 'fraction_numerical': 0.004516711833785004, 'mean_word_length': 3.624217118997912, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6526171', 'n_tokens_mistral': 711, 'n_tokens_neox': 712, 'n_words': 171}