<commit_before>#!/usr/bin/env python3

"""
Created on 23 Mar 2017

@author: <NAME> (<EMAIL>)

WARNING: only one MQTT client can run at any one time on one TCP/IP host.

Requires ClientAuth document.

command line example:
./scs_dev/status_sampler.py | ./scs_dev/mqtt_client.py -p -e
"""

import json
import random
import sys
import time

from collections import OrderedDict

from scs_core.data.json import JSONify
from scs_core.data.localized_datetime import LocalizedDatetime
from scs_core.data.publication import Publication
from scs_core.osio.client.client_auth import ClientAuth
from scs_core.sys.exception_report import ExceptionReport

from scs_dev.cmd.cmd_mqtt_client import CmdMQTTClient

from scs_host.client.mqtt_client import MQTTClient
from scs_host.sys.host import Host


# TODO: host and client auth should be generalised - point to file as command line param

# --------------------------------------------------------------------------------------------------------------------

if __name__ == '__main__':

    client = None

    # ----------------------------------------------------------------------------------------------------------------
    # cmd...

    cmd = CmdMQTTClient()
    if not cmd.is_valid():
        cmd.print_help(sys.stderr)
        exit()

    log_file = open(cmd.log, 'a') if cmd.log else None

    if cmd.verbose:
        print(cmd, file=sys.stderr)

    try:
        # ------------------------------------------------------------------------------------------------------------
        # resource...

        auth = ClientAuth.load_from_host(Host)

        if auth is None:
            print("ClientAuth not available.", file=sys.stderr)
            exit()

        if cmd.verbose:
            print(auth, file=sys.stderr)


        client = MQTTClient()
        client.connect(ClientAuth.MQTT_HOST, auth.client_id, auth.user_id, auth.client_password)

        if cmd.verbose:
            print(client, file=sys.stderr)


        # ------------------------------------------------------------------------------------------------------------
        # run...

        if cmd.publish:
            for line in sys.stdin:
                datum = json.loads(line, object_pairs_hook=OrderedDict)

                while True:
                    publication = Publication.construct_from_jdict(datum)

                    try:
                        if cmd.log:
                            log_file.write("%s: rec: %s\n" % (LocalizedDatetime.now().as_iso8601(), datum['rec']))
                            log_file.flush()

                        payload = JSONify.dumps(publication.payload)

                        success = client.publish(publication.topic, payload, ClientAuth.MQTT_TIMEOUT)

                        if cmd.log and not success:
                            log_file.write("%s: abandoned")
                            log_file.flush()

                        break

                    except Exception as ex:
                        if cmd.log:
                            report = JSONify.dumps(ExceptionReport.construct(ex))
                            log_file.write("%s\n" % report)
                            log_file.flush()

                    time.sleep(random.uniform(1.0, 2.0))           # Don't hammer the client!

                if cmd.log:
                    log_file.write("%s: done\n" % LocalizedDatetime.now().as_iso8601())
                    log_file.write("-\n")
                    log_file.flush()

                if cmd.echo:
                    print(line, end="")
                    sys.stdout.flush()


    # ----------------------------------------------------------------------------------------------------------------
    # end...

    except KeyboardInterrupt as ex:
        if cmd.verbose:
            print("mqtt_client: KeyboardInterrupt", file=sys.stderr)

    except Exception as ex:
        if cmd.log:
            report = JSONify.dumps(ExceptionReport.construct(ex))
            log_file.write("%s: %s\n" % (LocalizedDatetime.now().as_iso8601(), report))
            log_file.flush()

        print(JSONify.dumps(ExceptionReport.construct(ex)), file=sys.stderr)

    finally:
        if client:
            client.disconnect()

        if cmd.log:
            log_file.close()
<commit_msg>Fixed a bug in mqtt_client.<commit_after>#!/usr/bin/env python3

"""
Created on 23 Mar 2017

@author: <NAME> (<EMAIL>)

WARNING: only one MQTT client can run at any one time on one TCP/IP host.

Requires ClientAuth document.

command line example:
./scs_dev/status_sampler.py | ./scs_dev/mqtt_client.py -p -e
"""

import json
import random
import sys
import time

from collections import OrderedDict

from scs_core.data.json import JSONify
from scs_core.data.localized_datetime import LocalizedDatetime
from scs_core.data.publication import Publication
from scs_core.osio.client.client_auth import ClientAuth
from scs_core.sys.exception_report import ExceptionReport

from scs_dev.cmd.cmd_mqtt_client import CmdMQTTClient

from scs_host.client.mqtt_client import MQTTClient
from scs_host.sys.host import Host


# TODO: host and client auth should be generalised - point to file as command line param

# --------------------------------------------------------------------------------------------------------------------

if __name__ == '__main__':

    client = None

    # ----------------------------------------------------------------------------------------------------------------
    # cmd...

    cmd = CmdMQTTClient()
    if not cmd.is_valid():
        cmd.print_help(sys.stderr)
        exit()

    log_file = open(cmd.log, 'a') if cmd.log else None

    if cmd.verbose:
        print(cmd, file=sys.stderr)

    try:
        # ------------------------------------------------------------------------------------------------------------
        # resource...

        auth = ClientAuth.load_from_host(Host)

        if auth is None:
            print("ClientAuth not available.", file=sys.stderr)
            exit()

        if cmd.verbose:
            print(auth, file=sys.stderr)


        client = MQTTClient()
        client.connect(ClientAuth.MQTT_HOST, auth.client_id, auth.user_id, auth.client_password)

        if cmd.verbose:
            print(client, file=sys.stderr)


        # ------------------------------------------------------------------------------------------------------------
        # run...

        if cmd.publish:
            for line in sys.stdin:
                datum = json.loads(line, object_pairs_hook=OrderedDict)

                while True:
                    publication = Publication.construct_from_jdict(datum)

                    try:
                        if cmd.log:
                            log_file.write("%s: rec: %s\n" % (LocalizedDatetime.now().as_iso8601(),
                                                              datum['payload']['rec']))
                            log_file.flush()

                        payload = JSONify.dumps(publication.payload)

                        success = client.publish(publication.topic, payload, ClientAuth.MQTT_TIMEOUT)

                        if cmd.log and not success:
                            log_file.write("%s: abandoned")
                            log_file.flush()

                        break

                    except Exception as ex:
                        if cmd.log:
                            report = JSONify.dumps(ExceptionReport.construct(ex))
                            log_file.write("%s\n" % report)
                            log_file.flush()

                    time.sleep(random.uniform(1.0, 2.0))           # Don't hammer the client!

                if cmd.log:
                    log_file.write("%s: done\n" % LocalizedDatetime.now().as_iso8601())
                    log_file.write("-\n")
                    log_file.flush()

                if cmd.echo:
                    print(line, end="")
                    sys.stdout.flush()


    # ----------------------------------------------------------------------------------------------------------------
    # end...

    except KeyboardInterrupt as ex:
        if cmd.verbose:
            print("mqtt_client: KeyboardInterrupt", file=sys.stderr)

    except Exception as ex:
        if cmd.log:
            report = JSONify.dumps(ExceptionReport.construct(ex))
            log_file.write("%s: %s\n" % (LocalizedDatetime.now().as_iso8601(), report))
            log_file.flush()

        print(JSONify.dumps(ExceptionReport.construct(ex)), file=sys.stderr)

    finally:
        if client:
            client.disconnect()

        if cmd.log:
            log_file.close()

{'fraction_non_alphanumeric': 0.21251298626341913, 'fraction_numerical': 0.005309938820270114, 'mean_word_length': 1.2023385866802236, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3182200', 'n_tokens_mistral': 2264, 'n_tokens_neox': 2123, 'n_words': 492}