<jupyter_start><jupyter_text>a.i<jupyter_code># import statements
from time import sleep
from kafka import KafkaConsumer
import datetime as dt
import matplotlib.pyplot as plt
import json

%matplotlib notebook

topic = 'assignment'

def annotate_max(x, y, ax = None):
    ymax = max(y)
    xpos = y.index(ymax)
    xmax = x[xpos]
    text = 'Max: Time={}, Value={}'.format(xmax, ymax)
    if not ax:
        ax=plt.gca()
    ax.annotate(text, xy=(xmax, ymax), xytext=(xmax, ymax+5), arrowprops=dict(facecolor='red', shrink=0.05),)
    
def annotate_min(x, y, ax = None):
    ymin = min(y)
    xpos = y.index(ymin)
    xmin = x[xpos]
    text = 'Min: Time={}, Value={}'.format(xmin, ymin)
    if not ax:
        ax=plt.gca()
    ax.annotate(text, xy=(xmin, ymin), xytext=(xmin, ymin+5), arrowprops=dict(facecolor='orange', shrink=0.05),)

def connect_kafka_consumer():
    _consumer = None
    try:
         _consumer = KafkaConsumer(topic,
                                   consumer_timeout_ms=10000, # stop iteration if no message after 10 sec
                                   auto_offset_reset='earliest', # comment this if you don't want to consume earliest available message
                                   bootstrap_servers=['127.0.0.1:9092'],
                                   api_version=(0, 10))
    except Exception as ex:
        print('Exception while connecting Kafka')
        print(str(ex))
    finally:
        return _consumer

def init_plots():
    try:
        width = 9.5
        height = 6
        fig = plt.figure(figsize=(width,height)) # create new figure
        ax = fig.add_subplot(111) # adding the subplot axes to the given grid position
        fig.suptitle('Air temperature against arrival time') # giving figure a title
        ax.set_xlabel('Time')
        ax.set_ylabel('Air_temperature')
        ax.set_ylim(0,40) 
        ax.set_yticks([0,5,10,15,20,25,30])
        fig.show() # displaying the figure
        fig.canvas.draw() # drawing on the canvas
        return fig, ax
    except Exception as ex:
        print(str(ex))
    
def consume_messages(consumer, fig, ax):
    try:
        # container for x and y values
        x, y = [], []
        for message in consumer:
            dd = str(message.value.decode('utf-8'))
            data = json.loads(dd)
            producer = data.get("sender_id")
            if producer == "Producer 1":
                tem = data.get("air_temperature_celcius")
                time = data.get("datetime")
                if time is not None and tem is not None:
                    x.append(time) 
                    y.append(tem)
            # we start plotting only when we have 10 data points
            if len(y) > 10:
                ax.clear()
                ax.plot(x, y)
                ax.set_xlabel('Time')
                ax.set_ylabel('Air_temperature')
                ax.set_ylim(0,40) 
                ax.set_yticks([0,5,10,15,20,25,30])
                annotate_max(x,y)
                annotate_min(x,y)
                fig.canvas.draw()
                x.pop(0) # removing the item in the first position
                y.pop(0)
        plt.close('all')
    except Exception as ex:
        print(str(ex))
    
if __name__ == '__main__':
    
    consumer = connect_kafka_consumer()
    fig, ax = init_plots()
    consume_messages(consumer, fig, ax)<jupyter_output><empty_output><jupyter_text>b.i<jupyter_code>import pymongo
from pymongo import MongoClient
from pprint import pprint

client = MongoClient () # connect on the default host and port
db = client.fit5148_assignment_db #create databese fit5148_assignment_db
hotspot = db.hotspottest #create collection hotspot 

results = hotspot.aggregate([
            {"$group":
                {
                    
                    "_id":"$datetime",
                    "fire":{"$sum":1}
                
            }
            },
            {
            "$sort" : {"fire":-1}
            },
            {"$limit":10}
        ])
time = []
fire = []
for x in results:
    time.append(x["_id"])
    fire.append(x["fire"])
import matplotlib.pyplot as plt
import numpy as np
plt.bar(time,fire)
plt.xticks(rotation=90)
plt.show()<jupyter_output><empty_output><jupyter_text>b.ii<jupyter_code>import gmplot
# gmplot contains a simple wrapper around 
# Google’s geocoding service enabling map initilization to the location of your choice.
gmap = gmplot.GoogleMapPlotter(-37.4713, 144.7852, 10) # latitude and longitude of melbourne city

from pymongo import MongoClient


# Preparing the data for plotting
unoccupiedList = []

# Looping through all the data
# Adding the latitude and longitude to the list of unoccupied spots
import pymongo
from pymongo import MongoClient
from pprint import pprint

client = MongoClient () # connect on the default host and port
db = client.fit5148_assignment_db 
hotspot = db.hotspottest #if you want to test, please change the db.hotspottest to db.Hotspot
results = hotspot.aggregate([
            {"$lookup":
                {
                    "from": "Climate",
                    "localField": "date",
                    "foreignField" : "date",
                    "as": "climate_info"
                }
            },
            {"$project":
                { 
                    "latitude":1,
                    "longitude":1, 
                    "confidence":1,
                    "climate_info.air_temperature_celcius":1, 
                    "climate_info.relative_humidity":1, 
                    "_id":0,
                    "surface_temperature_celcius":1,
                }
            }
        ])
hidden_gem_lat = ""
hidden_gem_lon = ""
humidity = ""
confidence = ""
air_tem = ""
sur_tem = ""
for x in results:
    unoccupiedList.append((float(x['latitude']), float(x['longitude'])))
    hidden_gem_lat = float(x['latitude'])
    hidden_gem_lon = float(x['longitude'])
    humidity = str(x["climate_info"][0]["relative_humidity"])
    air_tem = str(x["climate_info"][0]["air_temperature_celcius"])
    confidence = str(x["confidence"])
    sur_tem = str(x["surface_temperature_celcius"])
    info = "humidity: "  + humidity + ", confidence: " + confidence + ", air_tem: " + air_tem + ", sur_tem: " + sur_tem
    gmap.marker(hidden_gem_lat, hidden_gem_lon, title=info)
        
# The step below is required by the gmap scatter function to prepare data in correct format
unoccupied_lats, unoccupied_lons = zip(*unoccupiedList) 

# Plotting the points on the map
gmap.scatter(unoccupied_lats, unoccupied_lons, '#FF4500', size=8, marker=False)

import webbrowser
# Drawing the map
gmap.draw("availableParkings.html")
webbrowser.open_new("availableParkings.html")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.113033104682374, 'fraction_numerical': 0.017745302713987474, 'mean_word_length': 2.219875180028805, 'pattern_counts': {'":': 19, '<': 13, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '555206', 'license': 'Apache-2.0', 'n_tokens_mistral': 2211, 'n_tokens_neox': 2052, 'n_words': 503, 'path': 'Assignment_TaskC_Data_Visualisation.ipynb', 'repo_name': 'xhua5296/Victoria-Fire-Event-Data-processing', 'size': '6706'}