<jupyter_start><jupyter_text>1. MQQ4 trials<jupyter_code>%load_ext autoreload
%autoreload 2

import paho.mqtt.client as mqtt
from pydriller import RepositoryMining
import pandas as pd
import pathlib
import datetime
import plotly.graph_objects as go
list_received_messages = []

# The callback for when the client receives a CONNACK response from the server.
def on_connect(client, userdata, flags, rc):
    print("Connected with result code "+str(rc))

    # Subscribing in on_connect() means that if we lose the connection and
    # reconnect then subscriptions will be renewed.
    # client.subscribe("$SYS/#")

# The callback for when a PUBLISH message is received from the server.
def on_message(client, userdata, msg):
    list_received_messages.append(msg.topic+" "+str(msg.payload))<jupyter_output><empty_output><jupyter_text>Instantiate a client<jupyter_code># Client(client_id=””, clean_session=True, userdata=None, protocol=MQTTv311, transport=”tcp”)
client = mqtt.Client()
client.on_connect = on_connect
client.on_message = on_message<jupyter_output><empty_output><jupyter_text>Connect to a broker http://www.steves-internet-guide.com/client-connections-python-mqtt/<jupyter_code># client.connect(host, port=1883, keepalive=60, bind_address="")

host = "192.168.1.23" # "mqtt.eclipse.org"
client.connect(host, 1883, 60)
client.loop_start() #start the loop. Activate handling of callbacks
client.subscribe("house/light")
client.subscribe("house/toast")
message_info = client.publish("house/light","gherthr")
message_info = client.publish("house/toast","45")
message_info.is_published()
list_received_messages
client.loop_stop()<jupyter_output><empty_output><jupyter_text>2. Pathlib trials<jupyter_code>pathlib.Path("./")<jupyter_output><empty_output><jupyter_text>3. Formatting of data trials<jupyter_code>caca = {'Registration Timestamp': [datetime.datetime(2020, 2, 1, 18, 52, 24, 684720)], 'Topic': ['house/light'], 'Value': [b'hrtht']}
pd.DataFrame.from_dict(caca)<jupyter_output><empty_output><jupyter_text>4. Pydrill for github<jupyter_code>repo_dir = "https://github.com/manuwhs/dashboards.git"
repo_mining = RepositoryMining(repo_dir)
commits = repo_mining.traverse_commits()
for commit in commits:
    print('On {} the author {} includes {} modified files'.format(commit.committer_date, commit.author.name, len(commit.modifications)))
    for modification in commit.modifications:
        print('Filename {}'.format(modification.filename))
        if modification.filename[-3:] == ".py":
            # print('Diff: {}'.format(modification.diff))
            print(f"   Lines added: {modification.added}")
            print(f"   Lines removed: {modification.removed}")


for commit in commits:
    print('On {} the author {} includes {} modified files'.format(commit.committer_date, commit.author.name, len(commit.modifications)))
    for modification in commit.modifications:
        print('Filename {}'.format(modification.filename))
        if modification.filename[-3:] == ".py":
            # print('Diff: {}'.format(modification.diff))
            print(f"   Lines added: {modification.added}")
            print(f"   Lines removed: {modification.removed}")<jupyter_output>On 2020-02-01 12:59:06+01:00 the author <NAME> includes 1 modified files
Filename README.md
On 2020-02-01 13:58:58+01:00 the author <NAME> includes 30 modified files
Filename Aptfile
Filename LICENSE
Filename app.json
Filename .gitignore
Filename Procfile
Filename README.md
Filename app.py
   Lines added: 710
   Lines removed: 0
Filename dash-logo.png
Filename resizing_script.js
Filename s1.css
Filename styles.css
Filename controls.py
   Lines added: 145
   Lines removed: 0
Filename Oil_and_Gas_Annual_Production__1985_-_2000.csv
Filename Oil_and_Gas_Annual_Production__Beginning_2001.csv
Filename OilandGasMetadata.html
Filename dash-logo copy.png
Filename points.pkl
Filename wellspublic.csv
Filename explore.ipynb
Filename requirements.txt
Filename animated1.gif
Filename screenshot1.png
Filename screenshot2.png
Filename screenshot3.png
Filename apps_directory_mapping.py
   Lines added: 1
   Lines removed: 0
Filename deploy
Filename init.R
Filename predeploy.py
   Lines ad[...]<jupyter_text>Generate charts with the repo info<jupyter_code>repo_dir = "https://github.com/manuwhs/traphing"
repo_mining = RepositoryMining(repo_dir)
commits = repo_mining.traverse_commits()
data = {"Commit Timestamp": [], "Author":[],
        "Added Python lines":[], "Removed Python lines":[], "Modified files": []}

for commit in commits:
    data["Author"].append(commit.author.name)
    data["Commit Timestamp"].append(commit.committer_date)
    data["Modified files"].append(len(commit.modifications))
    data["Added Python lines"].append(0)
    data["Removed Python lines"].append(0)
    
    for modification in commit.modifications:
        if modification.filename[-3:] == ".py":
            data["Added Python lines"][-1] += modification.added
            data["Removed Python lines"][-1] += modification.removed

df = pd.DataFrame.from_dict(data).set_index("Commit Timestamp").sort_index()
df
import datetime as dt
dates = [dt.datetime(year = x.year, month = x.month, day = x.day) for x in df.index]


fig = go.Figure()
fig.add_trace(go.Bar(x=dates,
                y= df["Added Python lines"],
                name='Added lines',
                marker_color='rgb(55, 83, 109)',
                # width = 10000000,
                ))
fig.add_trace(go.Bar(x=dates,
                y= df["Removed Python lines"],
                name='Removed lines',
                marker_color='rgb(26, 118, 255)'
                ))

fig.update_layout(
    title='Repo modifications',
    xaxis_tickfont_size=14,
    yaxis=dict(
        title='Lines added',
        titlefont_size=16,
        tickfont_size=14,
    ),
    legend=dict(
        x=0,
        y=1.0,
        bgcolor='rgba(255, 255, 255, 0)',
        bordercolor='rgba(255, 255, 255, 0)'
    ),
    barmode='group',
    bargap=0.15, # gap between bars of adjacent location coordinates.
    bargroupgap=0.1 # gap between bars of the same location coordinate.
)

figure = go.FigureWidget(fig)
figure
import sys
sys.path.insert(0, "../")
from plotting import holidays
holidays(df)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12310151878497202, 'fraction_numerical': 0.028936850519584334, 'mean_word_length': 4.3061916878710775, 'pattern_counts': {'":': 8, '<': 30, '<?xml version=': 0, '>': 30, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '531910', 'license': 'MIT', 'n_tokens_mistral': 2115, 'n_tokens_neox': 1948, 'n_words': 462, 'path': 'apps/NerdCave/notebooks/MQQ4 trials.ipynb', 'repo_name': 'manuwhs/dashboards', 'size': '6271'}