<jupyter_start><jupyter_text>DNS TXT records analysisIn this notebook we'll take a look into TXT records statistics. We'll use data generated by shell script located in `src/txt_analysis/monthly_statistics.sh`. To get the data in a usable form, we download it first from HDFS by issuing following command:```hdfs dfs -copyToLocal 201*results.txt ~```Then we just run `src/combine.sh` script.We now have `201YMMDD_results.json` files that we can use for presentation.We put them in `data/txt_analysis_results` directory.<jupyter_code>import os
import json
import re
import matplotlib.pyplot as plt
import matplotlib.colors as clrs
import numpy as np
from itertools import cycle

file_location = (
    os.path.normcase(
        os.path.join(os.getcwd(), "../data/txt_analysis_results")
    )
)

data_points = []
for filename in os.listdir(file_location):
    with open(os.path.join(file_location, filename), "r") as file:
        data_points.append(eval(file.read()))
        
def convert_LD_to_DL(dictionaries):
    dl = {}
    for key, value in dictionaries[0].items():
        if type(value) is not dict:
            dl[key] = get_summary_for_from(key, dictionaries)
        else:
            dl[key] = convert_LD_to_DL([entry[key] for entry in dictionaries])
    return dl

def get_summary_for_from(key, list_of_dictionaries):
    summary = np.array([])
    for result in list_of_dictionaries:
        summary = np.append(summary, result[key])
    return summary

def sum_over_children(dictionary):
    summary = np.zeros(len(data_points))
    for key, value in dictionary.items():
        if type(value) is not dict:
            summary += value
        else:
            summary += sum_over_children(value)
    return summary

def sum_over_children_single(dictionary):
    summary = 0
    for key, value in dictionary.items():
        if type(value) is not dict:
            summary += value
        else:
            summary += sum_over_children_single(value)
    return summary
    
summary = convert_LD_to_DL(data_points)
labels = summary["day"]

class Line:
    def __init__(self, points, color, info):
        self.points = points
        self.color = color
        self.info = info

def make_a_time_graph(title, data, labels=labels):
    fig, ax = plt.subplots()
    prepare_basic_line_graph(title)
    for entry in data:
        plt.plot(labels, entry.points, entry.color, label=entry.info)
    ax.legend(loc='center right', shadow=True, bbox_to_anchor=(1.2, 0.5))
    plt.show()
    
def prepare_basic_line_graph(title):
    fig_size = plt.rcParams["figure.figsize"]
    fig_size[0] = 12
    fig_size[1] = 8
    plt.rcParams["figure.figsize"] = fig_size
    plt.xlabel('Month')
    plt.xticks(rotation=60)
    plt.ylabel('Number')
    plt.grid()
    plt.title(title)
    
def make_a_pie_chart(title, values, labels):
    plt.title(title)
    plt.pie(values, labels=labels)
    plt.show()<jupyter_output><empty_output><jupyter_text>TXT records in the whole dataset<jupyter_code>prepare_basic_line_graph('Number of records in researched months')
plt.plot(labels, summary["total"], color='blue')
plt.plot(labels, summary["txt"], color='purple')
plt.plot(labels, summary["correct_txt"], color='green')

plt.show()

percentages = summary["txt"] / summary["total"]
print("Highest percentage: {:.2%}".format(percentages.max()))<jupyter_output><empty_output><jupyter_text>As we can see, TXT records make up only a small percentage of all records.Let's look at incorrect values.<jupyter_code>lines = [Line(summary["txt"] - summary["correct_txt"], 'red', "Empty records")]
make_a_time_graph('Number of empty TXT records in researched months', lines)<jupyter_output><empty_output><jupyter_text>UsagesLet's see now, what are the most common uses for TXT records.<jupyter_code>lines = []
itr = 0
for key, value in summary["specific"].items():
    if type(value) is dict:
        lines.append(Line(sum_over_children(value), 'C'+str(itr), key))
    else:
        lines.append(Line(value, 'C' + str(itr), key))
    itr += 1
make_a_time_graph('Different usages of TXT records', lines)
values = []
labels = []
for key, value in data_points[-1]["specific"].items():
    labels.append(key)
    if type(value) is dict:
        values.append(sum_over_children_single(value))
    else:
        values.append(value)
values, labels = (list(t) for t in zip(*sorted(zip(values, labels))))

labels.append("other")
values.append(data_points[-1]["correct_txt"] - sum(values))
make_a_pie_chart('Different usages of TXT records for 2017-12-01', values, labels)<jupyter_output><empty_output><jupyter_text>As we can see, mail security, domain identifiers and just keys make up most of the collected TXT records. Let's see the components of mail security:<jupyter_code>print (data_points[-1]["specific"]["mail-security"])<jupyter_output><empty_output><jupyter_text>Presenting a pie chart makes no sense here, obviously.  Domain identifiersThe big part of TXT records is domain identifiers group. The proportions in this group look as follows:<jupyter_code>values = []
labels = []
itr = 0
for key, value in data_points[-1]["specific"]["domain-identifiers"].items():
    labels.append(key)
    values.append(value)
    itr += 1
values, labels = (list(t) for t in zip(*sorted(zip(values, labels))))

make_a_pie_chart('Different usages of TXT records for 2017-12-01', values, labels)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12049758633494244, 'fraction_numerical': 0.008169327887114742, 'mean_word_length': 4.286555446516192, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '463398', 'license': 'MIT', 'n_tokens_mistral': 1767, 'n_tokens_neox': 1710, 'n_words': 459, 'path': 'notebooks/04-txt-usage.ipynb', 'repo_name': 'sheikhomar/mako', 'size': '5386'}