<jupyter_start><jupyter_text>Analysis of Zeek logs<jupyter_code>import pandas as pd
import json
import gzip
import glob
import os
import seaborn as sb

%matplotlib inline
import matplotlib.pyplot as plt<jupyter_output><empty_output><jupyter_text>Extract gz files<jupyter_code>path = './data/2020-06-03/' 
all_files = glob.glob(os.path.join(path,"ssh.*.log.gz"))
df_list = []

for jsonfilename in all_files:
    with gzip.GzipFile(jsonfilename, 'r') as json_file:
        for line in json_file:
            json_data = json.loads(line)
           
            timestamp = json_data['ts']
            source_ip = json_data['id.orig_h']
            source_port = json_data['id.orig_p']
            destination_ip = json_data['id.resp_h']
            destination_port = json_data['id.resp_p']
            
            if 'auth_success' in json_data:
                authentication_success = json_data['auth_success']
            else:
                authentication_success = 'NaN'
            authentication_attempts = json_data['auth_attempts']
            
            if 'client' in json_data:
                client = json_data['client']
            else:
                client = 'NaN'
            
            server = json_data['server']
            

            df_list.append({'timestamp':pd.to_datetime(timestamp),
                           'source_ip': str(source_ip),
                           'source_port': int(source_port),
                           'destination_ip':str(destination_ip),
                           'destination_port':int(destination_port),
                           'authentication_success': bool(authentication_success),
                           'authentication_attempts':int(authentication_attempts),
                           'client':str(client),
                           'server':str(server)})<jupyter_output><empty_output><jupyter_text>Create data frame from dictionary<jupyter_code>ssh_df = pd.DataFrame(df_list, columns =['timestamp', 'source_ip', 'source_port', 'destination_ip', 
                                          'destination_port', 'authentication_success', 'authentication_attempts',
                                         'client', 'server'])
ssh_df.head()
ssh_df.shape<jupyter_output><empty_output><jupyter_text>*Store dataframe to csv*<jupyter_code>ssh_df.to_csv('./data/2020-06-03/ssh_logs.csv', index = False)
ssh_df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 9404 entries, 0 to 9403
Data columns (total 9 columns):
timestamp                  9404 non-null datetime64[ns]
source_ip                  9404 non-null object
source_port                9404 non-null int64
destination_ip             9404 non-null object
destination_port           9404 non-null int64
authentication_success     9404 non-null bool
authentication_attempts    9404 non-null int64
client                     9404 non-null object
server                     9404 non-null object
dtypes: bool(1), datetime64[ns](1), int64(3), object(4)
memory usage: 597.0+ KB<jupyter_text>Attackers Connections<jupyter_code>df_merged = pd.read_csv('./data/2020-06-03/ssh_logs.csv')
df_merged.head()<jupyter_output><empty_output><jupyter_text>What are Attackers Source Ip's<jupyter_code>plt.figure(figsize=(12,6))
sb.countplot(data=df_merged, y='source_ip', order=df_merged.source_ip.value_counts().iloc[:5].index,
            orient = 'h');
sb.set_style("dark")
plt.ylabel('Source IPs')
plt.xlabel('Count');<jupyter_output><empty_output><jupyter_text>Attackers Source Port<jupyter_code>plt.figure(figsize=(12,5))
sb.countplot(data=df_merged, x='source_port', order=df_merged.source_port.value_counts().iloc[:5].index,
            orient = 'h');
sb.set_style("dark")
plt.title('Attacker HTTP Source Port')
plt.ylabel('Count')
plt.xlabel('Source Ports');<jupyter_output><empty_output><jupyter_text>Attackers Destination IP<jupyter_code>#plt.figure(figsize=(6,6))
sb.countplot(data=df_merged, x='destination_ip', order=df_merged.destination_ip.value_counts().iloc[:5].index,
            orient = 'h');
sb.set_style("whitegrid")
plt.ylabel('Destination IPs')
plt.xlabel('Count');<jupyter_output><empty_output><jupyter_text>Attackers Destination PORTs<jupyter_code>plt.figure(figsize=(12,5))
sb.countplot(data=df_merged, x='destination_port', order=df_merged.destination_port.value_counts().iloc[:5].index,
            orient = 'h');
sb.set_style("dark")
plt.title('Attacker HTTP Destination Port')
plt.ylabel('Count')
plt.xlabel('Destination Ports');<jupyter_output><empty_output><jupyter_text>What are the attacker SSH authentication successful rate?<jupyter_code>plt.figure(figsize=(6,6))
plt.pie(sum_auth, labels= sum_auth.index, autopct='%1.1f%%')
plt.title('Authentication Sucess Rate')
plt.show();<jupyter_output><empty_output><jupyter_text>What are the number of authentication attempts?<jupyter_code>plt.figure(figsize=(12,5))
sb.countplot(data=df_merged, y='authentication_attempts', order=df_merged.authentication_attempts.value_counts().iloc[:5].index,
            orient = 'h');
sb.set_style("dark")
plt.title('Authentication Attemps')
plt.ylabel('No of authentication attempts')
plt.xlabel('Count');<jupyter_output><empty_output><jupyter_text>Investigate further authentication attempts 0<jupyter_code>df_merged.query('authentication_attempts == 0' )
df_merged.query('authentication_attempts == 0 and authentication_success == True' )<jupyter_output><empty_output><jupyter_text>What are the client SSH version?<jupyter_code>df_merged.replace(to_replace="SSH-2.0-PuTTY", value ="SSH-2.0-PUTTY", inplace = True)
plt.figure(figsize=(12,5))
sb.countplot(data=df_merged, y='client', order=df_merged.client.value_counts().iloc[:5].index,
            orient = 'h');
sb.set_style("dark")
plt.title('SSH Client Version')
plt.ylabel('SSH Version')
plt.xlabel('Count');<jupyter_output><empty_output><jupyter_text>What are the SSH Server Versions<jupyter_code>df_merged['server'].value_counts()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1284326858673811, 'fraction_numerical': 0.020763563295378432, 'mean_word_length': 3.7860576923076925, 'pattern_counts': {'":': 0, '<': 57, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '528740', 'license': 'MIT', 'n_tokens_mistral': 2064, 'n_tokens_neox': 1932, 'n_words': 299, 'path': 'ZeekSSHLogsAnalysis.ipynb', 'repo_name': 'sehrixhshafeeq/Zeek-Logs-Analysis', 'size': '5972'}