<jupyter_start><jupyter_text>DRIVE MOUNTER<jupyter_code>from google.colab import drive
drive.mount('/content/drive')<jupyter_output>Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly

Enter your authorization code:
··········
Mounted at /content/drive<jupyter_text>DATA EXTRACTION<jupyter_code>import pandas as pd
import numpy as np
import json
import csv
import bz2
import os
import sys
def extractor(fle,writer,up_l,down_l):
  total=[]
  source_file=bz2.BZ2File(fle,'r')
  prev='ip'
  temp_l=[]
  for i,line in enumerate(source_file):
    if i<down_l: 
      continue
    if i==up_l: 
      break
    print(i)
    data=json.loads(line)
    if 'from' not in data.keys() or 'dst_addr' not in data.keys():
      continue
    src_addr=data['from']
    dest_addr=data["dst_addr"]
    temp=data['result']
    for iter in temp:
      if "result" not in iter.keys():
        continue
      data_temp=iter['result']
      for ext in data_temp:
        if "from" in ext.keys():
          if prev==str(ext["from"]): continue
          temp_l.append(ext["from"])
          prev=str(ext["from"])
    if len(temp_l)==0:
      continue
    writer.write(str(src_addr)+","+str(dest_addr)+"\n")
    total.append(temp_l)  
    temp_l=[]
  down_l=up_l
  up_l=up_l+20000
  if i<down_l-1:
    flag=1
  else:
    flag=0

  return total,down_l,up_l,flag  
          
def merge(name_w,total):
  df=pd.read_csv(name_w)
  df['Hops']=total
  df.to_csv(name_w,index=False)
def caller_ext():
  Path="drive/My Drive/iitkgp_summer_project"
  files=os.listdir(Path)
  up_l=20000
  down_l=0
  i=files[0]
  fle=Path+"/"+i
  flag=0
  name=Path+'/'+i[-7:-4]
  if os.path.exists(name):
    print("error")
    exit(0)
  else:
    os.mkdir(name)

  counter_n=1
  while(flag==0):
    
    name_w=name+'/IP_rec'+i[-7:-4]+"_"+str(counter_n)+".csv"
    print(name_w)
    text=open(name_w,"w")

    text.write("source,dest\n")
    text.close()
    writer=open(name_w,'a')

    total,down_l,up_l,flag=extractor(fle,writer,up_l,down_l)
    writer.close()
    merge(name_w, total)  
    counter_n+=1
caller_ext()<jupyter_output><empty_output><jupyter_text>DATA PREPROCESSING<jupyter_code>import pandas as pd
import json
import ast
import os
def preprocess(fle,path,fldr):
  Path="drive/My Drive/iitkgp_summer_project"
  df=pd.read_csv(fldr+'/'+fle)
  z=0
  df_l=df.values.tolist()
  data={}
  while z<len(df):
    lst=ast.literal_eval(df_l[z][2])
    k=df_l[z][0]
    lst.insert(0,k)
    lst.append(df_l[z][1])
    j=0
    while j<len(lst)-1:
      if data.get(lst[j])==None:
        data[lst[j]]=[lst[j+1]]
      else:
        ips=data.get(lst[j])
        ips.append(lst[j+1])
        data[lst[j]]=ips
        #print(ips)
      j+=1
    
    if data.get(lst[j])==None:
      data[lst[j]]=[]
    z+=1
  dump=path+"/"+fle[:-4]+'.json'
  print(dump)
  if os.path.exists(dump):
    os.remove(dump)
  with open(dump, 'w') as fp:
    json.dump(data, fp)
def preprocess_caller(fldr):
  files=os.listdir(fldr)
  p=fldr+'_json'
  if os.path.exists(p):
    print(error)
    exit(0)
  else:
    os.mkdir(p)
  for i in files:
    #for j in i:
      preprocess(i,p,fldr)



    
Path="drive/My Drive/iitkgp_summer_project"
files=os.listdir(Path)
up_l=20000
down_l=0
i=files[0]
fle=Path+"/"+i
#flag=0
name=Path+'/'+i[-7:-4]
preprocess_caller(name)<jupyter_output>drive/My Drive/iitkgp_summer_project/300_json/IP_rec300_1.json
drive/My Drive/iitkgp_summer_project/300_json/IP_rec300_2.json
drive/My Drive/iitkgp_summer_project/300_json/IP_rec300_3.json
drive/My Drive/iitkgp_summer_project/300_json/IP_rec300_4.json
drive/My Drive/iitkgp_summer_project/300_json/IP_rec300_5.json
drive/My Drive/iitkgp_summer_project/300_json/IP_rec300_6.json
drive/My Drive/iitkgp_summer_project/300_json/IP_rec300_7.json
drive/My Drive/iitkgp_summer_project/300_json/IP_rec300_8.json
drive/My Drive/iitkgp_summer_project/300_json/IP_rec300_9.json
drive/My Drive/iitkgp_summer_project/300_json/IP_rec300_10.json
drive/My Drive/iitkgp_summer_project/300_json/IP_rec300_11.json
drive/My Drive/iitkgp_summer_project/300_json/IP_rec300_12.json
drive/My Drive/iitkgp_summer_project/300_json/IP_rec300_13.json
drive/My Drive/iitkgp_summer_project/300_json/IP_rec300_14.json
drive/My Drive/iitkgp_summer_project/300_json/IP_rec300_15.json
drive/My Drive/iitkgp_summer_project/300_json/IP_[...]<jupyter_text>GRAPH CREATION<jupyter_code>import json
import pickle
import os
class Graph:
  def __init__(self,g_dict=None):
    if g_dict is None:
      g_dict=[]
    self.g_dict=g_dict

  def getVertices(self):
    return list(self.g_dict.keys())
  
  def findedge(self):
    edge=[]
    for v in self.g_dict:
      for nxt_v in self.g_dict[v]:
        if (nxt_v,v) not in edge:
          edge.append((nxt_v,v))
          print(len(edge))
    return edge
  
  def edges(self):
      return self.findedge()

  def addVertex(self,v):
    if v not in self.g_dict:
      self.g_dict[v]=[]

  def addedge(self,edge):
    edge=set(edge)
    (v1,v2)=tuple(edge)
    if v1 in self.g_dict:
      self.g_dict[v1].append(v2)
    else:
      self.g_dict[v1]=[v2]

def create_graph(fle,path,fldr):
  Path="drive/My Drive/iitkgp_summer_project"
  f=open(fldr+'/'+fle,"r")
  data=json.loads(f.read())
  graph_elements=data
  g=Graph(graph_elements)
  edge=g.edges()
  ver=g.getVertices()
  
  ext_1=fle[:-5]+'_graph_detail.txt'
  file_d=path+'/'+ext_1
  print(file_d)
  detail=open(file_d,'w')
  detail.write(str(ver)+'\n')
  detail.write(str(edge))
  detail.close()

def graph_caller(fldr):
  files=os.listdir(fldr)
  p=fldr[:-5]+'_graph_detail'
  print(p)
  k=0
  if os.path.exists(p):
    print("error")
    exit(0)
  else:
    os.mkdir(p)
    print("created")
  for i in files:
    #for j in i:
    create_graph(i,p,fldr)

Path="drive/My Drive/iitkgp_summer_project"
files=os.listdir(Path)
up_l=20000
down_l=0
i=files[0]
fle=Path+"/"+i
#flag=0
name=Path+'/'+i[-7:-4]+'_json'
print(name)
graph_caller(name)<jupyter_output>[1;30;43mStreaming output truncated to the last 5000 lines.[0m
55372
55373
55374
55375
55376
55377
55378
55379
55380
55381
55382
55383
55384
55385
55386
55387
55388
55389
55390
55391
55392
55393
55394
55395
55396
55397
55398
55399
55400
55401
55402
55403
55404
55405
55406
55407
55408
55409
55410
55411
55412
55413
55414
55415
55416
55417
55418
55419
55420
55421
55422
55423
55424
55425
55426
55427
55428
55429
55430
55431
55432
55433
55434
55435
55436
55437
55438
55439
55440
55441
55442
55443
55444
55445
55446
55447
55448
55449
55450
55451
55452
55453
55454
55455
55456
55457
55458
55459
55460
55461
55462
55463
55464
55465
55466
55467
55468
55469
55470
55471
55472
55473
55474
55475
55476
55477
55478
55479
55480
55481
55482
55483
55484
55485
55486
55487
55488
55489
55490
55491
55492
55493
55494
55495
55496
55497
55498
55499
55500
55501
55502
55503
55504
55505
55506
55507
55508
55509
55510
55511
55512
55513
55514
55515
55516
55517
55518
55519
55520
55521
55522
55523
55524
55525
55526
55527[...]<jupyter_text>PRINTING GRAPH<jupyter_code>import networkx as nx
import matplotlib.pyplot as plt
G=nx.Graph()
G.add_nodes_from(ver)
G.add_edges_from(edge)
plt.figure(1,figsize=(70,70)) 
nx.draw_networkx(G,with_labels=False,node_size=200,node_shape='o',style='solid',linewidths=10.0)
plt.savefig("drive/My Drive/iitkgp_summer_project/fig_1.png")
#plt.show()<jupyter_output><empty_output><jupyter_text>ANALYSIS FROM GRAPH<jupyter_code>!pip install cymruwhois

import json
import pandas as pd
from cymruwhois import Client
import socket
import os
Path="drive/My Drive/iitkgp_summer_project"
fle='/IP_rec300.json'
def get_number_of_connections(fle,file_w,fldr):
  #Path="drive/My Drive/iitkgp_summer_project"
  f=open(fle,"r")
  data=json.loads(f.read())
  writer=open(file_w,'a')
  for i in data.keys():
    lst=data.get(i)
    l=len(lst)
    writer.write(i+","+str(l)+"\n")
  writer.close()

def count_executer(fle,p,fldr):
  Path="drive/My Drive/iitkgp_summer_project"
  
  
  file_j=fldr+"/"+fle
  ext=p+"/"+fle[:-5]
  #ext=fldr[:-5]
  file_w=ext+'_IP_details.csv'
  writer=open(file_w,'w')
  writer.write('IP,connection\n')
  writer.close()

  get_number_of_connections(file_j,file_w,fldr)
def exec_caller(fldr):
  files=os.listdir(fldr)
  ext=fldr[:-5]
  p=ext+'_IP_detail'
  print(p)
  if os.path.exists(p):
    print("error")
    exit(0)
  else:
    os.mkdir(p)
    print("Created")
  for i in files:
    #for j in i:
      count_executer(i,p,fldr)


Path="drive/My Drive/iitkgp_summer_project"
files=os.listdir(Path)
up_l=20000
down_l=0
i=files[0]
fle=Path+"/"+i
#flag=0
name=Path+'/'+i[-7:-4]+'_json'
print(name)
exec_caller(name)
fle='/IP_rec300.csv'
df2=pd.read_csv(Path+fle)
print(df2.head())
def create_source(fle):
  df2=pd.read_csv(fle)
  print(df2.head())
  i=0
  df_l=df2.values.tolist()
  source={}
  while i<len(df_l):
    if source.get(df_l[i][0]) is None:
      source[df_l[i][0]]=0
    i+=1
  return source

def Mark_ip(fle_ip,fle):
  k=0
  status=[]
  source=create_source(fle)
  while k<len(df):
    print(k)
    if df.iloc[k]['connection']==0:
      status.append('dest')
    elif source.get(df.iloc[k]['IP']) is not None:
      status.append('sender')
    else:
      status.append('Intermediate')
  
    k+=1

  df['Status']=status
  df.to_csv(fle_ip,index=False)


def Mark_caller(fldr_ip,fldr_n):
  file_ip=os.listdir(fldr_ip)
  file_n=os.listdir(fldr_n)

  for i in file_ip,file_n:
    print(i)
    print(j)

    Mark_ip(i,j)

df=pd.read_csv(file_w)
c=Client()
asn=[]
as_name=[]

j=48870
k=0
while k<j:
  asn.append(df.iloc[k]['AS_num'])
  as_name.append(df.iloc[k]['AS_name'])
  k+=1
while j<len(df):
  print(j)
  try:
    ip=df.iloc[j]['IP']
  
    print("A")
    r=c.lookup(ip)
    print(r.asn)
    print(r.owner)
    asn.append(r.asn)
    as_name.append(r.owner)
  except:
    print("error")
    asn.append("NA")
    as_name.append("NA")

  j+=1

df['AS_num']=asn
df['AS_name']=as_name
df.to_csv(file_w,index=False)

print(len(asn))
df=pd.read_csv(file_w)
c=Client()
hn=[]

j=0
while j<len(df):
  print(j)
  ip=df.iloc[j]['IP']
  try:
    h=socket.gethostbyaddr(ip)
    hn.append(h[0])
  except:
    hn.append('NA')
  j+=1
df['host']=hn

df.to_csv(file_w,index=False)<jupyter_output>[1;30;43mStreaming output truncated to the last 5000 lines.[0m
50524
50525
50526
50527
50528
50529
50530
50531
50532
50533
50534
50535
50536
50537
50538
50539
50540
50541
50542
50543
50544
50545
50546
50547
50548
50549
50550
50551
50552
50553
50554
50555
50556
50557
50558
50559
50560
50561
50562
50563
50564
50565
50566
50567
50568
50569
50570
50571
50572
50573
50574
50575
50576
50577
50578
50579
50580
50581
50582
50583
50584
50585
50586
50587
50588
50589
50590
50591
50592
50593
50594
50595
50596
50597
50598
50599
50600
50601
50602
50603
50604
50605
50606
50607
50608
50609
50610
50611
50612
50613
50614
50615
50616
50617
50618
50619
50620
50621
50622
50623
50624
50625
50626
50627
50628
50629
50630
50631
50632
50633
50634
50635
50636
50637
50638
50639
50640
50641
50642
50643
50644
50645
50646
50647
50648
50649
50650
50651
50652
50653
50654
50655
50656
50657
50658
50659
50664
50665
50666
50667
50668
50669
50670
50671
50672
50673
50674
50675
50676
50677
50678
50679
50680
50681
50682
50683[...]
{'fraction_non_alphanumeric': 0.13512110726643597, 'fraction_numerical': 0.1629757785467128, 'mean_word_length': 4.4073900841908324, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 21, 'https://': 1, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '315333', 'license': 'MIT', 'n_tokens_mistral': 6360, 'n_tokens_neox': 5185, 'n_words': 894, 'path': 'RIPE_Extractor_modular_2.ipynb', 'repo_name': 'abr-98/COVID-19_effect_networks_RIPE', 'size': '11560'}