<jupyter_start><jupyter_text>导入数据<jupyter_code>df_dga=pd.read_csv("all_dga.txt", sep=' ',names=['domain','label'],header=None)
#df_dga.head()
df_dga.shape
df = pd.read_csv("top-1m.csv", names=['domain'],encoding='utf-8')
df['label']=0
df=df.sample(frac=1)
df = df.sample(frac=1).reset_index(drop=True)
df.head()<jupyter_output><empty_output><jupyter_text>特征提取<jupyter_code>featureSet = pd.DataFrame(columns=('d','entropy','vowel_ratio','digits_ratio',\
'repeat_letter','consecutive_digits_ratio','2-gram','3-gram','tld','len','label'))
from urllib.parse import urlparse
import tldextract
def getFeatures(domain, label): 
    result = []
    url = str(domain)
    result.append(domain)
    result.append(count_entropy(domain))
    result.append(count_vowel_ratio(domain))
    result.append(count_digits_ratio(domain))
    result.append(count_repeat_letter(domain))    
    result.append(consecutive_digits_ratio(domain))
    result.append(count_Bigram(domain))
    result.append(count_Trigram(domain))
    result.append(count_TLD(domain))
    result.append(len(domain))
    result.append(label)
    return result
                  
 
for i in range(5000):
    features = getFeatures(df["domain"].loc[i], df["label"].loc[i])    
    featureSet.loc[i] = features
for i in range(5000):
    features = getFeatures(df_dga["domain"].loc[i], df_dga["label"].loc[i])    
    featureSet.loc[i+5000] = features
featureSet.head(10)<jupyter_output><empty_output><jupyter_text>训练<jupyter_code>X = featureSet.drop(['d','label'],axis=1).values
y = featureSet['label'].values
X_train, X_test, y_train, y_test = train_test_split(X, y ,test_size=0.2)
from sklearn import tree, linear_model
clf = tree.DecisionTreeClassifier(max_depth=10)
clf.fit(X_train,y_train.astype('int'))
score = clf.score(X_test,y_test.astype('int'))
print ("%s"%(score))
    
from sklearn.metrics import confusion_matrix
res = clf.predict(X)
#recall = confusion_matrix(y, res)
mt=confusion_matrix(y.astype('int'), res)
print("%f,%f" % (mt[0][0],mt[0][1]))
print("%f,%f" % (mt[1][0],mt[1][1]))
print("Accuracy rate : %f %%" % (((mt[0][0]+mt[1][1]) / float(mt.sum()))*100))
print('Recall rate : %f %%' % ( (mt[0][0] / float(mt.sum(axis=0)[0])*100)))<jupyter_output>4507.000000,493.000000
183.000000,4817.000000
Accuracy rate : 93.240000 %
Recall rate : 96.098081 %<jupyter_text>test<jupyter_code>#输入域名判断是否为DGA域名
test = 'baidu.com'
result = pd.DataFrame(columns=('d','entropy','vowel_ratio','digits_ratio',\
'repeat_letter','consecutive_digits_ratio','2-gram','3-gram','tld','len','label'))
results = getFeatures(test, 0)
result.loc[0] = results
result = result.drop(['d', 'label'], axis = 1).values
if(clf.predict(result)):
    print("this is a DGA url" )
else:
    print("this is a benign url" )
#输入域名判断是否为DGA域名
test = 'gllcsbqrbfch.com'
result = pd.DataFrame(columns=('d','entropy','vowel_ratio','digits_ratio',\
'repeat_letter','consecutive_digits_ratio','2-gram','3-gram','tld','len','label'))
results = getFeatures(test, 0)
result.loc[0] = results
result = result.drop(['d', 'label'], axis = 1).values
if(clf.predict(result)):
    print("this is a DGA url" )
else:
    print("this is a benign url" )<jupyter_output>this is a DGA url
{'fraction_non_alphanumeric': 0.19247648902821315, 'fraction_numerical': 0.03510971786833856, 'mean_word_length': 6.561611374407583, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '88981', 'license': 'MIT', 'n_tokens_mistral': 1312, 'n_tokens_neox': 1251, 'n_words': 174, 'path': 'Homework/2019/Task4/11/code/DecisionTree.ipynb', 'repo_name': 'ohhuola/Data-Mining-for-Cybersecurity', 'size': '3190'}