<commit_before>#################################################################################################
# Name: Checks (part for Shakedown)
# Author: <NAME> (nxkennedy)
#
# Description: Script to analyze a directory or file for what may be malicious content. Writes
#              findings to csv to assist manual analysis.
#       Currently checks for:
#                   * IP Addresses
#                   * URLs
#                   * Shellcode
#                   * URL Encoding
#                   * HTML Encoding
#
# Use Case: Vetting newly downloaded scripts for network usage
#
# Usage: python shakedown.py <directory_or_file>
#
#################################################################################################


import re

# Search for ips
def check_for_ips(string):
    ip = re.compile(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}")
    match = ip.search(string)

    if match:
        return match


# Search for URLs
def check_for_urls(string):
    url =  re.compile(r"""((?:[a-z][\w-]+:(?:/{1,3}|[a-z0-9%])|www\d{0,‌​3}[.]|[a-z0-9.\-]+[.‌​][a-z]{2,4}/)(?:[^\s‌​()<>]+|(([^\s()<‌​>]+|(([^\s()<>]+‌​)))*))+(?:&#‌​40;([^\s()<>]+|((‌​;[^\s()<>]+)))*&‌​#41;|[^\s`!()[&#‌​93;{};:'".,<>?«»“”‘’‌​]))""", re.DOTALL)
    match = url.search(string)

    if match:
        return match


# Search for possible shellcode
def check_for_encoding_shellcode(string):
    shellcode = re.compile(r"[0%\\][xX][0-9a-fA-F]+")
    match = shellcode.search(string)

    if match:
        return match


# Search for strings with URL encoding
def check_for_encoding_url(string):
    url = re.compile(r"%[0-9a-fA-F]")
    match = url.search(string)<commit_msg>added ipv6 regex<commit_after>#################################################################################################
# Name: Checks (part for Shakedown)
# Author: <NAME> (nxkennedy)
#
# Description: Script to analyze a directory or file for what may be malicious content. Writes
#              findings to csv to assist manual analysis.
#       Currently checks for:
#                   * IP Addresses
#                   * URLs
#                   * Shellcode
#                   * URL Encoding
#                   * HTML Encoding
#
# Use Case: Vetting newly downloaded scripts for network usage
#
# Usage: python shakedown.py <directory_or_file>
#
#################################################################################################


import re

# Search for ips
def check_for_ips(string):
    ip = re.compile(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}")
    ipv6 = re.compile('^(?:(?:[0-9A-Fa-f]{1,4}:){6}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|::(?:[0-9A-Fa-f]{1,4}:){5}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){4}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){3}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,2}[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){2}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,3}[0-9A-Fa-f]{1,4})?::[0-9A-Fa-f]{1,4}:(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,4}[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,5}[0-9A-Fa-f]{1,4})?::[0-9A-Fa-f]{1,4}|(?:(?:[0-9A-Fa-f]{1,4}:){,6}[0-9A-Fa-f]{1,4})?::)$')
    match = ip.search(string)
    match6 = ipv6.search(string)

    if match or match6:
        return True


# Search for URLs
def check_for_urls(string):
    #url =  re.compile(r"""((?:[a-z][\w-]+:(?:/{1,3}|[a-z0-9%])|www\d{0,‌​3}[.]|[a-z0-9.\-]+[.‌​][a-z]{2,4}/)(?:[^\s‌​()<>]+|(([^\s()<‌​>]+|(([^\s()<>]+‌​)))*))+(?:&#‌​40;([^\s()<>]+|((‌​;[^\s()<>]+)))*&‌​#41;|[^\s`!()[&#‌​93;{};:'".,<>?«»“”‘’‌​]))""", re.DOTALL)
    #@TODO current url parsing reads IPv6 addresses as URLS because of "protocol://" colon detection. Needs fixing.
    url = re.compile(r"""(?i)\b((?:[a-z][\w-]+:(?:/{1,3}|[a-z0-9%])|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:'".,<>?«»“”‘’]))""")
    match = url.search(string)

    if match:
        return match


# Search for possible shellcode
def check_for_encoding_shellcode(string):
    shellcode = re.compile(r"[0%\\][xX][0-9a-fA-F]+")
    match = shellcode.search(string)

    if match:
        return match


# Search for strings with URL encoding
def check_for_encoding_url(string):
    url = re.compile(r"%[0-9a-fA-F]")
    match = url.search(string)
{'fraction_non_alphanumeric': 0.4203119056675542, 'fraction_numerical': 0.09376188664891594, 'mean_word_length': 3.8694444444444445, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2918115', 'n_tokens_mistral': 2627, 'n_tokens_neox': 2532, 'n_words': 253}