<jupyter_start><jupyter_text>Author : <NAME> Topic : Data Science For Security Analysis Email : <EMAIL>  Identify files via file extensions A regular expression to check for file extensions.  Note: This approach is not recommended for thorough limitation of file types (parse the file header instead). However, this regex is still a useful alternative to e.g., a Python's `endswith` approach for quick pre-filtering for certain files of interest.<jupyter_code>pattern = r'(?i)(\w+)\.(jpeg|jpg|png|gif|tif|svg)$'

# remove `(?i)` to make regexpr case-sensitive

str_true = ('test.gif', 
            'image.jpeg', 
            'image.jpg',
            'image.TIF'
            )

str_false = ('test.pdf',
             'test.gif.pdf',
             )

for t in str_true:
    assert(bool(re.match(pattern, t)) == True), '%s is not True' %t
for f in str_false:
    assert(bool(re.match(pattern, f)) == False), '%s is not False' %f<jupyter_output><empty_output><jupyter_text>Username validation Checking for a valid user name that has a certain minimum and maximum length.Allowed characters:- letters (upper- and lower-case)- numbers- dashes- underscores<jupyter_code>min_len = 5 # minimum length for a valid username
max_len = 15 # maximum length for a valid username

pattern = r"^(?i)[a-z0-9_-]{%s,%s}$" %(min_len, max_len)

# remove `(?i)` to only allow lower-case letters



str_true = ('user123', '123_user', 'Username')
            
str_false = ('user', 'username1234_is-way-too-long', 'user$34354')

for t in str_true:
    assert(bool(re.match(pattern, t)) == True), '%s is not True' %t
for f in str_false:
    assert(bool(re.match(pattern, f)) == False), '%s is not False' %f<jupyter_output>C:\Users\imsanjoykb\Anaconda3\lib\site-packages\ipykernel_launcher.py:15: DeprecationWarning: Flags not at the start of the expression '^(?i)[a-z0-9_-]{5,15' (truncated)
  from ipykernel import kernelapp as app<jupyter_text>Checking for valid email addresses A regular expression that captures most email addresses.<jupyter_code>pattern = r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)"

str_true = ('<EMAIL>',)
            
str_false = ('<EMAIL>', '@<EMAIL>', '<EMAIL>')

for t in str_true:
    assert(bool(re.match(pattern, t)) == True), '%s is not True' %t
for f in str_false:
    assert(bool(re.match(pattern, f)) == False), '%s is not False' %f<jupyter_output><empty_output><jupyter_text>Check for a valid URL Checks for an URL if a string ...- starts with `https://`, or `http://`, or `www.`- or ends with a dot extension<jupyter_code>pattern = '^(https?:\/\/)?([\da-z\.-]+)\.([a-z\.]{2,6})([\/\w \.-]*)*\/?$'

str_true = ('https://github.com', 
            'http://github.com',
            'www.github.com',
            'github.com',
            'test.de',
            'https://github.com/rasbt',
            'test.jpeg' # !!! 
            )
            
str_false = ('testmailcom', 'http:testmailcom', )

for t in str_true:
    assert(bool(re.match(pattern, t)) == True), '%s is not True' %t

for f in str_false:
    assert(bool(re.match(pattern, f)) == False), '%s is not False' %f<jupyter_output><empty_output><jupyter_text>Checking for numbers  Positive integers<jupyter_code>pattern = '^\d+$'

str_true = ('123', '1', )
            
str_false = ('abc', '1.1', )

for t in str_true:
    assert(bool(re.match(pattern, t)) == True), '%s is not True' %t

for f in str_false:
    assert(bool(re.match(pattern, f)) == False), '%s is not False' %f<jupyter_output><empty_output><jupyter_text>Negative integers<jupyter_code>pattern = '^-\d+$'

str_true = ('-123', '-1', )
            
str_false = ('123', '-abc', '-1.1', )

for t in str_true:
    assert(bool(re.match(pattern, t)) == True), '%s is not True' %t

for f in str_false:
    assert(bool(re.match(pattern, f)) == False), '%s is not False' %f<jupyter_output><empty_output><jupyter_text>All integers<jupyter_code>pattern = '^-{0,1}\d+$'

str_true = ('-123', '-1', '1', '123',)
            
str_false = ('123.0', '-abc', '-1.1', )

for t in str_true:
    assert(bool(re.match(pattern, t)) == True), '%s is not True' %t

for f in str_false:
    assert(bool(re.match(pattern, f)) == False), '%s is not False' %f<jupyter_output><empty_output><jupyter_text>Positive numbers<jupyter_code>pattern = '^\d*\.{0,1}\d+$'

str_true = ('1', '123', '1.234', )
            
str_false = ('-abc', '-123', '-123.0')

for t in str_true:
    assert(bool(re.match(pattern, t)) == True), '%s is not True' %t

for f in str_false:
    assert(bool(re.match(pattern, f)) == False), '%s is not False' %f<jupyter_output><empty_output><jupyter_text>Negative numbers<jupyter_code>pattern = '^-\d*\.{0,1}\d+$'

str_true = ('-1', '-123', '-123.0', )
            
str_false = ('-abc', '1', '123', '1.234', )

for t in str_true:
    assert(bool(re.match(pattern, t)) == True), '%s is not True' %t

for f in str_false:
    assert(bool(re.match(pattern, f)) == False), '%s is not False' %f<jupyter_output><empty_output><jupyter_text>All numbers<jupyter_code>pattern = '^-{0,1}\d*\.{0,1}\d+$'

str_true = ('1', '123', '1.234', '-123', '-123.0')
            
str_false = ('-abc')

for t in str_true:
    assert(bool(re.match(pattern, t)) == True), '%s is not True' %t

for f in str_false:
    assert(bool(re.match(pattern, f)) == False), '%s is not False' %f<jupyter_output><empty_output><jupyter_text>Validating dates Validates dates in `mm/dd/yyyy` format.<jupyter_code>pattern = '^(0[1-9]|1[0-2])\/(0[1-9]|1\d|2\d|3[01])\/(19|20)\d{2}$'

str_true = ('01/08/2014', '12/30/2014', )
            
str_false = ('22/08/2014', '-123', '1/8/2014', '1/08/2014', '01/8/2014')

for t in str_true:
    assert(bool(re.match(pattern, t)) == True), '%s is not True' %t

for f in str_false:
    assert(bool(re.match(pattern, f)) == False), '%s is not False' %f<jupyter_output><empty_output><jupyter_text>12-Hour format<jupyter_code>pattern = r'^(1[012]|[1-9]):[0-5][0-9](\s)?(?i)(am|pm)$'

str_true = ('2:00pm', '7:30 AM', '12:05 am', )
            
str_false = ('22:00pm', '14:00', '3:12', '03:12pm', )

for t in str_true:
    assert(bool(re.match(pattern, t)) == True), '%s is not True' %t

for f in str_false:
    assert(bool(re.match(pattern, f)) == False), '%s is not False' %f<jupyter_output>C:\Users\imsanjoykb\Anaconda3\lib\site-packages\ipykernel_launcher.py:8: DeprecationWarning: Flags not at the start of the expression '^(1[012]|[1-9]):[0-5' (truncated)<jupyter_text>24-Hour format<jupyter_code>pattern = r'^([0-1]{1}[0-9]{1}|20|21|22|23):[0-5]{1}[0-9]{1}$'

str_true = ('14:00', '00:30', )
            
str_false = ('22:00pm', '4:00', )

for t in str_true:
    assert(bool(re.match(pattern, t)) == True), '%s is not True' %t
for f in str_false:
    assert(bool(re.match(pattern, f)) == False), '%s is not False' %f<jupyter_output><empty_output><jupyter_text>Checking for HTML tags<jupyter_code>pattern = r"""</?\w+((\s+\w+(\s*=\s*(?:".*?"|'.*?'|[^'">\s]+))?)+\s*|\s*)/?>"""

str_true = ('<a>', '<a href="something">', '</a>', '<img src>')
            
str_false = ('a>', '<a ', '< a >')

for t in str_true:
    assert(bool(re.match(pattern, t)) == True), '%s is not True' %t

for f in str_false:
    assert(bool(re.match(pattern, f)) == False), '%s is not False' %f<jupyter_output><empty_output><jupyter_text>Checking for IP addresses  Ipv4 Image source: http://en.wikipedia.org/wiki/File:Ipv4_address.svg<jupyter_code>pattern = r'^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$'

str_true = ('172.16.254.1', '1.2.3.4', '01.102.103.104', )
            
str_false = ('17216.254.1', '1.2.3.4.5', '01 .102.103.104', )

for t in str_true:
    assert(bool(re.match(pattern, t)) == True), '%s is not True' %t

for f in str_false:
    assert(bool(re.match(pattern, f)) == False), '%s is not False' %f<jupyter_output><empty_output><jupyter_text>Ipv6 ![](../Images/Ipv6_address.png)<jupyter_code>pattern = r'^\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?\s*$'

str_true = ('fc00:db20:35b:7399::5',
            'fc00:db20:35b:7399::5', 
            'fc00:e968:6179::de52:7100', 
            'fc00:db20:35b:7399::5:9652', )
            
str_false = ('fc00:e968:6179::de52:7100:fc00:e968:6179::de52:7100:7777:1313', # uses `::` twice
             '1200:0000:AB00:1234:O000:2552:7777:1313', ) # contains an O instead of 0

for t in str_true:
    assert(bool(re.match(pattern, t)) == True), '%s is not True' %t

for f in str_false:
    assert(bool(re.match(pattern, f)) == False), '%s is not False' %f<jupyter_output><empty_output><jupyter_text>Checking for MAC addresses ![](../Images/MACaddressV3.png)<jupyter_code>pattern = r'^(?i)([0-9A-F]{2}[:-]){5}([0-9A-F]{2})$'

str_true = ('94-AE-70-A0-66-83', 
            '58-f8-1a-00-44-c8',
            '00:A0:C9:14:C8:29'
            , )
            
str_false = ('0:00:00:00:00:00', 
             '94-AE-70-A0 -66-83', ) 

for t in str_true:
    assert(bool(re.match(pattern, t)) == True), '%s is not True' %t

for f in str_false:
    assert(bool(re.match(pattern, f)) == False), '%s is not False' %f
def check_ssl(url):
    """Check if the ssl certificate is valid."""
    try:
        requests.get(url, verify=True, timeout=3)
        return True
    except Exception:
        return False
check_ssl('http://atecobois.fr/es/app.html')
def get_ptr(url):
    """Return PTR associated with IP."""
    try:
        if valid_ip(url['host']):
            ip = url['host']
        else:
            ip = resolver.query(url['host'], 'A')
            ip = ip[0].to_text()

        if ip:
            r = reversename.from_address(ip)
            result = resolver.query(r, 'PTR')[0].to_text()
            return result
        else:
            return 'True'
    except Exception:
        return 'False'
get_ptr('http://atecobois.fr/es/app.html')
def get_country(url):
    """Return the country associated with IP."""
    try:
        if valid_ip(url['host']):
            ip = url['host']
        else:
            ip = resolver.query(url['host'], 'A')
            ip = ip[0].to_text()

        if ip:
            reader = geoip2.database.Reader(PATH + 'GeoLite2-Country.mmdb')
            response = reader.country(ip)
            return response.country.iso_code
        else:
            return 'True'
    except Exception:
        return 'False'
get_country('http://atecobois.fr/es/app.html')
def get_asn_number(url):
    """Return the ANS number associated with the IP."""
    try:
        with geoip2.database.Reader(PATH + 'GeoLite2-ASN.mmdb') as reader:
            if valid_ip(url['host']):
                ip = url['host']
            else:
                ip = resolver.query(url['host'], 'A')
                ip = ip[0].to_text()

            if ip:
                response = reader.asn(ip)
                return response.autonomous_system_number
            else:
                return 'True'
    except Exception:
        return 'False'
get_asn_number('http://atecobois.fr/es/app.html')
def check_blacklists_ip(url):
    """Check if the IP is malicious through Google Safebrowsing, Phishtank and WOT."""
    try:
        if valid_ip(url['host']):
            ip = url['host']
        else:
            ip = resolver.query(url['host'], 'A')
            ip = ip[0].to_text()

        if ip:
            if (google_safebrowsing(ip) or phishtank(ip) or wot(ip)):
                return True
            return False
        else:
            return 'True'
    except Exception:
        return 'False'
check_blacklists_ip('http://atecobois.fr/es/app.html')<jupyter_output><empty_output><jupyter_text>Parsing Phone Number<jupyter_code>import re
pattern = re.compile(r'^(\d{3})-(\d{3})-(\d{4})$')
pattern.search('415-867-5309')
_sre.SRE_Match object at 0x02FCDD40>
pattern.search('415-867-5309').groups()
('415', '867', '5309')<jupyter_output><empty_output><jupyter_text>Extract Subdomain Form a URL<jupyter_code>from tldextract import extract
 
urls_file = "urlhaus_url.csv"
#URLs should be in column A without a heading, in a CSV file named "urls_file.csv"
 
urls = [line.rstrip('\n') for line in open(urls_file, errors='ignore')]
lst=[] 
for url in urls:
    tsd, td, tsu = extract(url)
    url = tsd +'.' + td + '.' + tsu
    lst.append(url)<jupyter_output><empty_output><jupyter_text>Extract Domain From a URL<jupyter_code>from tldextract import extract
 
urls_file = "urlhaus_url.csv"
#URLs should be in column A without a heading, in a CSV file named "urls_file.csv"
 
urls = [line.rstrip('\n') for line in open(urls_file, errors='ignore')]
lst=[] 
for url in urls:
    tsd, td, tsu = extract(url)
    url = td + '.' + tsu
    lst.append(url)<jupyter_output><empty_output><jupyter_text>Count Name Server From a URL<jupyter_code>def count_name_servers(url):
    """Return number of NameServers (NS) resolved."""
    count = 0
    if count_ips(url):
        try:
            answers = resolver.query(url['host'], 'NS')
            return len(answers)
        except (resolver.NoAnswer, resolver.NXDOMAIN):
            split_host = url['host'].split('.')
            while len(split_host) > 0:
                split_host.pop(0)
                supposed_domain = '.'.join(split_host)
                try:
                    answers = resolver.query(supposed_domain, 'NS')
                    count = len(answers)
                    break
                except Exception:
                    count = 0
        except Exception:
            count = 0
    return count
count_name_servers('http://atecobois.fr/es/app.html')<jupyter_output><empty_output><jupyter_text>Time Activation of Domain<jupyter_code>def time_activation_domain(url):
    """Return time (in days) of domain activation."""
    if url['host'].startswith("www."):
        url['host'] = url['host'][4:]

    pythonwhois.net.socket.setdefaulttimeout(3.0)
    try:
        result_whois = pythonwhois.get_whois(url['host'].lower())
        if not result_whois:
            return '?'
        creation_date = str(result_whois['creation_date'][0])
        formated_date = " ".join(creation_date.split()[:1])
        d1 = datetime.strptime(formated_date, "%Y-%m-%d")
        d2 = datetime.now()
        return abs((d2 - d1).days)
    except Exception:
        return '?'<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.21564885496183206, 'fraction_numerical': 0.05567254540668597, 'mean_word_length': 3.196906931786799, 'pattern_counts': {'":': 0, '<': 100, '<?xml version=': 0, '>': 103, 'https://': 3, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '22', 'dir': 'jupyter-structured-clean-dedup', 'id': '365939', 'license': 'MIT', 'n_tokens_mistral': 6496, 'n_tokens_neox': 6104, 'n_words': 1156, 'path': 'Python For DS/Security Analysis/Data Science For Security Analysis.ipynb', 'repo_name': 'jrderek/Data-science-master-resources', 'size': '15220'}