<commit_before>import demistomock as demisto
from CommonServerPython import *
from CommonServerUserPython import *


# IMPORTS

import cabby
import requests
import urllib3
import dateutil
import pytz
import dateutil.parser
from bs4 import BeautifulSoup
from netaddr import IPAddress, iprange_to_cidrs, IPNetwork
from six import string_types
from typing import Dict, List

# Disable insecure warnings
requests.packages.urllib3.disable_warnings()

SOURCE_NAME = "Alien Vault OTX TAXII"


# disable insecure warnings
urllib3.disable_warnings()

EPOCH = datetime.utcfromtimestamp(0).replace(tzinfo=pytz.UTC)

""" Helper Methods """


def package_extract_properties(package):
    """Extracts properties from the STIX package"""
    result: Dict[str, str] = {}

    header = package.find_all('STIX_Header')
    if len(header) == 0:
        return result

    # share level
    mstructures = header[0].find_all('Marking_Structure')
    for ms in mstructures:
        type_ = ms.get('xsi:type')
        if type_ is result:
            continue

        color = ms.get('color')
        if color is result:
            continue

        type_ = type_.lower()
        if 'tlpmarkingstructuretype' not in type_:
            continue

        result['share_level'] = color.lower()  # https://www.us-cert.gov/tlp
        break

    # decode title
    title = next((c for c in header[0] if c.name == 'Title'), None)
    if title is not None:
        result['stix_package_title'] = title.text

    # decode description
    description = next((c for c in header[0] if c.name == 'Description'), None)
    if description is not None:
        result['stix_package_description'] = description.text

    # decode description
    sdescription = next((c for c in header[0] if c.name == 'Short_Description'), None)
    if sdescription is not None:
        result['stix_package_short_description'] = sdescription.text

    # decode identity name from information_source
    information_source = next((c for c in header[0] if c.name == 'Information_Source'), None)
    if information_source is not None:
        identity = next((c for c in information_source if c.name == 'Identity'), None)
        if identity is not None:
            name = next((c for c in identity if c.name == 'Name'))
            if name is not None:
                result['stix_package_information_source'] = name.text

    return result


def observable_extract_properties(observable):
    """Extracts properties from observable"""
    result = {}

    title = next((c for c in observable if c.name == 'Title'), None)
    if title is not None:
        title = title.text
        result['stix_title'] = title

    description = next((c for c in observable if c.name == 'Description'), None)
    if description is not None:
        description = description.text
        result['stix_description'] = description

    return result


""" TAXII STIX DECODE """


class AddressObject(object):
    """
    Implements address object indicator decoding
    based on: https://stixproject.github.io/data-model/1.2/AddressObj/AddressObjectType/
    """
    @staticmethod
    def decode(props, **kwargs):
        indicator = props.find('Address_Value')
        if indicator is None:
            return []
        indicator = indicator.string.encode('ascii', 'replace').decode()

        acategory = props.get('category', None)
        if acategory is None:
            try:
                # if ',' in the ip address then we have a range (10.0.0.0,10.0.0.255)
                # and we want to change it to a CIDR object (10.0.0.0/24)
                if ',' in indicator:
                    ips = indicator.split(',')
                    indicator = str(iprange_to_cidrs(ips[0], ips[1])[0])
                    cidr = IPNetwork(indicator)
                    if cidr.version == 4:
                        type_ = 'CIDR'
                    elif cidr.version == 6:
                        type_ = 'IPv6CIDR'
                    else:
                        LOG('Unknown ip version: {!r}'.format(cidr.version))
                        return []

                else:
                    ip = IPAddress(indicator)
                    if ip.version == 4:
                        type_ = 'IP'
                    elif ip.version == 6:
                        type_ = 'IPv6'
                    else:
                        LOG('Unknown ip version: {!r}'.format(ip.version))
                        return []

            except Exception:
                return []

        elif acategory == 'ipv4-addr':
            # if ',' in the ip address then we have a range (10.0.0.0,10.0.0.255)
            # and we want to change it to a CIDR object (10.0.0.0/24)
            if ',' in indicator:
                ips = indicator.split(',')
                indicator = str(iprange_to_cidrs(ips[0], ips[1])[0].cidr)
                type_ = 'CIDR'
<commit_msg>adding cidr support for indicators<commit_after>import demistomock as demisto
from CommonServerPython import *
from CommonServerUserPython import *


# IMPORTS

import cabby
import requests
import urllib3
import dateutil
import pytz
import dateutil.parser
from bs4 import BeautifulSoup
from netaddr import IPAddress, iprange_to_cidrs, IPNetwork
from six import string_types
from typing import Dict, List

# Disable insecure warnings
requests.packages.urllib3.disable_warnings()

SOURCE_NAME = "Alien Vault OTX TAXII"


# disable insecure warnings
urllib3.disable_warnings()

EPOCH = datetime.utcfromtimestamp(0).replace(tzinfo=pytz.UTC)

""" Helper Methods """


def package_extract_properties(package):
    """Extracts properties from the STIX package"""
    result: Dict[str, str] = {}

    header = package.find_all('STIX_Header')
    if len(header) == 0:
        return result

    # share level
    mstructures = header[0].find_all('Marking_Structure')
    for ms in mstructures:
        type_ = ms.get('xsi:type')
        if type_ is result:
            continue

        color = ms.get('color')
        if color is result:
            continue

        type_ = type_.lower()
        if 'tlpmarkingstructuretype' not in type_:
            continue

        result['share_level'] = color.lower()  # https://www.us-cert.gov/tlp
        break

    # decode title
    title = next((c for c in header[0] if c.name == 'Title'), None)
    if title is not None:
        result['stix_package_title'] = title.text

    # decode description
    description = next((c for c in header[0] if c.name == 'Description'), None)
    if description is not None:
        result['stix_package_description'] = description.text

    # decode description
    sdescription = next((c for c in header[0] if c.name == 'Short_Description'), None)
    if sdescription is not None:
        result['stix_package_short_description'] = sdescription.text

    # decode identity name from information_source
    information_source = next((c for c in header[0] if c.name == 'Information_Source'), None)
    if information_source is not None:
        identity = next((c for c in information_source if c.name == 'Identity'), None)
        if identity is not None:
            name = next((c for c in identity if c.name == 'Name'))
            if name is not None:
                result['stix_package_information_source'] = name.text

    return result


def observable_extract_properties(observable):
    """Extracts properties from observable"""
    result = {}

    title = next((c for c in observable if c.name == 'Title'), None)
    if title is not None:
        title = title.text
        result['stix_title'] = title

    description = next((c for c in observable if c.name == 'Description'), None)
    if description is not None:
        description = description.text
        result['stix_description'] = description

    return result


""" TAXII STIX DECODE """


class AddressObject(object):
    """
    Implements address object indicator decoding
    based on: https://stixproject.github.io/data-model/1.2/AddressObj/AddressObjectType/
    """
    @staticmethod
    def decode(props, **kwargs):
        indicator = props.find('Address_Value')
        if indicator is None:
            return []
        indicator = indicator.string.encode('ascii', 'replace').decode()

        acategory = props.get('category', None)
        if acategory is None:
            try:
                # if ',' in the ip address then we have a range (10.0.0.0,10.0.0.255)
                # and we want to change it to a CIDR object (10.0.0.0/24)
                if ',' in indicator:
                    ips = indicator.split(',')
                    indicator = str(iprange_to_cidrs(ips[0], ips[1])[0].cidr)
                    cidr = IPNetwork(indicator)
                    if cidr.version == 4:
                        type_ = 'CIDR'
                    elif cidr.version == 6:
                        type_ = 'IPv6CIDR'
                    else:
                        LOG('Unknown ip version: {!r}'.format(cidr.version))
                        return []

                else:
                    ip = IPAddress(indicator)
                    if ip.version == 4:
                        type_ = 'IP'
                    elif ip.version == 6:
                        type_ = 'IPv6'
                    else:
                        LOG('Unknown ip version: {!r}'.format(ip.version))
                        return []

            except Exception:
                return []

        elif acategory == 'ipv4-addr':
            # if ',' in the ip address then we have a range (10.0.0.0,10.0.0.255)
            # and we want to change it to a CIDR object (10.0.0.0/24)
            if ',' in indicator:
                ips = indicator.split(',')
                indicator = str(iprange_to_cidrs(ips[0], ips[1])[0].cidr)
                type_ = 'CIDR'

{'fraction_non_alphanumeric': 0.10218456512862392, 'fraction_numerical': 0.013066557778685178, 'mean_word_length': 2.05202492211838, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 12, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7062977', 'n_tokens_mistral': 2879, 'n_tokens_neox': 2736, 'n_words': 820}