<commit_before>__author__ = 'purg'

import hashlib
import mimetypes
import requests

from smqtk.data_rep import DataElement


MIMETYPES = mimetypes.MimeTypes()


class DataUrlElement (DataElement):
    """
    Representation of data loadable via a web URL address.
    """

    def __init__(self, url_address):
        """
        :param url_address: Web address of element
        :type url_address: str
        """
        super(DataUrlElement, self).__init__()

        self._url = url_address
        self._md5_cache = None

        # make sure that url has a http:// or https:// prefix
        if not (self._url[:7] == "http://" or self._url[:8] == "https://"):
            self._url = "http://" + self._url

    def content_type(self):
        """
        :return: Standard type/subtype string for this data element, or None if
            the content type is unknown.
        :rtype: str or None
        """
        # return MIMETYPES.guess_type(self._url)[0]
        return requests.get(self._url).headers['content-type']

    def md5(self):
        """
        :return: MD5 hex string of the data content.
        :rtype: str
        """
        if not self._md5_cache:
            self._md5_cache = hashlib.md5(self.get_bytes()).hexdigest()
        return self._md5_cache

    def uuid(self):
        """
        UUID for this data element.

        :return: UUID value for this data element. This return value should be
            hashable.<commit_msg>Added DataUrlElement link checking during construction<commit_after>__author__ = 'purg'

import hashlib
import mimetypes
import requests

from smqtk.data_rep import DataElement


MIMETYPES = mimetypes.MimeTypes()


class DataUrlElement (DataElement):
    """
    Representation of data loadable via a web URL address.
    """

    def __init__(self, url_address):
        """
        :param url_address: Web address of element
        :type url_address: str
        """
        super(DataUrlElement, self).__init__()

        self._url = url_address
        self._md5_cache = None

        # make sure that url has a http:// or https:// prefix
        if not (self._url[:7] == "http://" or self._url[:8] == "https://"):
            self._url = "http://" + self._url

        # Check that the URL is valid, i.e. actually points to something
        requests.get(self._url).raise_for_status()

    def content_type(self):
        """
        :return: Standard type/subtype string for this data element, or None if
            the content type is unknown.
        :rtype: str or None
        """
        # return MIMETYPES.guess_type(self._url)[0]
        return requests.get(self._url).headers['content-type']

    def md5(self):
        """
        :return: MD5 hex string of the data content.
        :rtype: str
        """
        if not self._md5_cache:
            self._md5_cache = hashlib.md5(self.get_bytes()).hexdigest()
        return self._md5_cache

    def uuid(self):
        """
        UUID for this data element.

        :return: UUID value for this data element. This return value should be
            hashable.
{'fraction_non_alphanumeric': 0.10568842921784098, 'fraction_numerical': 0.006464124111182934, 'mean_word_length': 2.174358974358974, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7055304', 'n_tokens_mistral': 912, 'n_tokens_neox': 898, 'n_words': 274}