<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import os
import io
import json
import copy
import tempfile
import shutil
import zipfile
import six
import requests
from .schema import Schema
from .resource import Resource
from .exceptions import (
    DataPackageException,
)


class DataPackage(object):
    '''Class for loading, validating and working with a Data Package.

    Args:
        metadata (dict, str or file-like object, optional): The contents of the
            `datapackage.json` file. It can be a ``dict`` with its contents,
            a ``string`` with the local path for the file or its URL, or a
            file-like object. It also can point to a `ZIP` file with one and
            only one `datapackage.json` (it can be in a subfolder). If
            you're passing a ``dict``, it's a good practice to also set the
            ``default_base_path`` parameter to the absolute `datapackage.json`
            path.
        schema (dict or str, optional): The schema to be used to validate this
            data package. If can be a ``dict`` with the schema's contents or a
            ``str``. The string can contain the schema's ID if it's in the
            registry, a local path, or an URL.
        default_base_path (str, optional): The default path to be used to load
            resources located on the local disk that don't define a base path
            themselves. This will usually be the path for the
            `datapackage.json` file. If the :data:`metadata` parameter was the
            path to the `datapackage.json`, this will automatically be set to
            its base path.

    Raises:
        DataPackageException: If the :data:`metadata` couldn't be loaded or was
            invalid.
        SchemaError: If the :data:`schema` couldn't be loaded or was invalid.
        ResourceError: If any resource defined in :data:`metadata` couldn't be
            loaded.
    '''

    def __init__(self, metadata=None, schema='base', default_base_path=None):
        metadata = self._extract_zip_if_possible(metadata)

        self._metadata = self._load_metadata(metadata)
        self._schema = Schema(schema)
        self._base_path = self._get_base_path(metadata, default_base_path)
        self._resources = self._load_resources(self.metadata,
                                               self.base_path)

    def __del__(self):
        self._remove_tempdir_if_exists()

    @property
    def metadata(self):
        '''dict: The metadata of this data package. Its attributes can be
        changed.'''
        return self._metadata

    @property
    def schema(self):
        ''':class:`.Schema`: The schema of this data package.'''
        return self._schema

    @property
    def base_path(self):
        '''str: The base path of this Data Package (can be None).'''
        return self.metadata.get('base', self._base_path)

    @property
    def resources(self):
        '''The resources defined in this data package (can be empty).

        To add or remove resources, alter the `resources` attribute of the
        :data:`metadata`.

        :returns: The resources.
        :rtype: tuple of :class:`.Resource`

        Raises:
            ResourceError: If any resource couldn't be loaded. This will only
                happen if you've changed the resources in the :data:`metadata`
                after this :class:`DataPackage` was created.
        '''
        self._resources = self._update_resources(self._resources,
                                                 self.metadata,
                                                 self.base_path)
        return self._resources

    @property
    def attributes(self):
        '''tuple: The union of the attributes defined in the schema and the
        data package (can be empty).'''
        attributes = set(self.to_dict().keys())
        try:
            attributes.update(self.schema.properties.keys())
        except AttributeError:
            pass
        return tuple(attributes)

    @property
    def required_attributes(self):
        '''tuple: The schema's required attributed (can be empty).'''
        required = ()
        try:
            if self.schema.required is not None:
                required = tuple(self.schema.required)
        except AttributeError:
            pass
        return required

    def to_dict(self):
        '''dict: Convert this Data Package to dict.'''
        return copy.deepcopy(self.metadata)

    def to_json(self):
        '''str: Convert this Data Package to a JSON string.'''
        return json.dumps(self.metadata)

    def safe(self):
        '''bool: Return if it's safe to load this datapackage's resources.

        A Data Package is safe if it has no resources, or if all of its
        resources are either:
            * Inline;
            * Remote;
            * Local relative to the Data Package's base path.
        '''
        local_resources = [resource for resource in self.resources
                           if resource.local_data_path]
        if not self.base_path:
            return len(local_resources) == 0
        else:
            for resource in local_resources:
                if not resource.local_data_path.startswith(self.base_path):
                    return False

        return True

    def save(self, file_or_path):
        '''Validates and saves this Data Package contents into a zip file.

        It creates a zip file into ``file_or_path`` with the contents of this
        Data Package and its resources. Every resource which content lives in
        the local filesystem will be copied to the zip file. Consider the
        following Data Package descriptor::
<commit_msg>[#24] Add docs warning about security issues with remote URLs<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import os
import io
import json
import copy
import tempfile
import shutil
import zipfile
import six
import requests
from .schema import Schema
from .resource import Resource
from .exceptions import (
    DataPackageException,
)


class DataPackage(object):
    '''Class for loading, validating and working with a Data Package.

    Args:
        metadata (dict, str or file-like object, optional): The contents of the
            `datapackage.json` file. It can be a ``dict`` with its contents,
            a ``string`` with the local path for the file or its URL, or a
            file-like object. It also can point to a `ZIP` file with one and
            only one `datapackage.json` (it can be in a subfolder). If
            you're passing a ``dict``, it's a good practice to also set the
            ``default_base_path`` parameter to the absolute `datapackage.json`
            path.
        schema (dict or str, optional): The schema to be used to validate this
            data package. If can be a ``dict`` with the schema's contents or a
            ``str``. The string can contain the schema's ID if it's in the
            registry, a local path, or an URL.
        default_base_path (str, optional): The default path to be used to load
            resources located on the local disk that don't define a base path
            themselves. This will usually be the path for the
            `datapackage.json` file. If the :data:`metadata` parameter was the
            path to the `datapackage.json`, this will automatically be set to
            its base path.

    Raises:
        DataPackageException: If the :data:`metadata` couldn't be loaded or was
            invalid.
        SchemaError: If the :data:`schema` couldn't be loaded or was invalid.
        ResourceError: If any resource defined in :data:`metadata` couldn't be
            loaded.
    '''

    def __init__(self, metadata=None, schema='base', default_base_path=None):
        metadata = self._extract_zip_if_possible(metadata)

        self._metadata = self._load_metadata(metadata)
        self._schema = Schema(schema)
        self._base_path = self._get_base_path(metadata, default_base_path)
        self._resources = self._load_resources(self.metadata,
                                               self.base_path)

    def __del__(self):
        self._remove_tempdir_if_exists()

    @property
    def metadata(self):
        '''dict: The metadata of this data package. Its attributes can be
        changed.'''
        return self._metadata

    @property
    def schema(self):
        ''':class:`.Schema`: The schema of this data package.'''
        return self._schema

    @property
    def base_path(self):
        '''str: The base path of this Data Package (can be None).'''
        return self.metadata.get('base', self._base_path)

    @property
    def resources(self):
        '''The resources defined in this data package (can be empty).

        To add or remove resources, alter the `resources` attribute of the
        :data:`metadata`.

        :returns: The resources.
        :rtype: tuple of :class:`.Resource`

        Raises:
            ResourceError: If any resource couldn't be loaded. This will only
                happen if you've changed the resources in the :data:`metadata`
                after this :class:`DataPackage` was created.
        '''
        self._resources = self._update_resources(self._resources,
                                                 self.metadata,
                                                 self.base_path)
        return self._resources

    @property
    def attributes(self):
        '''tuple: The union of the attributes defined in the schema and the
        data package (can be empty).'''
        attributes = set(self.to_dict().keys())
        try:
            attributes.update(self.schema.properties.keys())
        except AttributeError:
            pass
        return tuple(attributes)

    @property
    def required_attributes(self):
        '''tuple: The schema's required attributed (can be empty).'''
        required = ()
        try:
            if self.schema.required is not None:
                required = tuple(self.schema.required)
        except AttributeError:
            pass
        return required

    def to_dict(self):
        '''dict: Convert this Data Package to dict.'''
        return copy.deepcopy(self.metadata)

    def to_json(self):
        '''str: Convert this Data Package to a JSON string.'''
        return json.dumps(self.metadata)

    def safe(self):
        '''bool: Return if it's safe to load this datapackage's resources.

        A Data Package is safe if it has no resources, or if all of its
        resources are either:
            * Inline;
            * Remote;
            * Local relative to the Data Package's base path.

        Even though we don't check the remote resources' URLs, keep in mind
        that they can be an attack vector as well. For example, a malicious
        user may set a resource URL to an address only accessible by the
        machine that's parsing the datapackage. That might be a problem or not,
        depending on your specific usage.
        '''
        local_resources = [resource for resource in self.resources
                           if resource.local_data_path]
        if not self.base_path:
            return len(local_resources) == 0
        else:
            for resource in local_resources:
                if not resource.local_data_path.startswith(self.base_path):
                    return False

        return True

    def save(self, file_or_path):
        '''Validates and saves this Data Package contents into a zip file.

        It creates a zip file into ``file_or_path`` with the contents of this
        Data Package and its resources. Every resource which content lives in
        the local filesystem will be copied to the zip file. Consider the
        following Data Package descriptor::

{'fraction_non_alphanumeric': 0.07283333333333333, 'fraction_numerical': 0.0003333333333333333, 'mean_word_length': 2.165655499868109, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4481017', 'n_tokens_mistral': 3001, 'n_tokens_neox': 2890, 'n_words': 1246}