<commit_before># Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import io
import os

import google.auth
import google.auth.transport.requests as tr_requests
import pytest
from six.moves import http_client

import gooresmed
from tests.system import utils


CURR_DIR = os.path.dirname(os.path.realpath(__file__))
DATA_DIR = os.path.join(CURR_DIR, u'..', u'data')
IMG_FILES = (
    os.path.realpath(os.path.join(DATA_DIR, u'image1.jpg')),
    os.path.realpath(os.path.join(DATA_DIR, u'image2.jpg')),
)


@pytest.fixture(scope=u'module')
def authorized_transport():
    credentials, _ = google.auth.default(scopes=(utils.GCS_RW_SCOPE,))
    yield tr_requests.AuthorizedSession(credentials)


@pytest.fixture(scope=u'module')
def add_files(authorized_transport):
    blob_names = []
    for img_file in IMG_FILES:
        with open(img_file, u'rb') as file_obj:
            actual_contents = file_obj.read()

        blob_name = os.path.basename(img_file)
        blob_names.append(blob_name)
        upload_url = utils.SIMPLE_UPLOAD_TEMPLATE.format(blob_name=blob_name)
        upload = gooresmed.SimpleUpload(upload_url)
        response = upload.transmit(
            authorized_transport, actual_contents, u'image/jpeg')
        assert response.status_code == http_client.OK

    yield

    # Clean-up the blobs we created.
    for blob_name in blob_names:
        metadata_url = utils.METADATA_URL_TEMPLATE.format(blob_name=blob_name)
        response = authorized_transport.delete(metadata_url)
        assert response.status_code == http_client.NO_CONTENT


def test_download_full(add_files, authorized_transport):
    for img_file in IMG_FILES:
        with open(img_file, u'rb') as file_obj:
            actual_contents = file_obj.read()

        blob_name = os.path.basename(img_file)

        # Create the actual download object.
        media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
        download = gooresmed.Download(media_url)
        # Consume the resource.
        response = download.consume(authorized_transport)
        assert response.status_code == http_client.OK
        assert response.content == actual_contents
        # Make sure the download is tombstoned.
        with pytest.raises(ValueError):
            download.consume(authorized_transport)


def test_download_partial(add_files, authorized_transport):
    slices = (
        slice(1024, 16386, None),  # obj[1024:16386]
        slice(None, 8192, None),  # obj[:8192]
        slice(-256, None, None),  # obj[-256:]
        slice(262144, None, None),  # obj[262144:]
    )
    for img_file in IMG_FILES:
        with open(img_file, u'rb') as file_obj:
            actual_contents = file_obj.read()<commit_msg>Adding two system tests for download failures.<commit_after># Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import io
import os

import google.auth
import google.auth.transport.requests as tr_requests
import pytest
from six.moves import http_client

import gooresmed
from tests.system import utils


CURR_DIR = os.path.dirname(os.path.realpath(__file__))
DATA_DIR = os.path.join(CURR_DIR, u'..', u'data')
IMG_FILES = (
    os.path.realpath(os.path.join(DATA_DIR, u'image1.jpg')),
    os.path.realpath(os.path.join(DATA_DIR, u'image2.jpg')),
)


@pytest.fixture(scope=u'module')
def authorized_transport():
    credentials, _ = google.auth.default(scopes=(utils.GCS_RW_SCOPE,))
    yield tr_requests.AuthorizedSession(credentials)


@pytest.fixture(scope=u'module')
def add_files(authorized_transport):
    blob_names = []
    for img_file in IMG_FILES:
        with open(img_file, u'rb') as file_obj:
            actual_contents = file_obj.read()

        blob_name = os.path.basename(img_file)
        blob_names.append(blob_name)
        upload_url = utils.SIMPLE_UPLOAD_TEMPLATE.format(blob_name=blob_name)
        upload = gooresmed.SimpleUpload(upload_url)
        response = upload.transmit(
            authorized_transport, actual_contents, u'image/jpeg')
        assert response.status_code == http_client.OK

    yield

    # Clean-up the blobs we created.
    for blob_name in blob_names:
        metadata_url = utils.METADATA_URL_TEMPLATE.format(blob_name=blob_name)
        response = authorized_transport.delete(metadata_url)
        assert response.status_code == http_client.NO_CONTENT


def test_download_full(add_files, authorized_transport):
    for img_file in IMG_FILES:
        with open(img_file, u'rb') as file_obj:
            actual_contents = file_obj.read()

        blob_name = os.path.basename(img_file)

        # Create the actual download object.
        media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
        download = gooresmed.Download(media_url)
        # Consume the resource.
        response = download.consume(authorized_transport)
        assert response.status_code == http_client.OK
        assert response.content == actual_contents
        # Make sure the download is tombstoned.
        with pytest.raises(ValueError):
            download.consume(authorized_transport)


def test_non_existent_file(authorized_transport):
    blob_name = u'does-not-exist.txt'
    media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
    download = gooresmed.Download(media_url)

    # Try to consume the resource and fail.
    with pytest.raises(gooresmed.InvalidResponse) as exc_info:
        download.consume(authorized_transport)

    error = exc_info.value
    response = error.response
    assert response.status_code == http_client.NOT_FOUND
    assert response.content == b'Not Found'
    assert len(error.args) == 5
    assert error.args[1] == http_client.NOT_FOUND
    assert error.args[3] == http_client.OK
    assert error.args[4] == http_client.PARTIAL_CONTENT
    # Make sure the download is tombstoned (even though it failed).
    with pytest.raises(ValueError):
        download.consume(authorized_transport)


@pytest.fixture(scope=u'module')
def simple_file(authorized_transport):
    blob_name = u'basic-file.txt'
    upload_url = utils.SIMPLE_UPLOAD_TEMPLATE.format(blob_name=blob_name)
    upload = gooresmed.SimpleUpload(upload_url)
    data = b'Simple contents'
    response = upload.transmit(authorized_transport, data, u'text/plain')
    assert response.status_code == http_client.OK

    yield blob_name, data

    # Clean-up the blob we created.
    metadata_url = utils.METADATA_URL_TEMPLATE.format(blob_name=blob_name)
    response = authorized_transport.delete(metadata_url)
    assert response.status_code == http_client.NO_CONTENT


def test_bad_range(simple_file, authorized_transport):
    blob_name, data = simple_file
    # Make sure we have an invalid range.
    start = 32
    end = 63
    assert len(data) < start < end
    # Create the actual download object.
    media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
    download = gooresmed.Download(media_url, start=start, end=end)

    # Try to consume the resource and fail.
    with pytest.raises(gooresmed.InvalidResponse) as exc_info:
        download.consume(authorized_transport)

    error = exc_info.value
    response = error.response
    assert response.status_code == http_client.REQUESTED_RANGE_NOT_SATISFIABLE
    assert response.content == b'Request range not satisfiable'
    assert len(error.args) == 5
    assert error.args[1] == http_client.REQUESTED_RANGE_NOT_SATISFIABLE
    assert error.args[3] == http_client.OK
    assert error.args[4] == http_client.PARTIAL_CONTENT
    # Make sure the download is tombstoned (even though it failed).
    with pytest.raises(ValueError):
        download.consume(authorized_transport)


def test_download_partial(add_files, authorized_transport):
    slices = (
        slice(1024, 16386, None),  # obj[1024:16386]
        slice(None, 8192, None),  # obj[:8192]
        slice(-256, None, None),  # obj[-256:]
        slice(262144, None, None),  # obj[262144:]
    )
    for img_file in IMG_FILES:
        with open(img_file, u'rb') as file_obj:
            actual_contents = file_obj.read()
{'fraction_non_alphanumeric': 0.0916593309859155, 'fraction_numerical': 0.013204225352112676, 'mean_word_length': 3.651484135107472, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1438955', 'n_tokens_mistral': 2903, 'n_tokens_neox': 2813, 'n_words': 669}