<commit_before>#
# Copyright 2016 University of Oxford
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""
Main interface for the htsget library.
"""
from __future__ import division
from __future__ import print_function

import logging

import htsget.protocol as protocol
import htsget.exceptions as exceptions

import requests

CONTENT_LENGTH = "Content-Length"


def get(
        url, output, fmt=None, reference_name=None, reference_md5=None,
        start=None, end=None, fields=None, tags=None, notags=None,
        max_retries=5, timeout=10, retry_wait=5):
    """
    Runs a request to the specified URL and write the resulting data to
    the specified file-like object.
    """
    manager = SynchronousDownloadManager(
        url, output, fmt=fmt, reference_name=reference_name, reference_md5=reference_md5,
        start=start, end=end, fields=fields, tags=tags, notags=notags,
        max_retries=max_retries, timeout=timeout, retry_wait=retry_wait)
    manager.run()


class SynchronousDownloadManager(protocol.DownloadManager):
    """
    Class implementing the GA4GH streaming API synchronously using the
    requests library.
    """
    def __get(self, *args, **kwargs):
        try:
            response = requests.get(*args, **kwargs)
            print("response = ", response.json())
        except requests.RequestException as re:
            raise exceptions.RetryableError(str(re))
        try:
            response.raise_for_status()
        except requests.HTTPError as he:
            # TODO classify other errors that we consider unrecoverable.
            if response.status_code == 404:
                raise he
            else:
                raise exceptions.RetryableError(str(he))
        return response

    def _handle_ticket_request(self):
        logging.debug("handle_ticket_request(url={})".format(self.ticket_request_url))
        response = self.__get(self.ticket_request_url, timeout=self.timeout)
        self.ticket = response.json()

    def _handle_http_url(self, url, headers):
        logging.debug("handle_http_url(url={}, headers={})".format(url, headers))
        response = self.__get(url, headers=headers, stream=True, timeout=self.timeout)
        length = 0
        piece_size = 65536
        try:
            for piece in response.iter_content(piece_size):
                length += len(piece)<commit_msg>Fixed bug.<commit_after>#
# Copyright 2016 University of Oxford
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""
Main interface for the htsget library.
"""
from __future__ import division
from __future__ import print_function

import logging

import htsget.protocol as protocol
import htsget.exceptions as exceptions

import requests

CONTENT_LENGTH = "Content-Length"


def get(
        url, output, fmt=None, reference_name=None, reference_md5=None,
        start=None, end=None, fields=None, tags=None, notags=None,
        max_retries=5, timeout=10, retry_wait=5):
    """
    Runs a request to the specified URL and write the resulting data to
    the specified file-like object.
    """
    manager = SynchronousDownloadManager(
        url, output, fmt=fmt, reference_name=reference_name, reference_md5=reference_md5,
        start=start, end=end, fields=fields, tags=tags, notags=notags,
        max_retries=max_retries, timeout=timeout, retry_wait=retry_wait)
    manager.run()


class SynchronousDownloadManager(protocol.DownloadManager):
    """
    Class implementing the GA4GH streaming API synchronously using the
    requests library.
    """
    def __get(self, *args, **kwargs):
        try:
            response = requests.get(*args, **kwargs)
        except requests.RequestException as re:
            raise exceptions.RetryableError(str(re))
        try:
            response.raise_for_status()
        except requests.HTTPError as he:
            # TODO classify other errors that we consider unrecoverable.
            if response.status_code == 404:
                raise he
            else:
                raise exceptions.RetryableError(str(he))
        return response

    def _handle_ticket_request(self):
        logging.debug("handle_ticket_request(url={})".format(self.ticket_request_url))
        response = self.__get(self.ticket_request_url, timeout=self.timeout)
        self.ticket = response.json()

    def _handle_http_url(self, url, headers):
        logging.debug("handle_http_url(url={}, headers={})".format(url, headers))
        response = self.__get(url, headers=headers, stream=True, timeout=self.timeout)
        length = 0
        piece_size = 65536
        try:
            for piece in response.iter_content(piece_size):
                length += len(piece)
{'fraction_non_alphanumeric': 0.08839584433879204, 'fraction_numerical': 0.008804366966015144, 'mean_word_length': 3.2136498516320473, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1008611', 'n_tokens_mistral': 1577, 'n_tokens_neox': 1531, 'n_words': 486}