<commit_before>"""
A simple package manager for Universe packages.
"""
import logging
import subprocess
import json
import os
import tempfile
try:
    import requests
    _HAS_REQUESTS = True
except ImportError:
    _HAS_REQUESTS = False


LOGGER = logging.getLogger(__name__)


class PackageManager:
    """A simple package manager for retrieving universe packages"""
    def __init__(self, universe_url="https://universe.mesosphere.com/repo",
                 dcos_version="1.10",
                 package_version="4"):

        self.universe_url = "https://universe.mesosphere.com/repo"
        self._headers = {
            "User-Agent": "dcos/{}".format(dcos_version),
            "Accept": "application/vnd.dcos.universe.repo+json;"
                      "charset=utf-8;version=v{}".format(package_version),
        }

        self.__package_cache = None

    def get_packages(self):
        """Query the uninverse to get a list of packages"""
        if self.__package_cache is None:
            if not _HAS_REQUESTS:
                LOGGER.info("Requests package not found. Using curl")
                self.__package_cache = self._get_packages_with_curl()
            else:
                self.__package_cache = self._get_packages_with_requests()

        return self.__package_cache

    def _get_packages_with_curl(self):
        """Use curl to download the packages from the universe"""
        with tempfile.TemporaryDirectory() as tmp_dir:
            tmp_filename = os.path.join(tmp_dir, 'packages.json')

            cmd = ["curl",
                   "--write-out",  "%{http_code}",
                   "--silent",
                   "-L",
                   "--max-time", "5",
                   "-X", "GET",
                   "-o", tmp_filename, ]
            for k, header in self._headers.items():
                cmd.extend(["-H", "{}: {}".format(k, header)])

            cmd.append(self.universe_url)
<commit_msg>Also retry if package cache is empty<commit_after>"""
A simple package manager for Universe packages.
"""
import logging
import subprocess
import json
import os
import tempfile
try:
    import requests
    _HAS_REQUESTS = True
except ImportError:
    _HAS_REQUESTS = False


LOGGER = logging.getLogger(__name__)


class PackageManager:
    """A simple package manager for retrieving universe packages"""
    def __init__(self, universe_url="https://universe.mesosphere.com/repo",
                 dcos_version="1.10",
                 package_version="4"):

        self.universe_url = "https://universe.mesosphere.com/repo"
        self._headers = {
            "User-Agent": "dcos/{}".format(dcos_version),
            "Accept": "application/vnd.dcos.universe.repo+json;"
                      "charset=utf-8;version=v{}".format(package_version),
        }

        self.__package_cache = None

    def get_packages(self):
        """Query the uninverse to get a list of packages"""
        if not self.__package_cache:
            if not _HAS_REQUESTS:
                LOGGER.info("Requests package not found. Using curl")
                self.__package_cache = self._get_packages_with_curl()
            else:
                self.__package_cache = self._get_packages_with_requests()

        return self.__package_cache

    def _get_packages_with_curl(self):
        """Use curl to download the packages from the universe"""
        with tempfile.TemporaryDirectory() as tmp_dir:
            tmp_filename = os.path.join(tmp_dir, 'packages.json')

            cmd = ["curl",
                   "--write-out",  "%{http_code}",
                   "--silent",
                   "-L",
                   "--max-time", "5",
                   "-X", "GET",
                   "-o", tmp_filename, ]
            for k, header in self._headers.items():
                cmd.extend(["-H", "{}: {}".format(k, header)])

            cmd.append(self.universe_url)

{'fraction_non_alphanumeric': 0.11764705882352941, 'fraction_numerical': 0.0030824556896994607, 'mean_word_length': 2.0469483568075115, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4312601', 'n_tokens_mistral': 1107, 'n_tokens_neox': 1053, 'n_words': 253}