<commit_before>import hashlib
import json
import logging
import os
import re
import subprocess
import sys
from builtins import bytes

logger = logging.getLogger('c7n_azure.dependency_manager')


class DependencyManager(object):

    @staticmethod
    def _run(cmd, verbose=False, **kwargs):
        if verbose:
            stdout = stderr = None
        else:
            stdout = stderr = subprocess.PIPE

        logger.debug(' '.join(cmd))
        return subprocess.run(cmd, stdout=stdout, stderr=stderr, **kwargs)

    @staticmethod
    def _get_installed_distributions():
        try:
            from pip._internal.utils.misc import get_installed_distributions
        except ImportError:
            from pip import get_installed_distributions

        return get_installed_distributions()

    @staticmethod
    def get_dependency_packages_list(packages, excluded_packages):
        dists = DependencyManager._get_installed_distributions()
        res = []
        for p in packages:
            res.extend([str(r) for r in next(d.requires() for d in dists if (p + ' ') in str(d))])

        res = [t for t in res if not any((e in t) for e in excluded_packages + packages)]

        # boto3 is a dependency for both c7n and c7n_mailer.. Remove the duplicate from the list
        # because not all versions of pip can handle this.
        res = [t for t in res if not any((e != t and t in e) for e in res)]
        return res

    @staticmethod<commit_msg>azure - depmgr fix duplicate packages causing failed download (#3306)<commit_after>import hashlib
import json
import logging
import os
import re
import subprocess
import sys
from builtins import bytes

logger = logging.getLogger('c7n_azure.dependency_manager')


class DependencyManager(object):

    @staticmethod
    def _run(cmd, verbose=False, **kwargs):
        if verbose:
            stdout = stderr = None
        else:
            stdout = stderr = subprocess.PIPE

        logger.debug(' '.join(cmd))
        return subprocess.run(cmd, stdout=stdout, stderr=stderr, **kwargs)

    @staticmethod
    def _get_installed_distributions():
        try:
            from pip._internal.utils.misc import get_installed_distributions
        except ImportError:
            from pip import get_installed_distributions

        return get_installed_distributions()

    @staticmethod
    def get_dependency_packages_list(packages, excluded_packages):
        dists = DependencyManager._get_installed_distributions()
        res = []
        for p in packages:
            res.extend([str(r) for r in next(d.requires() for d in dists if (p + ' ') in str(d))])

        res = [t for t in res if not any((e in t) for e in excluded_packages + packages)]

        # boto3 is a dependency for both c7n and c7n_mailer.. Remove the duplicate from the list
        # because not all versions of pip can handle this.
        # use regex to get rid of duplicates excluding version number
        regex = "^[^<>~=]*"
        for i, val in enumerate(res):
            pname = re.match(regex, val)
            if sum(pname[0].lower() in e.lower() for e in res) > 1:
                logger.debug("removing duplicate dependency:" + val)
                res.pop(i)
        return res

    @staticmethod
{'fraction_non_alphanumeric': 0.08156355801785165, 'fraction_numerical': 0.0043090181594336715, 'mean_word_length': 2.631284916201117, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '363691', 'n_tokens_mistral': 945, 'n_tokens_neox': 882, 'n_words': 310}