<commit_before>from __future__ import absolute_import
import argparse
import importlib
import logging
import os
import pkgutil
import re
import subprocess

import six
import tldextract
from lexicon import providers
from lexicon.config import (
    ArgsConfigSource,
    ConfigResolver,
    legacy_config_resolver,
)


LOGGER = logging.getLogger(__name__)


def _get_available_providers():
    available_providers = {}
    for _, modname, _ in pkgutil.iter_modules(providers.__path__):
        if modname != 'base' and modname != 'auto':
            try:
                available_providers[modname] = importlib.import_module(
                    'lexicon.providers.' + modname)
            except ImportError:
                LOGGER.warn('Warning, the provider {0} cannot be loaded due '
                            'to missing optional dependencies.'
                            .format(modname))

    return available_providers


AVAILABLE_PROVIDERS = _get_available_providers()


def _get_ns_records_domains_for_domain(domain):
    tlds = [tldextract.extract(ns_entry)
            for ns_entry in _get_ns_records_for_domain(domain)]

    return set(['{0}.{1}'.format(tld.domain, tld.suffix) for tld in tlds])


def _get_ns_records_for_domain(domain):
    # Available both for Windows and Linux (if dnsutils is installed for the latter)
    try:
        output = subprocess.check_output(['nslookup', '-querytype=NS', domain],
                                         stderr=subprocess.STDOUT, universal_newlines=True)
    except subprocess.CalledProcessError as e:
        if 'NXDOMAIN' in e.output:
            raise ValueError(
                'Error, domain {0} could not be resolved.'.format(domain))

    pattern = re.compile(r'nameserver = (.*?)\.*{0}'.format(os.linesep))
    match = pattern.findall(output)

    if not match:
        raise ValueError('Error, could not find ns entries for domain {0}. '
                         'Does this domain is correctly configured ?'.format(domain))

    return match


def _relevant_provider_for_domain(domain):
    nameserver_domains = _get_ns_records_domains_for_domain(domain)
    relevant_providers = []

    for provider_name, provider_module in AVAILABLE_PROVIDERS.items():
        ns_domains = provider_module.NAMESERVER_DOMAINS

        # Test plain domain string comparison
        if set([ns_domain for ns_domain in ns_domains
                if isinstance(ns_domain, six.string_types)]) & nameserver_domains:
            relevant_providers.append((provider_name, provider_module))
            continue

        # Test domains regexp matching
        for ns_domain in ns_domains:
            if hasattr(ns_domain, 'match') \
                    and [nameserver_domain for nameserver_domain
                         in nameserver_domains if ns_domain.match(nameserver_domain)]:<commit_msg>Correct nslookup output on Windows<commit_after>from __future__ import absolute_import
import argparse
import importlib
import logging
import os
import pkgutil
import re
import subprocess

import six
import tldextract
from lexicon import providers
from lexicon.config import (
    ArgsConfigSource,
    ConfigResolver,
    legacy_config_resolver,
)


LOGGER = logging.getLogger(__name__)


def _get_available_providers():
    available_providers = {}
    for _, modname, _ in pkgutil.iter_modules(providers.__path__):
        if modname != 'base' and modname != 'auto':
            try:
                available_providers[modname] = importlib.import_module(
                    'lexicon.providers.' + modname)
            except ImportError:
                LOGGER.warn('Warning, the provider {0} cannot be loaded due '
                            'to missing optional dependencies.'
                            .format(modname))

    return available_providers


AVAILABLE_PROVIDERS = _get_available_providers()


def _get_ns_records_domains_for_domain(domain):
    tlds = [tldextract.extract(ns_entry)
            for ns_entry in _get_ns_records_for_domain(domain)]

    return set(['{0}.{1}'.format(tld.domain, tld.suffix) for tld in tlds])


def _get_ns_records_for_domain(domain):
    # Available both for Windows and Linux (if dnsutils is installed for the latter)
    try:
        output = subprocess.check_output(['nslookup', '-querytype=NS', domain],
                                         stderr=subprocess.STDOUT, universal_newlines=True)
    except subprocess.CalledProcessError as e:
        if 'NXDOMAIN' in e.output:
            raise ValueError(
                'Error, domain {0} could not be resolved.'.format(domain))
        output = e.output

    pattern = re.compile(r'nameserver = (.*?)\.*\n')
    match = pattern.findall(output)

    if not match:
        raise ValueError('Error, could not find ns entries for domain {0}. '
                         'Does this domain is correctly configured ?'.format(domain))

    return match


def _relevant_provider_for_domain(domain):
    nameserver_domains = _get_ns_records_domains_for_domain(domain)
    relevant_providers = []

    for provider_name, provider_module in AVAILABLE_PROVIDERS.items():
        ns_domains = provider_module.NAMESERVER_DOMAINS

        # Test plain domain string comparison
        if set([ns_domain for ns_domain in ns_domains
                if isinstance(ns_domain, six.string_types)]) & nameserver_domains:
            relevant_providers.append((provider_name, provider_module))
            continue

        # Test domains regexp matching
        for ns_domain in ns_domains:
            if hasattr(ns_domain, 'match') \
                    and [nameserver_domain for nameserver_domain
                         in nameserver_domains if ns_domain.match(nameserver_domain)]:
{'fraction_non_alphanumeric': 0.07740695439454831, 'fraction_numerical': 0.0019220688450113576, 'mean_word_length': 2.702457956015524, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6168567', 'n_tokens_mistral': 1640, 'n_tokens_neox': 1562, 'n_words': 409}