<commit_before>from __future__ import absolute_import

import socket

from ._iputils import parse_ip


def parse_server(server):
    """
    >>> parse_server("192.0.2.0") == (socket.AF_INET, "192.0.2.0", 53)
    True
    >>> parse_server("2001:db8::") == (socket.AF_INET6, "2001:db8::", 53)
    True
    """

    family, ip = parse_ip(server)
    return family, ip, 53


def read_resolv_conf(line_iterator):
    for line in line_iterator:
        line = line.strip()
        if not line or line.startswith("#"):
            continue

        pieces = line.split(None, 1)
        if len(pieces) < 2:
            continue

        key, value = pieces
        yield key.lower(), value


def read_hosts(line_iterator):
    for line in line_iterator:
        line = line.strip()
        if not line or line.startswith("#"):
            continue

        pieces = line.split()
        if len(pieces) < 2:
            continue

        try:
            _, ip = parse_ip(pieces[0])
        except ValueError:
            pass
        names = set(pieces[1:])
        yield ip, names


def uniques(values):
    seen_values = set()

    for value in values:
        if value in seen_values:
            continue
        seen_values.add(value)
        yield value
<commit_msg>idiokit.dns: Also semicolons may start comment lines in /etc/resolv.conf. Trim /etc/hosts lines ending with comments.<commit_after>from __future__ import absolute_import

import socket

from ._iputils import parse_ip


def parse_server(server):
    """
    >>> parse_server("192.0.2.0") == (socket.AF_INET, "192.0.2.0", 53)
    True
    >>> parse_server("2001:db8::") == (socket.AF_INET6, "2001:db8::", 53)
    True
    """

    family, ip = parse_ip(server)
    return family, ip, 53


def read_resolv_conf(line_iterator):
    for line in line_iterator:
        line = line.strip()
        if not line or line.startswith("#") or line.startswith(";"):
            continue

        pieces = line.split(None, 1)
        if len(pieces) < 2:
            continue

        key, value = pieces
        yield key.lower(), value


def read_hosts(line_iterator):
    for line in line_iterator:
        comment_start = line.find("#")
        if comment_start >= -1:
            line = line[:comment_start]

        line = line.strip()
        if not line:
            continue

        pieces = line.split()
        if len(pieces) < 2:
            continue

        try:
            _, ip = parse_ip(pieces[0])
        except ValueError:
            pass
        names = set(pieces[1:])
        yield ip, names


def uniques(values):
    seen_values = set()

    for value in values:
        if value in seen_values:
            continue
        seen_values.add(value)
        yield value

{'fraction_non_alphanumeric': 0.11224489795918367, 'fraction_numerical': 0.025145772594752185, 'mean_word_length': 2.0231277533039647, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4851846', 'n_tokens_mistral': 903, 'n_tokens_neox': 839, 'n_words': 227}