<commit_before>#!/usr/bin/env python3
import berserker_resolver
import os
import shutil
import time

from contextlib import contextmanager
from urllib.parse import urlparse


NAMESERVERS = ('8.8.8.8', '8.8.4.4', '172.16.31.10', '192.168.3.11')
HOSTS_PATH = os.environ.get('HOSTSUPDATE_HOSTS_PATH')
CCD_FILEPATH = os.environ.get('HOSTSUPDATE_CCD_FILEPATH')


@contextmanager
def timed():
    t = time.time()
    yield
    print('Elapsed time: {:.1f}s'.format(time.time() - t))


def prepare_host(url):
    if '//' not in url:
        url = 'http://' + url

    hostname = urlparse(url).hostname
    if not hostname:
        raise ValueError('Cannot get hostname from: {!r}'.format(url))

    return hostname


def get_hosts(filepath):
    with open(filepath, 'r') as f:
        return set(prepare_host(h.strip()) for h in f)


def write_ovpn_ccd(ips, dstpath):
    with open('ovpn.ccd', 'w') as f:
        for ip in ips:
            f.write('push "route {ip} 255.255.255.255"\n'.format(ip=ip))

    shutil.copyfile('ovpn.ccd', dstpath)


def main():
    hosts = get_hosts(HOSTS_PATH)

    print('Start to resolve {} hostnames...'.format(len(hosts)))
    resolver = berserker_resolver.Resolver(
        tries=12, nameservers=NAMESERVERS, www=True, www_combine=True)

    resolved_ips = set()
    for resolves in resolver.resolve(hosts).values():
        resolved_ips.update(r.address for r in resolves)

    if CCD_FILEPATH:
        write_ovpn_ccd(resolved_ips, CCD_FILEPATH)
        print('Written {} IP addresses.'.format(len(resolved_ips)))
    else:
        print('"HOSTSUPDATE_CCD_FILEPATH" environment variable is not found.')


if __name__ == '__main__':
    with timed():
        main()<commit_msg>Squash IP-s if more than 5 in one subnet<commit_after>#!/usr/bin/env python3
import berserker_resolver
import os
import shutil
import time

from contextlib import contextmanager
from itertools import groupby
from urllib.parse import urlparse


NAMESERVERS = ('8.8.8.8', '8.8.4.4', '172.16.31.10', '192.168.3.11')
HOSTS_PATH = os.environ.get('HOSTSUPDATE_HOSTS_PATH')
CCD_FILEPATH = os.environ.get('HOSTSUPDATE_CCD_FILEPATH')


@contextmanager
def timed():
    t = time.time()
    yield
    print('Elapsed time: {:.1f}s'.format(time.time() - t))


def prepare_host(url):
    if '//' not in url:
        url = 'http://' + url

    hostname = urlparse(url).hostname
    if not hostname:
        raise ValueError('Cannot get hostname from: {!r}'.format(url))

    return hostname


def get_hosts(filepath):
    with open(filepath, 'r') as f:
        return set(prepare_host(h.strip()) for h in f)


def write_ovpn_ccd(ip_mask_list, dstpath):
    with open('ovpn.ccd', 'w') as f:
        for ip, mask in ip_mask_list:
            f.write('push "route {ip} {mask}"\n'.format(ip=ip, mask=mask))

    shutil.copyfile('ovpn.ccd', dstpath)


def make_24_subnet(ip):
    return ip.rsplit('.', 1)[0] + '.0'


def get_squashed_ips_with_masks(ips):
    ip_mask_list = []
    for squashed_ip, g in groupby(sorted(ips), make_24_subnet):
        grouped_ips = list(g)
        if len(grouped_ips) > 5:
            ip_mask_list.append(
                (squashed_ip, '255.255.255.0'))
        else:
            ip_mask_list.extend(
                (ip, '255.255.255.255') for ip in grouped_ips)
    return ip_mask_list


def main():
    hosts = get_hosts(HOSTS_PATH)

    print('Start to resolve {} hostnames...'.format(len(hosts)))
    resolver = berserker_resolver.Resolver(
        tries=12, nameservers=NAMESERVERS, www=True, www_combine=True)

    resolved_ips = set()
    for resolves in resolver.resolve(hosts).values():
        resolved_ips.update(r.address for r in resolves)

    if CCD_FILEPATH:
        ip_mask_list = get_squashed_ips_with_masks(resolved_ips)
        write_ovpn_ccd(ip_mask_list, CCD_FILEPATH)
        print('Recieved {} IP addresses.'.format(len(resolved_ips)))
        print('Written {} squashed IP addresses.'.format(len(ip_mask_list)))
    else:
        print('"HOSTSUPDATE_CCD_FILEPATH" environment variable is not found.')


if __name__ == '__main__':
    with timed():
        main()
{'fraction_non_alphanumeric': 0.13429198147696808, 'fraction_numerical': 0.025103582744333414, 'mean_word_length': 3.266112266112266, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 11, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '327962', 'n_tokens_mistral': 1544, 'n_tokens_neox': 1450, 'n_words': 322}