<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import sys
import re
import socket
import getopt
import threading
import subprocess
import shlex
import time
import select

blackhole = (
'fdf8:f53e:61e4::18',
'fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b',
'2001::212',
'fc00:db20:35b:7399::5',
'fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b',
'fc00:db20:35b:7399::5',
'fd00:c2b6:b24b:be67:2827:688d:e6a1:6a3b',
'fc00:e968:6179::de52:7100',
'fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b',
'fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b',
'fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b',
'fd00:c2b6:b24b:be67:2827:688d:e6a1:6a3b',
'fc00:db20:35b:7399::5')

dns = {
'google_a':'fdf8:f53e:61e4::18',
'google_b':'fc00:db20:35b:7399::5',
'he_net':'fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b',
'lax_he_net':'fc00:e968:6179::de52:7100'
}

config = {
'dns':dns['google_b'],
'infile':'',
'outfile':'',
'querytype':'aaaa',
'cname': False,
'threadnum':10
}

hosts = []
done_num = 0
thread_lock = threading.Lock()
running = True

class worker_thread(threading.Thread):
    def __init__(self, start_pt, end_pt):
        threading.Thread.__init__(self)
        self.start_pt = start_pt
        self.end_pt = end_pt
    
    def run(self):
        global hosts, done_num
        for i in range(self.start_pt, self.end_pt):
            if not running: break

            line = hosts[i].strip()
            
            if line == '' or line[0:2] == '##':
                hosts[i] = line + '\r\n'
                with thread_lock: done_num += 1
                continue

            # uncomment line
            line = line.lstrip('#')
            # split comment that appended to line
            comment = ''
            p = line.find('#')
            if p > 0:
                comment = line[p:]
                line = line[:p]

            arr = line.split()

            if len(arr) == 1:
                domain = arr[0]
            else:
                domain = arr[1]

            flag = False
            if validate_domain(domain):
                cname, ip = query_domain(domain, False)

                if ip == '' or ip in blackhole:
                    cname, ip = query_domain(domain, True)

                if ip:
                    flag = True
                    arr[0] = ip
                    if len(arr) == 1:
                        arr.append(domain)
                    if config['cname'] and cname:
                        arr.append('#' + cname)
                    else:
                        arr.append(comment)

            if not flag:
                arr[0] = '#' + arr[0]
                arr.append(comment)

            hosts[i] = ' '.join(arr)
            hosts[i] += '\r\n'
            with thread_lock: done_num += 1

class watcher_thread(threading.Thread):
    def run(self):
        total_num = len(hosts)

        wn = int(config['threadnum'])
        if wn > total_num:
            wn = total_num
        print "There are %d threads working..." % wn
        print "Press 'Enter' to exit.\n"

        while True:
            if sys.stdin in select.select([sys.stdin], [], [], 0)[0]:
                t = raw_input()
                global running
                with thread_lock:
                    running = False
                print 'Waiting threads to exit...'
                break

            with thread_lock:
                dn = done_num

            outbuf = "Total: %d lines, Done: %d lines, Ratio: %d %%.\r"\
                   % (total_num, dn, dn * 100 / total_num)
            print outbuf,
            sys.stdout.flush()

            if done_num == total_num:
                print outbuf
                break

            time.sleep(1)

def query_domain(domain, tcp):
    cmd = "dig +short +time=2 -6 %s @'%s' '%s'"\
        % (config['querytype'], config['dns'], domain)

    if tcp:
        cmd = cmd[:3] + ' +tcp' + cmd[3:]

    proc = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE)
    out, err = proc.communicate()
    outarr = out.splitlines()

    cname = ip = ''

    if len(outarr) == 0:
        return ()
    else:
        for v in outarr:
            if cname == '' and validate_domain(v[:-1]):
                cname = v[:-1]
            if ip == '' and validate_ip_addr(v):
                ip = v
                break
    
    return (cname, ip)

def validate_domain(domain):
    pattern = '^((?!-)[*A-Za-z0-9-]{1,63}(?<!-)\\.)+[A-Za-z]{2,6}$'
    p = re.compile(pattern)
    m = p.match(domain)
    if m:
        return True
    else:
        return False

def validate_ip_addr(ip_addr):
    if ':' in ip_addr:
        try:
            socket.inet_pton(socket.AF_INET6, ip_addr)
            return True
        except socket.error:
            return False
    else:
        try:<commit_msg>fix the bug when there is no respond<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import sys
import re
import socket
import getopt
import threading
import subprocess
import shlex
import time
import select

blackhole = (
'fdf8:f53e:61e4::18',
'fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b',
'2001::212',
'fc00:db20:35b:7399::5',
'fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b',
'fc00:db20:35b:7399::5',
'fd00:c2b6:b24b:be67:2827:688d:e6a1:6a3b',
'fc00:e968:6179::de52:7100',
'fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b',
'fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b',
'fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b',
'fd00:c2b6:b24b:be67:2827:688d:e6a1:6a3b',
'fc00:db20:35b:7399::5')

dns = {
'google_a':'fdf8:f53e:61e4::18',
'google_b':'fc00:db20:35b:7399::5',
'he_net':'fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b',
'lax_he_net':'fc00:e968:6179::de52:7100'
}

config = {
'dns':dns['google_b'],
'infile':'',
'outfile':'',
'querytype':'aaaa',
'cname': False,
'threadnum':10
}

hosts = []
done_num = 0
thread_lock = threading.Lock()
running = True

class worker_thread(threading.Thread):
    def __init__(self, start_pt, end_pt):
        threading.Thread.__init__(self)
        self.start_pt = start_pt
        self.end_pt = end_pt
    
    def run(self):
        global hosts, done_num
        for i in range(self.start_pt, self.end_pt):
            if not running: break

            line = hosts[i].strip()
            
            if line == '' or line[0:2] == '##':
                hosts[i] = line + '\r\n'
                with thread_lock: done_num += 1
                continue

            # uncomment line
            line = line.lstrip('#')
            # split comment that appended to line
            comment = ''
            p = line.find('#')
            if p > 0:
                comment = line[p:]
                line = line[:p]
            arr = line.split()

            if len(arr) == 1:
                domain = arr[0]
            else:
                domain = arr[1]

            flag = False
            if validate_domain(domain):
                cname, ip = query_domain(domain, False)

                if ip == '' or ip in blackhole:
                    cname, ip = query_domain(domain, True)

                if ip:
                    flag = True
                    arr[0] = ip
                    if len(arr) == 1:
                        arr.append(domain)
                    if config['cname'] and cname:
                        arr.append('#' + cname)
                    else:
                        if comment:
                            arr.append(comment)

            if not flag:
                arr[0] = '#' + arr[0]
                if comment:
                    arr.append(comment)

            hosts[i] = ' '.join(arr)
            hosts[i] += '\r\n'
            with thread_lock: done_num += 1

class watcher_thread(threading.Thread):
    def run(self):
        total_num = len(hosts)

        wn = int(config['threadnum'])
        if wn > total_num:
            wn = total_num
        print "There are %d threads working..." % wn
        print "Press 'Enter' to exit.\n"

        while True:
            if sys.stdin in select.select([sys.stdin], [], [], 0)[0]:
                t = raw_input()
                global running
                with thread_lock:
                    running = False
                print 'Waiting threads to exit...'
                break

            with thread_lock:
                dn = done_num

            outbuf = "Total: %d lines, Done: %d lines, Ratio: %d %%.\r"\
                   % (total_num, dn, dn * 100 / total_num)
            print outbuf,
            sys.stdout.flush()

            if done_num == total_num:
                print outbuf
                break

            time.sleep(1)

def query_domain(domain, tcp):
    cmd = "dig +short +time=2 -6 %s @'%s' '%s'"\
        % (config['querytype'], config['dns'], domain)

    if tcp:
        cmd = cmd[:3] + ' +tcp' + cmd[3:]

    proc = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE)
    out, err = proc.communicate()
    outarr = out.splitlines()

    cname = ip = ''
    for v in outarr:
        if cname == '' and validate_domain(v[:-1]):
            cname = v[:-1]
        if ip == '' and validate_ip_addr(v):
            ip = v
            break
    
    return (cname, ip)

def validate_domain(domain):
    pattern = '^((?!-)[*A-Za-z0-9-]{1,63}(?<!-)\\.)+[A-Za-z]{2,6}$'
    p = re.compile(pattern)
    m = p.match(domain)
    if m:
        return True
    else:
        return False

def validate_ip_addr(ip_addr):
    if ':' in ip_addr:
        try:
            socket.inet_pton(socket.AF_INET6, ip_addr)
            return True
        except socket.error:
            return False
    else:
        try:
{'fraction_non_alphanumeric': 0.14210747271200672, 'fraction_numerical': 0.05992863140218304, 'mean_word_length': 1.694091037602488, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6462393', 'n_tokens_mistral': 3669, 'n_tokens_neox': 3293, 'n_words': 681}