<commit_before>import httplib2
import logging
import os
import socket
import sys
import urllib
from BeautifulSoup import BeautifulSoup, SoupStrainer


def create_config_files(directory):
    """
    Initialize directory ready for vpn walker
    :param directory: the path where you want this to happen
    :return:
    """
    # Some constant strings
    crt_url = "http://www.ipvanish.com/software/configs/ca.ipvanish.com.crt"
    config_set_url = "http://www.ipvanish.com/software/configs/"

    config_urls = []
    # Getting all available config files on webpage
    http = httplib2.Http()
    status, response = http.request(config_set_url)
    for link in BeautifulSoup(response, parseOnlyThese=SoupStrainer('a')):
        if link.has_key('href') and link['href'].endswith('.ovpn'):
            config_urls.append(config_set_url + link['href'])

    if not os.path.exists(directory):
        os.makedirs(directory)

    server_country = {}

    # Download certificate and configs
    url_opener = urllib.URLopener()
    url_opener.retrieve(crt_url, os.path.join(directory, '../ca.ipvanish.com.crt'))
    logging.info("Starting to download IPVanish config files. This might take a while.")
    for ovpn_link in config_urls:
        filename = ovpn_link.split('/')[-1]
        file_path = os.path.join(directory, filename)
        url_opener.retrieve(ovpn_link, file_path)
        # add dns update options to each file
        with open(file_path, 'a') as f:
            f.write("up /etc/openvpn/update-resolv-conf\n")
            f.write("down /etc/openvpn/update-resolv-conf\n")

    # rename all config files using their ip address
    for filename in os.listdir(directory):
        country = filename.split('-')[1]
        file_path = os.path.join(directory, filename)
        lines = [line.rstrip('\n') for line in open(file_path)]

        # get ip address for this vpn
        ip = ""
        for line in lines:
            if line.startswith('remote'):
                hostname = line.split(' ')[1]
                ip = socket.gethostbyname(hostname)
                break

        if len(ip) > 0:
            new_path = os.path.join(directory, ip + '.ovpn')
            os.rename(file_path, new_path)
            server_country[ip] = country
        else:
            logging.warn("Unable to resolve hostname and remove %s" % filename)
            os.remove(file_path)

    with open(os.path.join(directory, 'servers.txt'), 'w') as f:
        for ip in server_country:
            f.write('|'.join([ip, server_country[ip]]) + '\n')


if __name__ == "__main__":
    if len(sys.argv) != 2:
        print "Usage {0} <directory to create VPNs in>".format(sys.argv[0])
        sys.exit(1)
    create_config_files(sys.argv[1])
<commit_msg>updated IPVanish config fetcher, fixes #269<commit_after>import httplib2
import logging
import os
import shutil
import socket
import sys
import urllib
import zipfile


def unzip(source_filename, dest_dir):
    with zipfile.ZipFile(source_filename) as zf:
        zf.extractall(dest_dir)


def create_config_files(directory):
    """
    Initialize directory ready for vpn walker
    :param directory: the path where you want this to happen
    :return:
    """
    # Some constant strings
    config_zip_url = "http://www.ipvanish.com/software/configs/configs.zip"

    if not os.path.exists(directory):
        os.makedirs(directory)

    logging.info("Starting to download IPVanish config file zip")
    url_opener = urllib.URLopener()
    zip_path = os.path.join(directory, '../configs.zip')
    unzip_path = os.path.join(directory, '../unzipped')
    if not os.path.exists(unzip_path):
        os.makedirs(unzip_path)

    url_opener.retrieve(config_zip_url, zip_path)
    logging.info("Extracting zip file")
    unzip(zip_path, unzip_path)

    # remove zip file
    os.remove(zip_path)
    # copy ca and key to root path
    shutil.copyfile(os.path.join(unzip_path, 'ca.ipvanish.com.crt'),
                    os.path.join(directory, '../ca.crt'))

    # move all config files to /vpns

    server_country = {}
    for filename in os.listdir(unzip_path):
        if filename.endswith('.ovpn'):
            country = filename.split('-')[1]

            file_path = os.path.join(unzip_path, filename)
            lines = [line.rstrip('\n') for line in open(file_path)]

            # get ip address for this vpn
            ip = ""
            for line in lines:
                if line.startswith('remote'):
                    hostname = line.split(' ')[1]
                    ip = socket.gethostbyname(hostname)
                    break

            if len(ip) > 0:
                new_path = os.path.join(directory, ip + '.ovpn')
                shutil.copyfile(file_path, new_path)
                server_country[ip] = country
            else:
                logging.warn("Unable to resolve hostname and remove %s" % filename)
                os.remove(file_path)

    with open(os.path.join(directory, 'servers.txt'), 'w') as f:
        for ip in server_country:
            f.write('|'.join([ip, server_country[ip]]) + '\n')

    # remove extracted folder
    shutil.rmtree(unzip_path)

if __name__ == "__main__":
    if len(sys.argv) != 2:
        print "Usage {0} <directory to create VPNs in>".format(sys.argv[0])
        sys.exit(1)
    create_config_files(sys.argv[1])

{'fraction_non_alphanumeric': 0.11736032995875516, 'fraction_numerical': 0.00431196100487439, 'mean_word_length': 2.4914921465968587, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2452696', 'n_tokens_mistral': 1712, 'n_tokens_neox': 1678, 'n_words': 406}