<commit_before>import gevent
from gevent import monkey
monkey.patch_all()

import os
import requests
import subprocess
import sys
import ujson as json

this_dir = os.path.realpath(os.path.dirname(__file__))
sys.path.append(os.path.realpath(os.path.join(os.pardir, os.pardir)))

from geodata.whosonfirst.crawl import WhosOnFirstCrawler
from geodata.file_utils import ensure_dir

SEED_URLS_JSON = 'https://raw.githubusercontent.com/whosonfirst-data/whosonfirst-data-postalcode/master/data.json'


def clone_repo(wof_dir, repo):
    repo_name = repo.rstrip('/').rsplit('/', 1)[-1]
    repo_dir = os.path.join(wof_dir, repo_name)

    subprocess.check_call(['rm', '-rf', repo_dir])
    subprocess.check_call(['git', 'clone', repo, repo_dir])

    return repo_dir


def download_wof_postcodes(wof_dir):
    ensure_dir(wof_dir)
    crawler = WhosOnFirstCrawler(wof_dir)

    response = requests.get(SEED_URLS_JSON)
    if response.ok:
        content = json.loads(response.content)

        for d in content:
            repo_name = d['name']

            if int(d.get('count', 0)) > 0:
                repo = d['url']
                print('doing {}'.format(repo_name))

                repo_dir = clone_repo(wof_dir, repo)

                for i, postcode in enumerate(crawler.crawl(repo_dir)):
                    if i % 100 == 0 and i > 0:
                        print('downloaded {} postcodes from WoF'.format(i))
            else:
                print('skipping {}'.format(repo_name))

if __name__ == '__main__':
    if len(sys.argv) < 2:
        sys.exit('Usage: python download_wof_postal_codes.py wof_base_dir')
    download_wof_postcodes(sys.argv[1])<commit_msg>[wof] don't need to crawl for admin data now that whosonfirst-data is separate from venues, etc.<commit_after>import os
import requests
import subprocess
import sys
import ujson as json

this_dir = os.path.realpath(os.path.dirname(__file__))
sys.path.append(os.path.realpath(os.path.join(os.pardir, os.pardir)))

from geodata.file_utils import ensure_dir


WOF_PLACE_DATA_REPO = 'https://github.com/whosonfirst-data/whosonfirst-data'
SEED_URLS_JSON = 'https://raw.githubusercontent.com/whosonfirst-data/whosonfirst-data-postalcode/master/data.json'


def clone_repo(wof_dir, repo):
    repo_name = repo.rstrip('/').rsplit('/', 1)[-1]
    repo_dir = os.path.join(wof_dir, repo_name)

    subprocess.check_call(['rm', '-rf', repo_dir])
    subprocess.check_call(['git', 'clone', repo, repo_dir])

    return repo_dir


def download_wof_postcodes(wof_dir):
    ensure_dir(wof_dir)

    clone_repo(wof_dir, WOF_PLACE_DATA_REPO)

    response = requests.get(SEED_URLS_JSON)
    if response.ok:
        content = json.loads(response.content)

        for d in content:
            repo_name = d['name']

            if int(d.get('count', 0)) > 0:
                repo = d['url']
                print('doing {}'.format(repo_name))

                repo_dir = clone_repo(wof_dir, repo)

            else:
                print('skipping {}'.format(repo_name))

if __name__ == '__main__':
    if len(sys.argv) < 2:
        sys.exit('Usage: python download_wof_postal_codes.py wof_base_dir')
    download_wof_postcodes(sys.argv[1])
{'fraction_non_alphanumeric': 0.1263322884012539, 'fraction_numerical': 0.005329153605015674, 'mean_word_length': 3.133419689119171, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6611008', 'n_tokens_mistral': 1157, 'n_tokens_neox': 1101, 'n_words': 207}