<commit_before>#!/usr/bin/env python
"""
$ python ia_cdx_checker.py elxn42-tweets-urls-fixed-uniq-no-count.txt | cat > elx42_urls_in_ia.txt
"""

from __future__ import print_function

import sys
import json
import fileinput
import io
from urllib2 import Request, urlopen, URLError, HTTPError

for line in fileinput.input():
    elx42_url = line.rstrip('\n')
    try:
        url = 'http://web.archive.org/cdx/search/cdx?url=' + elx42_url + '&output=json&limit=-2'
        request = Request(url, headers={'User-Agent': "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30)"})
        jsonData = urlopen(request)
        data = json.load(jsonData)
        first_date = data[1][1]
        second_date = data[2][1]
        if first_date.startswith('201508'):
            print(elx42_url)
        if first_date.startswith('201509'):
            print(elx42_url)
        if first_date.startswith('201510'):
            print(elx42_url)
        if first_date.startswith('201511'):
            print(elx42_url)
        if first_date.startswith('201512'):
            print(elx42_url)
        if second_date.startswith('201508'):
            print(elx42_url)
        if second_date.startswith('201509'):
            print(elx42_url)
        if second_date.startswith('201510'):
            print(elx42_url)
        if second_date.startswith('201511'):
            print(elx42_url)
        if second_date.startswith('201512'):
            print(elx42_url)
    except HTTPError as e:
        status_code = e.code
    except IndexError as d:
        index_error = d
    except ValueError as f:
        value_error = f<commit_msg>Let's make this multithreaded. Pool size of three seems to be ok with IA CDX API.<commit_after>#!/usr/bin/env python
"""
$ python ia_cdx_checker.py elxn42-tweets-urls-fixed-uniq-no-count.txt | cat > elx42_urls_in_ia.txt
"""

from __future__ import print_function

import sys
import json
import fileinput
import logging
import io
from urllib2 import Request, urlopen, URLError, HTTPError
import multiprocessing

POOL_SIZE = 3

logging.basicConfig(filename="ia_cdx.log", level=logging.INFO)

def check_ia_cdx(line):
    elx42_url = line.rstrip('\n')
    e = None
    d = None
    f = None
    try:
        url = 'http://web.archive.org/cdx/search/cdx?url=' + elx42_url + '&output=json&limit=-2'
        request = Request(url, headers={'User-Agent': "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30)"})
        jsonData = urlopen(request)
        data = json.load(jsonData)
        first_date = data[1][1]
        second_date = data[2][1]
        if first_date.startswith('201508'):
            print(elx42_url)
        if first_date.startswith('201509'):
            print(elx42_url)
        if first_date.startswith('201510'):
            print(elx42_url)
        if first_date.startswith('201511'):
            print(elx42_url)
        if first_date.startswith('201512'):
            print(elx42_url)
        if second_date.startswith('201508'):
            print(elx42_url)
        if second_date.startswith('201509'):
            print(elx42_url)
        if second_date.startswith('201510'):
            print(elx42_url)
        if second_date.startswith('201511'):
            print(elx42_url)
        if second_date.startswith('201512'):
            print(elx42_url)
    except HTTPError as e:
        logging.error("http error: %s when looking up %s", e, elx42_url)
    except IndexError as d:
        logging.error("index error: %s when looking up %s", d, elx42_url)
    except ValueError as f:
        logging.error("value error: %s when looking up %s", f, elx42_url)
    #except BadStatusLine as g:
    #    bad_status_line = g

def main():
    pool = multiprocessing.Pool(POOL_SIZE)
    for line in pool.imap_unordered(check_ia_cdx, fileinput.input()):
        this_is_bad = line

if __name__ == "__main__":
    main()
{'fraction_non_alphanumeric': 0.11890397184514831, 'fraction_numerical': 0.06359979889391654, 'mean_word_length': 2.6842592592592593, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4726622', 'n_tokens_mistral': 1544, 'n_tokens_neox': 1414, 'n_words': 294}