<commit_before>#!/usr/bin/env python3

import sys
import json
import requests
import re

def scrape(thread_url):
	m = re.search(r'4chan.org/(.*?)/thread/(\d*)(:?/.*)?', thread_url)
	if not m:
		print("[FAIL] Invalid thread url, skipping: "+thread_url, file=sys.stderr)
		return

	board, thread_id = m.group(1, 2)

	r = requests.get(thread_url+'.json')
	r = requests.get('https://a.4cdn.org/{board}/thread/{thread_id}.json'.format(**locals()))

	if r.status_code == 404:
		print("[FAIL] Thread 404'd, skipping: "+thread_url, file=sys.stderr)
		return

	r.raise_for_status() # Throw at other errors.
	print("[ OK ] "+thread_url, file=sys.stderr)

	data = r.json()

	with open('thread.{board}.{thread_id}.json'.format(**locals()), 'w') as thread_f:
		json.dump(data, thread_f,
			sort_keys=True,
			ensure_ascii=False,
			indent='\t')
		thread_f.write('\n')

	for post in data['posts']:
		try:
			filename = str(post['tim']) + post['ext']
		except KeyError: # No image in post
			continue
		file_url = "http://images.4chan.org/{board}/src/{filename}".format(**locals())
		print(file_url)
	sys.stdout.write('\n')

def main(urls):
	if not urls:
		urls = [ line.strip() for line in sys.stdin if not line.isspace() ]

	for thread_url in urls:
		scrape(thread_url)

if __name__ == "__main__":
	main(sys.argv[1:])
<commit_msg>Use thread names ("semantic URL" parts) in thread info filename.<commit_after>#!/usr/bin/env python3

import sys
import json
import requests
import re

def scrape(thread_url):
	m = re.search(r'4chan.org/(.*?)/thread/(\d*)(:?/.*)?', thread_url)
	if not m:
		print("[FAIL] Invalid thread url, skipping: "+thread_url, file=sys.stderr)
		return

	board, thread_id = m.group(1, 2)

	r = requests.get(thread_url+'.json')
	r = requests.get('https://a.4cdn.org/{board}/thread/{thread_id}.json'.format(**locals()))

	if r.status_code == 404:
		print("[FAIL] Thread 404'd, skipping: "+thread_url, file=sys.stderr)
		return

	r.raise_for_status() # Throw at other errors.
	print("[ OK ] "+thread_url, file=sys.stderr)

	data = r.json()

	semantic_url = data['posts'][0]['semantic_url']
	with open('thread.{board}.{thread_id}.{semantic_url}.json'.format(**locals()), 'w') as thread_f:
		json.dump(data, thread_f,
			sort_keys=True,
			ensure_ascii=False,
			indent='\t')
		thread_f.write('\n')

	for post in data['posts']:
		try:
			filename = str(post['tim']) + post['ext']
		except KeyError: # No image in post
			continue
		file_url = "http://images.4chan.org/{board}/src/{filename}".format(**locals())
		print(file_url)
	sys.stdout.write('\n')

def main(urls):
	if not urls:
		urls = [ line.strip() for line in sys.stdin if not line.isspace() ]

	for thread_url in urls:
		scrape(thread_url)

if __name__ == "__main__":
	main(sys.argv[1:])

{'fraction_non_alphanumeric': 0.18734085121862495, 'fraction_numerical': 0.009821753364859949, 'mean_word_length': 4.434782608695652, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2569860', 'n_tokens_mistral': 1112, 'n_tokens_neox': 1034, 'n_words': 227}