<commit_before>#!/usr/bin/python3
import time
import pycurl
import os
import errno
import code
import re
import json
import sys
import getopt
import filecmp
from replace_post_tex import replace_dollar_tex
from replace_post_tex import replace_display_tex
from replace_post_tex import replace_inline_tex
from io import BytesIO
from bs4 import BeautifulSoup

root_url = "http://math.stackexchange.com"
vt100_WARNING = '\033[93m'
vt100_RESET = '\033[0m'
DIVISIONS = 500

def print_err(err_str):
	f = open("error.log", "a")
	print(vt100_WARNING)
	err_str = "[error] " + err_str
	f.write(err_str + '\n')
	print(err_str)
	print(vt100_RESET)
	f.close()

def curl(sub_url, c):
	buf = BytesIO()
	print('[curl] %s' % sub_url)
	url = root_url + sub_url
	url = url.encode('iso-8859-1')
	c.setopt(c.URL, url)
	c.setopt(c.WRITEFUNCTION, buf.write)
	errs = 0
	while True:
		try:
			c.perform()
		except (KeyboardInterrupt, SystemExit):
			print('user aborting...')
			raise
		except:
			errs = errs + 1
			if errs > 3:
				buf.close()
				raise
			print("[curl] try again...")
			continue
		break
	res_str = buf.getvalue()
	buf.close()
	return res_str

def extract_p_tag_text(soup, id_str):
	txt = ''
	id_tag = soup.find(id=id_str)
	if id_tag is None:
		raise Exception("id_tag is None")
	p_tags = id_tag.find_all('p')
	for p in p_tags:
		if p.string is not None:
			txt += str(p.string)
			txt += '\n'
	return txt

def crawl_post_page(sub_url, c):
	try:
		post_page = curl(sub_url, c)
	except:
		raise
	s = BeautifulSoup(post_page, "html.parser")
	# get title
	question_header = s.find(id='question-header')
	if question_header is None:
		raise
	title = str(question_header.h1.string)
	post_txt = title + '\n\n'
	# get question
	try:
		post_txt += extract_p_tag_text(s, "question")
	except:
		raise
	post_txt += '\n'
	# get answers
	try:
		post_txt += extract_p_tag_text(s, "answers")
	except:
		raise<commit_msg>Use .text instead of .string to extract text.<commit_after>#!/usr/bin/python3
import time
import pycurl
import os
import errno
import code
import re
import json
import sys
import getopt
import filecmp
from replace_post_tex import replace_dollar_tex
from replace_post_tex import replace_display_tex
from replace_post_tex import replace_inline_tex
from io import BytesIO
from bs4 import BeautifulSoup

root_url = "http://math.stackexchange.com"
vt100_WARNING = '\033[93m'
vt100_RESET = '\033[0m'
DIVISIONS = 500

def print_err(err_str):
	f = open("error.log", "a")
	print(vt100_WARNING)
	err_str = "[error] " + err_str
	f.write(err_str + '\n')
	print(err_str)
	print(vt100_RESET)
	f.close()

def curl(sub_url, c):
	buf = BytesIO()
	print('[curl] %s' % sub_url)
	url = root_url + sub_url
	url = url.encode('iso-8859-1')
	c.setopt(c.URL, url)
	c.setopt(c.WRITEFUNCTION, buf.write)
	errs = 0
	while True:
		try:
			c.perform()
		except (KeyboardInterrupt, SystemExit):
			print('user aborting...')
			raise
		except:
			errs = errs + 1
			if errs > 3:
				buf.close()
				raise
			print("[curl] try again...")
			continue
		break
	res_str = buf.getvalue()
	buf.close()
	return res_str

def extract_p_tag_text(soup):
	txt = ''
	p_tags = soup.find_all('p')
	for p in p_tags:
		if p.text is not ' ':
			txt += str(p.text)
			txt += '\n'
	return txt

def crawl_post_page(sub_url, c):
	try:
		post_page = curl(sub_url, c)
	except:
		raise
	s = BeautifulSoup(post_page, "html.parser")
	# get title
	question_header = s.find(id='question-header')
	if question_header is None:
		raise
	title = str(question_header.h1.string)
	post_txt = title + '\n\n'
	# get question
	question = s.find(id="question")
	if question is None:
		raise Exception("question tag is None")
	post_txt += extract_p_tag_text(question)
	post_txt += '\n'
	# get answers
	answers = s.find(id="answers")
	if answers is None:
		raise Exception("answers tag is None")
	for answer in answers.findAll('div',{'class':'answer'}):
		post_txt += extract_p_tag_text(answer)
		post_txt += '\n'
{'fraction_non_alphanumeric': 0.12725425450850902, 'fraction_numerical': 0.01778003556007112, 'mean_word_length': 4.140992167101827, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6879941', 'n_tokens_mistral': 1666, 'n_tokens_neox': 1528, 'n_words': 386}