<commit_before>'''
Author: <NAME>
readPDF util, calls `pdftotext` command line util from Apache Xpdf to convert to txt
then parse line by line and record everything into featureMap (no record column puts NA)
'''

import re, os, json, urllib.request
from scoreMap import SCORE_MAP

URL = "http://www.hear.org/pier/"
JSON_FILE = "plantpdfs.json"


# make sure in download phase the file name has no space
def getTextName(pdfPath):
	return pdfPath[:-3] + "txt"

def generateText(pdfPath, textPath):
	try:
		os.system("pdftotext -table {} {}".format(pdfPath, textPath))
		return True
	except:
		return False

def isTableLine(tokens):
	try:
		float(tokens[0])
		return len(tokens) > 1
	except:
		return False

def analyzeTableLine(dic, tokens):
	try:
		featureID = int(tokens[0])
	except:
		featureID = int(float(tokens[0]) * 100)

	if featureID == 301 and tokens[-1] == "n":
		dic[featureID] = dic[205]
	else:
		# print(tokens[-1], SCORE_MAP[featureID])
		score = SCORE_MAP[featureID].get(tokens[-1])
		if score != None:
			dic[featureID] = (tokens[-1], score)
		else:
			dic[featureID] = "NA"

def reachEnd(tokens):
	return tokens[0] == "Designation:"

def calculateScore(featureDict):
	return 


def readPDF(pdfPath, featureDict):

	textPath = getTextName(pdfPath)

	if True or generateText(pdfPath):

		file = open(textPath, encoding="ISO-8859-1")

		for line in file:

			tokens = [s for s in re.split("\s+", line) if s]

			if tokens:
				if isTableLine(tokens):
					# print(tokens)
					analyzeTableLine(featureDict, tokens)
				elif reachEnd(tokens):
					return

	else:
		print("Please make sure Xpdf installed and pdfPath correct")


def import_files(json_file):
	data = json.load(open(json_file))

	if not os.path.exists("data"):
		os.makedirs("data")
	for line in data:
		file_name = line.split('/')[-1].replace("%20", '_')
		try:
			urllib.request.urlretrieve(URL + line, "data/" + file_name)
			print("downloaded:", file_name)
		except:
			print("****failed download:", file_name)



def main():
	for f in os.listdir("data"):
		try:
			print("*****************************************")
			print("*********START:", f)
			pdf_name = "data/" + f
			txt_name = getTextName(pdf_name)
			# generateText(pdf_name, txt_name)

			featureDict = dict()
			readPDF(txt_name, featureDict)

			for k in sorted(featureDict.keys()):
				print(">>", k, ":", featureDict[k])
			print(sum([v[1] for v in featureDict.values() if v != "NA"]))
		except:
			print("FAILED!!!!!!!!", f)

if __name__ == '__main__':
	# import_files(JSON_FILE)
	main()
	







<commit_msg>Jack Ye: add write to csv funcitonality<commit_after>'''
Author: <NAME>
readPDF util, calls `pdftotext` command line util from Apache Xpdf to convert to txt
then parse line by line and record everything into featureMap (no record column puts NA)
'''

import re, os, json, csv, urllib.request
from scoreMap import SCORE_MAP


URL = "http://www.hear.org/pier/"
JSON_FILE = "plantpdfs.json"
CSV_PATH = "pier_data.csv"
DATA_DIR = "data"


def getTextName(pdfPath):
	return pdfPath[:-3] + "txt"

def getPlantName(pdfPath):
	return pdfPath[:-4]


def generateText(pdfPath, textPath):
	try:
		os.system("pdftotext -table {} {}".format(pdfPath, textPath))
	except:
		print("Please make sure Xpdf installed and pdfPath correct")


def isTableLine(tokens):
	try:
		float(tokens[0])
		return len(tokens) > 1
	except:
		return False


def analyzeTableLine(dic, tokens):
	try:
		featureID = int(tokens[0])
	except:
		featureID = int(float(tokens[0]) * 100)

	if featureID == 301 and tokens[-1] == "n":
		dic[featureID] = dic[205]
	else:
		# print(tokens[-1], SCORE_MAP[featureID])
		score = SCORE_MAP[featureID].get(tokens[-1])
		if score != None:
			dic[featureID] = (tokens[-1], score)
		else:
			dic[featureID] = ("NA", "NA")


def reachEnd(tokens):
	return tokens[0] == "Designation:"


def calculateScore(featureDict):
	return sum([v[1] for v in featureDict.values() if v != "NA"])


def writeToCSV(csvPath, plant, featureDict):
	print("	Writing",plant,"data to csv file")
	with open(csvPath, 'a') as file:
		for k in sorted(featureDict.keys()):
			lst = [str(i) for i in [plant, k] + list(featureDict[k])]
			file.write(",".join(lst))
			file.write("\n")


def readPDF(txtPath):

	featureDict = dict()
	file = open(txtPath, encoding="ISO-8859-1")

	print("	Analyzing table lines")
	for line in file:
		tokens = [s for s in re.split("\s+", line) if s]
		if tokens:
			if isTableLine(tokens):
				analyzeTableLine(featureDict, tokens)
			elif reachEnd(tokens):
				return featureDict



def importFiles(jsonFile):
	data = json.load(open(jsonFile))

	if not os.path.exists(DATA_DIR):
		os.makedirs(DATA_DIR)
	for line in data:
		fileName = line.split('/')[-1].replace("%20", '_')
		try:
			urllib.request.urlretrieve(URL + line, DATA_DIR + "/" + fileName)
			print("Downloaded:", fileName)
		except:
			print("**Failed download:", fileName)



def main():
	
	importFiles(JSON_FILE)

	for f in os.listdir(DATA_DIR):
		try:
			print("START:", f)
			pdfName = DATA_DIR + "/" + f
			txtName = getTextName(pdfName)
			generateText(pdfName, txtName)
			featureDict = readPDF(txtName)

			# for k in sorted(featureDict.keys()):
			# 	print(">>", k, ":", featureDict[k])
			# print(calculateScore(featureDict))
			writeToCSV(CSV_PATH, getPlantName(f), featureDict)
			print("	SUCCESSED", f)

		except:
			print("	FAILED", f)


if __name__ == '__main__':
	main()
	








{'fraction_non_alphanumeric': 0.16007393715341958, 'fraction_numerical': 0.010536044362292052, 'mean_word_length': 4.248302618816683, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4753664', 'n_tokens_mistral': 2147, 'n_tokens_neox': 1920, 'n_words': 506}