<commit_before>#!/usr/bin/env python2.7
import sys, os, shutil, logging, tempfile, time, subprocess, math, re, urllib, json, codecs, csv, traceback, platform
import xml.etree.ElementTree as et
from lib.stemutil import stem
import copy
import textprocessor

class Mallet(textprocessor.TextProcessor):
	"""
	Base class for MALLET functionality
	"""

	def _basic_params(self):
		self.name = "mallet"

	def _import_dfr_metadata(self, dfr_dir):
		citation_file = os.path.join(dfr_dir, "citations.CSV")
		citations = {}
		for rowdict in self.parse_csv(citation_file):
			doi = rowdict.pop("id")
			citations[doi] = rowdict
			self.metadata[doi] = {'title': citations[doi].get("title", ""), 'date': citations[doi].get('pubdate',''), 'year': citations[doi].get('pubdate','')[0:4], 'label': "jstor", 'itemID': doi}
		return citations

	def _import_dfr(self, dfr_dir):
		citations = self._import_dfr_metadata(dfr_dir)

		wordcounts_dir = os.path.join(dfr_dir, "wordcounts")
		for doi in citations.keys():
			try:
				this_text = ''		
				for rowdict in self.parse_csv(os.path.join(wordcounts_dir, "wordcounts_" + doi.replace('/','_') + ".CSV")):
					word = rowdict["WORDCOUNTS"]
					if word in self.stopwords:
						continue
					if self.stemming:
						prestem = word
						if word not in self.stemmed:
							self.stemmed[prestem] = stem(self, prestem)
						word = self.stemmed[prestem]
					count = int(rowdict["WEIGHT"])

					this_text += (word + u' ') * count
				if len(this_text) < 20:
					continue
				yield doi, this_text
			except:
				logging.error(doi)
				logging.error(traceback.format_exc())

	def _output_text(self, text, f, filename):
		text = re.sub(r"[^\w ]+", u' ', text.lower(), flags=re.UNICODE)
		if self.stemming:
			newtext = u''
			for word in text.split():
				if word not in self.stemmed:
					self.stemmed[word] = stem(self, word)
				newtext += self.stemmed[word] + u' '
			text = newtext
		f.write(u'\t'.join([filename, self.metadata[filename]["label"], text]) + u'\n')
		self.docs.append(filename)

	def _import_files(self):
		if self.stemming:
			self.stemmed = {}
		self.docs = []
		self.segmentation = getattr(self, "segmentation", False)

		with codecs.open(self.texts_file, 'w', encoding='utf-8') as f:
			for filename in self.files:
				with codecs.open(filename, 'r', encoding='utf-8') as input_file:
					text = input_file.read()
					if self.segmentation:
						segments = filter(lambda x: x.count(' ') > 5, text.split("\n\n"))
						for i, text_seg in enumerate(segments):
							seg_filename = filename + "#" + str(i)
							self.metadata[seg_filename] = self.metadata[filename]
							self._output_text(text_seg, f, seg_filename)
					else:
						self._output_text(text, f, filename)
			if self.dfr:
				for doi, text in self._import_dfr(self.dfr_dir):
					f.write(u'\t'.join([doi, self.metadata[doi]["label"], text]) + u'\n')
					self.docs.append(doi)
		with codecs.open(os.path.join(self.mallet_out_dir, "dmap"), 'w', encoding='utf-8') as dmap:
			dmap.writelines([x + u'\n' for x in self.docs])
		self.doc_count = len(self.docs)<commit_msg>ignore short words when stemming and not TFIDF<commit_after>#!/usr/bin/env python2.7
import sys, os, shutil, logging, tempfile, time, subprocess, math, re, urllib, json, codecs, csv, traceback, platform
import xml.etree.ElementTree as et
from lib.stemutil import stem
import copy
import textprocessor

class Mallet(textprocessor.TextProcessor):
	"""
	Base class for MALLET functionality
	"""

	def _basic_params(self):
		self.name = "mallet"

	def _import_dfr_metadata(self, dfr_dir):
		citation_file = os.path.join(dfr_dir, "citations.CSV")
		citations = {}
		for rowdict in self.parse_csv(citation_file):
			doi = rowdict.pop("id")
			citations[doi] = rowdict
			self.metadata[doi] = {'title': citations[doi].get("title", ""), 'date': citations[doi].get('pubdate',''), 'year': citations[doi].get('pubdate','')[0:4], 'label': "jstor", 'itemID': doi}
		return citations

	def _import_dfr(self, dfr_dir):
		citations = self._import_dfr_metadata(dfr_dir)

		wordcounts_dir = os.path.join(dfr_dir, "wordcounts")
		for doi in citations.keys():
			try:
				this_text = ''		
				for rowdict in self.parse_csv(os.path.join(wordcounts_dir, "wordcounts_" + doi.replace('/','_') + ".CSV")):
					word = rowdict["WORDCOUNTS"]
					if word in self.stopwords:
						continue
					if self.stemming:
						prestem = word
						if word not in self.stemmed:
							self.stemmed[prestem] = stem(self, prestem)
						word = self.stemmed[prestem]
					count = int(rowdict["WEIGHT"])

					this_text += (word + u' ') * count
				if len(this_text) < 20:
					continue
				yield doi, this_text
			except:
				logging.error(doi)
				logging.error(traceback.format_exc())

	def _output_text(self, text, f, filename):
		text = re.sub(r"[^\w ]+", u' ', text.lower(), flags=re.UNICODE)
		if self.stemming:
			newtext = u''
			for word in text.split():
				if word not in self.stemmed:
					self.stemmed[word] = stem(self, word)
				if len(self.stemmed[word]) < 4 or word in self.stopwords:
					continue
				newtext += self.stemmed[word] + u' '
			text = newtext
		f.write(u'\t'.join([filename, self.metadata[filename]["label"], text]) + u'\n')
		self.docs.append(filename)

	def _import_files(self):
		if self.stemming:
			self.stemmed = {}
		self.docs = []
		self.segmentation = getattr(self, "segmentation", False)

		with codecs.open(self.texts_file, 'w', encoding='utf-8') as f:
			for filename in self.files:
				with codecs.open(filename, 'r', encoding='utf-8') as input_file:
					text = input_file.read()
					if self.segmentation:
						segments = filter(lambda x: x.count(' ') > 5, text.split("\n\n"))
						for i, text_seg in enumerate(segments):
							seg_filename = filename + "#" + str(i)
							self.metadata[seg_filename] = self.metadata[filename]
							self._output_text(text_seg, f, seg_filename)
					else:
						self._output_text(text, f, filename)
			if self.dfr:
				for doi, text in self._import_dfr(self.dfr_dir):
					f.write(u'\t'.join([doi, self.metadata[doi]["label"], text]) + u'\n')
					self.docs.append(doi)
		with codecs.open(os.path.join(self.mallet_out_dir, "dmap"), 'w', encoding='utf-8') as dmap:
			dmap.writelines([x + u'\n' for x in self.docs])
		self.doc_count = len(self.docs)
{'fraction_non_alphanumeric': 0.15708140831607456, 'fraction_numerical': 0.003345547235940736, 'mean_word_length': 3.7416918429003023, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1209486', 'n_tokens_mistral': 2719, 'n_tokens_neox': 2229, 'n_words': 500}