<commit_before>import nltk
import sys

from nltk.corpus import wordnet as wn
import kindred
from nltk.parse.util import taggedsents_to_conll
from nltk.parse.stanford import StanfordDependencyParser

from kindred import Dependencies

def is_noun(tag):
    return tag in ['NN', 'NNS', 'NNP', 'NNPS']


def is_verb(tag):
    return tag in ['VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ']


def is_adverb(tag):
    return tag in ['RB', 'RBR', 'RBS']


def is_adjective(tag):
    return tag in ['JJ', 'JJR', 'JJS']


def penn_to_wn(tag):
	if is_adjective(tag):
		return wn.ADJ
	elif is_noun(tag):
		return wn.NOUN
	elif is_adverb(tag):
		return wn.ADV
	elif is_verb(tag):
		return wn.VERB
	return None

wordnet_lemmatizer = None
depparser = None
def parseSentence(sentence):
	global wordnet_lemmatizer
	global depparser
	
	if wordnet_lemmatizer is None:
		wordnet_lemmatizer = nltk.stem.WordNetLemmatizer()
		
	if depparser is None:
		depparser = Dependencies.getMaltParser()

	tokensAndPOS = nltk.pos_tag(nltk.word_tokenize(sentence))
	wordnetTypes = [ penn_to_wn(pos) for token,pos in tokensAndPOS ]
	lemmas = [ token if wnpos is None else wordnet_lemmatizer.lemmatize(token,pos=wnpos) for (token,pos),wnpos in zip(tokensAndPOS,wordnetTypes) ]

	substitutions = { '``': '"', "''": '"' }
	
	currentPosition = 0
	locs = []
	for t,_ in tokensAndPOS:
	
		# Deal with special characters output by parser
		if t in substitutions:
			t = substitutions[t]
			
		start = sentence.find(t,currentPosition)
		assert start != -1, "Error finding token: %s in sentence: %s" % (t,sentence)
		end = start + len(t)
		currentPosition = end
		locs.append((start,end))
		
	tokenInfo = [ kindred.Token(token,pos,lemma,start,end) for (token,pos),lemma,(start,end) in zip(tokensAndPOS,lemmas,locs) ]
	
	tokens = [ t for t,_ in tokensAndPOS ]
	
	depgraph = []
	if True:
		depparses = list(depparser.parse(tokens))
		assert len(depparses) == 1, "Expected parser to return a single parse"
		depparse = depparses[0]
		
		# Get the dependency graph from the parse
		depgraph = [ (i,details['head'],details['rel']) for i,details in depparse.nodes.items() ]
			
		# And we'll filter out any non-existent relations
		depgraph = [ (i,j,rel) for i,j,rel in depgraph if not rel is None and i>0 and j>0 ]
		
		# Remember that parser gives word indices starting from 1, so subtract to keep with starting from 0
		depgraph = [ (i-1,j-1,rel) for i,j,rel in depgraph ]
	
	print tokenInfo
	
	return tokenInfo,depgraph
	
wordnet_lemmatizer = None
depparser = None
def parseSentences(text):
	assert isinstance(text,basestring)

	global wordnet_lemmatizer
	global depparser
	
	if wordnet_lemmatizer is None:
		wordnet_lemmatizer = nltk.stem.WordNetLemmatizer()
		
	if depparser is None:
		#depparser = nltk.parse.malt.MaltParser('maltparser-1.9.0','engmalt.linear-1.7.mco')
		Dependencies.initializeStanfordParser()
		depparser = StanfordDependencyParser(model_path="edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz")

	text = text.encode('ascii','ignore')
		
	sentences = nltk.tokenize.sent_tokenize(text)
	tokenSets = [ nltk.word_tokenize(s) for s in sentences ]
		
	depparser_result = depparser.parse_sents(tokenSets)
	
	depparses = []
	for d in depparser_result:
		d = list(d)
		assert len(d) == 1, "Expected parser to return a single parse"
		depparses.append(d[0])
		
	output = []
	
	#print sentences
		
	currentSentencePos = 0<commit_msg>Download NLTK packages in Kindred<commit_after>import nltk
import sys

from nltk.corpus import wordnet as wn
import kindred
from nltk.parse.util import taggedsents_to_conll
from nltk.parse.stanford import StanfordDependencyParser

from kindred import Dependencies

nltkPackagesOkay = False
def checkNLTKPackages():
	global nltkPackagesOkay
	if not nltkPackagesOkay:
		requiredPackages = ['wordnet','punkt']
		for package in requiredPackages:
			nltk.download(package,quiet=True)
		nltkPackagesOkay = True

def is_noun(tag):
    return tag in ['NN', 'NNS', 'NNP', 'NNPS']


def is_verb(tag):
    return tag in ['VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ']


def is_adverb(tag):
    return tag in ['RB', 'RBR', 'RBS']


def is_adjective(tag):
    return tag in ['JJ', 'JJR', 'JJS']


def penn_to_wn(tag):
	if is_adjective(tag):
		return wn.ADJ
	elif is_noun(tag):
		return wn.NOUN
	elif is_adverb(tag):
		return wn.ADV
	elif is_verb(tag):
		return wn.VERB
	return None

wordnet_lemmatizer = None
depparser = None
def parseSentence(sentence):
	global wordnet_lemmatizer
	global depparser
	
	checkNLTKPackages()
	
	if wordnet_lemmatizer is None:
		wordnet_lemmatizer = nltk.stem.WordNetLemmatizer()
		
	if depparser is None:
		depparser = Dependencies.getMaltParser()

	tokensAndPOS = nltk.pos_tag(nltk.word_tokenize(sentence))
	wordnetTypes = [ penn_to_wn(pos) for token,pos in tokensAndPOS ]
	lemmas = [ token if wnpos is None else wordnet_lemmatizer.lemmatize(token,pos=wnpos) for (token,pos),wnpos in zip(tokensAndPOS,wordnetTypes) ]

	substitutions = { '``': '"', "''": '"' }
	
	currentPosition = 0
	locs = []
	for t,_ in tokensAndPOS:
	
		# Deal with special characters output by parser
		if t in substitutions:
			t = substitutions[t]
			
		start = sentence.find(t,currentPosition)
		assert start != -1, "Error finding token: %s in sentence: %s" % (t,sentence)
		end = start + len(t)
		currentPosition = end
		locs.append((start,end))
		
	tokenInfo = [ kindred.Token(token,pos,lemma,start,end) for (token,pos),lemma,(start,end) in zip(tokensAndPOS,lemmas,locs) ]
	
	tokens = [ t for t,_ in tokensAndPOS ]
	
	depgraph = []
	if True:
		depparses = list(depparser.parse(tokens))
		assert len(depparses) == 1, "Expected parser to return a single parse"
		depparse = depparses[0]
		
		# Get the dependency graph from the parse
		depgraph = [ (i,details['head'],details['rel']) for i,details in depparse.nodes.items() ]
			
		# And we'll filter out any non-existent relations
		depgraph = [ (i,j,rel) for i,j,rel in depgraph if not rel is None and i>0 and j>0 ]
		
		# Remember that parser gives word indices starting from 1, so subtract to keep with starting from 0
		depgraph = [ (i-1,j-1,rel) for i,j,rel in depgraph ]
	
	print tokenInfo
	
	return tokenInfo,depgraph
	
wordnet_lemmatizer = None
depparser = None
def parseSentences(text):
	assert isinstance(text,basestring)

	global wordnet_lemmatizer
	global depparser
	
	checkNLTKPackages()
	
	if wordnet_lemmatizer is None:
		wordnet_lemmatizer = nltk.stem.WordNetLemmatizer()
		
	if depparser is None:
		#depparser = nltk.parse.malt.MaltParser('maltparser-1.9.0','engmalt.linear-1.7.mco')
		Dependencies.initializeStanfordParser()
		depparser = StanfordDependencyParser(model_path="edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz")

	text = text.encode('ascii','ignore')
		
	sentences = nltk.tokenize.sent_tokenize(text)
	tokenSets = [ nltk.word_tokenize(s) for s in sentences ]
		
	depparser_result = depparser.parse_sents(tokenSets)
	
	depparses = []
	for d in depparser_result:
		d = list(d)
		assert len(d) == 1, "Expected parser to return a single parse"
		depparses.append(d[0])
		
	output = []
	
	#print sentences
		
	currentSentencePos = 0
{'fraction_non_alphanumeric': 0.11300549527969564, 'fraction_numerical': 0.0050725658729040436, 'mean_word_length': 4.506594259115594, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5590078', 'n_tokens_mistral': 2829, 'n_tokens_neox': 2554, 'n_words': 653}