<commit_before>from re     import search, IGNORECASE
from string import strip, punctuation
import nltk

"Features for the classifier."

def make_searcher(pattern, field='article_snippet', flags=IGNORECASE):
	"""Returns a feature template function that defines the feature
	   'has_pattern_' + pattern, which is present whenever the datum
	   contains that pattern.  Defaults to case-insensitive.
	"""
	def result(datum):
		if search(pattern, datum.__dict__[field], flags):
			return [field + '_has_pattern_' + pattern]
		else:
			return []
	return result

stemmer = nltk.LancasterStemmer().stem
# stemmer = nltk.WordNetLemmatizer().lemmatize

def stems_in(field):
	"""Makes a feature for each stem in the given field."""
	def result(datum):
		stemset = set()
		def mkfeature(string): return field + '_has_stem_' + string
		for word in datum.__dict__[field].split():
			stemset.add(stemmer(strip(word.lower(), punctuation)))
		return map(mkfeature, stemset)
	return result

# Since NER is slow, only do it for the first cutoff_for_ner characters of the
# article_snippet:
cutoff_for_ner = 1000
# Again, since NER is slow, only do it for the first
# per_sentence_cutoff_for_ner characters of each sentence:
per_sentence_cutoff_for_ner = 10

def named_entities(datum):
	result = []
	for sent in nltk.sent_tokenize(datum.article_snippet[:cutoff_for_ner]):
		for chunk in nltk.ne_chunk(nltk.pos_tag(nltk.word_tokenize(sent[:per_sentence_cutoff_for_ner]))):
			if hasattr(chunk, 'node'):
				result.append('article_snippet_has_entity_' + chunk.node + '_' +  '_'.join(stemmer(c[0].lower()) for c in chunk.leaves()))
	return result

matcher_features = map(make_searcher, [
	# These seem likely to show up in articles about disease:
	'estimated',
	'exposure',
	'percent',
	'population',
	'virus',
	'disease',
	'outbreak',
	'report',
	'prevention',
	'control',
	'disease',
	'confirmed',
	'diagnosed',
	'diagnosis',
	'associated',
	'emergency',
	'season', # TODO: Maybe exclude 'hurricane'.
	'dea(d|th)',
	'authorities',
	# 'sick', 		# Doesn't distinguish.
	# '\sill\s', 		# Doesn't distinguish.
	# 'issued.*advisory', 	# Too sparse.

	# Things I'm trying out because of their counts:
	'president',
	'&lt;img', # Has a picture.  Good for 'unrelated'.

	# These seem like they would show up in non-outbreak or non-disease articles:
	'rival',
	'sport',
	'champ',
	'protest',
	'hurricane',
	'economy',
	'wedding', # Good but sparse.
	'linux',
	'research',
	'prosecut'] # as in 'prosecution' and 'prosecuted'.
	)

other_features = [
	# TODO: URL features.
	named_entities,
	make_searcher('(epi|pan)demic', 'title'),
	make_searcher('H\dN\d', field='article_snippet', flags=0), # names of flu subtypes, like H1N1
	stems_in('title'),
	# TODO: Maybe strip away non-word symbols like in "(ap)" and "prosecutors:".
	# TODO: Maybe get the word STEMS in the title, not the words.
	# TODO: Replace all numbers with a single digit or something.

	]

feature_templates = matcher_features + other_features
<commit_msg>Added subdomain feature.<commit_after>from re     import search, IGNORECASE
from string import strip, punctuation
import re
import nltk

"Features for the classifier."

def make_searcher(pattern, field='article_snippet', flags=IGNORECASE):
	"""Returns a feature template function that defines the feature
	   'has_pattern_' + pattern, which is present whenever the datum
	   contains that pattern.  Defaults to case-insensitive.
	"""
	def result(datum):
		if search(pattern, datum.__dict__[field], flags):
			return [field + '_has_pattern_' + pattern]
		else:
			return []
	return result

stemmer = nltk.LancasterStemmer().stem
# stemmer = nltk.WordNetLemmatizer().lemmatize

def stems_in(field):
	"""Makes a feature for each stem in the given field."""
	def result(datum):
		stemset = set()
		def mkfeature(string): return field + '_has_stem_' + string
		for word in datum.__dict__[field].split():
			stemset.add(stemmer(strip(word.lower(), punctuation)))
		return map(mkfeature, stemset)
	return result

def subdomain(datum):
	"""Makes a feature for the subdomain.  For example, an article from
	   www2.portal.nature.com will have the subbdomain_portal.nature.com feature.
	"""
	m = re.match('http://(www.?\.)?([^/]*)/(([^/]*/)*)', datum.article_url)
	result = []
	if hasattr(m, 'groups') and len(m.groups()) > 1:
		result.append('subdomain_' + m.groups()[1])
	return result

# Since NER is slow, only do it for the first cutoff_for_ner characters of the
# article_snippet:
cutoff_for_ner = 1000
# Again, since NER is slow, only do it for the first
# per_sentence_cutoff_for_ner characters of each sentence:
per_sentence_cutoff_for_ner = 10

def named_entities(datum):
	result = []
	for sent in nltk.sent_tokenize(datum.article_snippet[:cutoff_for_ner]):
		for chunk in nltk.ne_chunk(nltk.pos_tag(nltk.word_tokenize(sent[:per_sentence_cutoff_for_ner]))):
			if hasattr(chunk, 'node'):
				result.append('article_snippet_has_entity_' + chunk.node + '_' +  '_'.join(stemmer(c[0].lower()) for c in chunk.leaves()))
	return result

matcher_features = map(make_searcher, [
	# These seem likely to show up in articles about disease:
	'estimated',
	'exposure',
	'percent',
	'population',
	'virus',
	'disease',
	'outbreak',
	'report',
	'prevention',
	'control',
	'disease',
	'confirmed',
	'diagnosed',
	'diagnosis',
	'associated',
	'emergency',
	'season', # TODO: Maybe exclude 'hurricane'.
	'dea(d|th)',
	'authorities',
	# 'sick', 		# Doesn't distinguish.
	# '\sill\s', 		# Doesn't distinguish.
	# 'issued.*advisory', 	# Too sparse.

	# Things I'm trying out because of their counts:
	'president',
	'&lt;img', # Has a picture.  Good for 'unrelated'.

	# These seem like they would show up in non-outbreak or non-disease articles:
	'rival',
	'sport',
	'champ',
	'protest',
	'hurricane',
	'economy',
	'wedding', # Good but sparse.
	'linux',
	'research',
	'prosecut'] # as in 'prosecution' and 'prosecuted'.
	)

other_features = [
	named_entities,
	make_searcher('(epi|pan)demic', 'title'),
	make_searcher('H\dN\d', field='article_snippet', flags=0), # names of flu subtypes, like H1N1
	stems_in('title'),
	# TODO: Maybe strip away non-word symbols like in "(ap)" and "prosecutors:".
	# TODO: Maybe get the word STEMS in the title, not the words.
	# TODO: Replace all numbers with a single digit or something.
	subdomain
	# TODO: Other URL features.
	]

feature_templates = matcher_features + other_features

{'fraction_non_alphanumeric': 0.13376095118898623, 'fraction_numerical': 0.0035982478097622028, 'mean_word_length': 4.833029197080292, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '285143', 'n_tokens_mistral': 2369, 'n_tokens_neox': 2218, 'n_words': 629}