<commit_before>#!/usr/bin/env python
__author__ = '<NAME>'

import string, pickle
from nltk import word_tokenize
from sklearn.feature_extraction.text import TfidfVectorizer


class RScore(object):
	"""
	This class calculates readability score - based on English word frequency.
	"""
	def __init__(self):
		self.score = 0
		self.freq = pickle.load(open("word_freq.db","rb"))

	def clear_text(self, file_name):
		"""
		Clear all punctuations
		"""
		text = open(file_name, 'r').read()
		text = text.lower()
		no_punctuation = text.translate(None, string.punctuation)
		return no_punctuation

	def tokenize(self, text):
		"""
		Tokenize text into words.
		"""
		tokens = word_tokenize(text)
		return tokens

	def tf_idf(self, file_name):
		"""
		Calculate tf-idf score.
		:return: dictionary where keys are words and values are tf scores.
		"""
		tfidf = TfidfVectorizer(tokenizer=self.tokenize, stop_words='english', norm="l1")
		tfs = tfidf.fit_transform([self.clear_text(file_name)]).toarray()
		word_tfidf = dict(zip(tfidf.get_feature_names(), tfs[0]))
		return word_tfidf

	def rscore(self, file_name):
		word_tfidf = self.tf_idf(file_name)
		for word in word_tfidf:
			try:
				self.score += self.freq[word]*word_tfidf[word]
			except:
				continue
		return self.score*10000/len(word_tfidf)

<commit_msg>Updated RScore.py<commit_after>#!/usr/bin/env python
__author__ = '<NAME>'

import string, pickle
from nltk import word_tokenize
from sklearn.feature_extraction.text import TfidfVectorizer


class RScore(object):
	"""
	This class calculates readability score - based on English word frequency.
	"""
	def __init__(self):
		self.score = 0
		self.freq = pickle.load(open("word_freq.db","rb"))

	def clear_text(self, file_name):
		"""
		Clear all punctuations
		"""
		text = open(file_name, 'r').read()
		text = text.lower()
		no_punctuation = text.translate(None, string.punctuation)
		return no_punctuation

	def tokenize(self, text):
		"""
		Tokenize text into words.
		"""
		tokens = word_tokenize(text)
		return tokens

	def tf_idf(self, file_name):
		"""
		Calculate tf-idf score.
		:return: dictionary where keys are words and values are tf scores.
		"""
		tfidf = TfidfVectorizer(tokenizer=self.tokenize, stop_words='english', norm="l1")
		tfs = tfidf.fit_transform([self.clear_text(file_name)]).toarray()
		word_tfidf = dict(zip(tfidf.get_feature_names(), tfs[0]))
		return word_tfidf

	def rscore(self, file_name):
		self.score = 0
		word_tfidf = self.tf_idf(file_name)
		for word in word_tfidf:
			try:
				self.score += self.freq[word]*word_tfidf[word]
			except:
				continue
		return self.score*10000/len(word_tfidf)


{'fraction_non_alphanumeric': 0.1256166982922201, 'fraction_numerical': 0.0064516129032258064, 'mean_word_length': 4.526205450733753, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6452097', 'n_tokens_mistral': 1052, 'n_tokens_neox': 952, 'n_words': 211}