<commit_before>import csv
import gzip
import glob
import json
# import gensim
import os
import pickle
import multiprocessing
import logging
import sys
# import line_profiler

from datetime import datetime
from os.path import basename, join
from spacy.en import English
from bs4 import BeautifulSoup
from document_preprocessing import DocumentPreprocessor

log = logging.getLogger()
log.setLevel(logging.DEBUG)

ch = logging.StreamHandler(sys.stdout)
formatter = logging.Formatter(
	'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
log.addHandler(ch)


class AmazonReviews(object):
	"""
	Amazon Reviews processing. Converting from JSON structure into Data Frames
	and csv files.
	"""

	def __init__(self, f_path):
		"""
		Initialize params

		Parameters
		----------
		f_path : str
			File path to Amazon Reviews.
		"""

		self.f_path = f_path
		self.stop_words = [u'all', u'just', u'over', u'both', u'through',
		                   u'its', u'before', u'herself', u'should', u'to',
		                   u'only', u'under', u'ours', u'then', u'them', u'his',
		                   u'very', u'they', u'during', u'now', u'him', u'nor',
		                   u'these', u'she', u'each', u'further', u'where',
		                   u'few', u'because', u'some', u'our', u'ourselves',
		                   u'out', u'what', u'for', u'while', u'above',
		                   u'between', u'be', u'we', u'who', u'wa', u'here',
		                   u'hers', u'by', u'on', u'about', u'theirs',
		                   u'against', u'or', u'own', u'into', u'yourself',
		                   u'down', u'your', u'from', u'her', u'their',<commit_msg>logging correction<commit_after>import csv
import gzip
import glob
import json
# import gensim
import os
import pickle
import multiprocessing
import logging
import sys
# import line_profiler

from datetime import datetime
from os.path import basename, join
from spacy.en import English
from bs4 import BeautifulSoup
from document_preprocessing import DocumentPreprocessor

log = logging.getLogger(__name__)


class AmazonReviews(object):
	"""
	Amazon Reviews processing. Converting from JSON structure into Data Frames
	and csv files.
	"""

	def __init__(self, f_path):
		"""
		Initialize params

		Parameters
		----------
		f_path : str
			File path to Amazon Reviews.
		"""

		self.f_path = f_path
		self.stop_words = [u'all', u'just', u'over', u'both', u'through',
		                   u'its', u'before', u'herself', u'should', u'to',
		                   u'only', u'under', u'ours', u'then', u'them', u'his',
		                   u'very', u'they', u'during', u'now', u'him', u'nor',
		                   u'these', u'she', u'each', u'further', u'where',
		                   u'few', u'because', u'some', u'our', u'ourselves',
		                   u'out', u'what', u'for', u'while', u'above',
		                   u'between', u'be', u'we', u'who', u'wa', u'here',
		                   u'hers', u'by', u'on', u'about', u'theirs',
		                   u'against', u'or', u'own', u'into', u'yourself',
		                   u'down', u'your', u'from', u'her', u'their',
{'fraction_non_alphanumeric': 0.1550785004806152, 'fraction_numerical': 0.0006408202499198974, 'mean_word_length': 2.7344497607655502, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4548839', 'n_tokens_mistral': 1118, 'n_tokens_neox': 1006, 'n_words': 279}