<commit_before># coding: utf-8
# authors: dagrha, ngreeney
# Game of Thrones (Book 1) Sentiment Analysis.
'''This class creates a basic chart and a statistical summary table for a selected
chapter in the novel Game of Thrones (George R.R. Martin). Can upload to WordPress
site automatically.'''

# collections from the Standard Library
import collections
import logging

# libepub (https://github.com/jharjono/libepub/)
from libepub.book import Book
# pandas dataframe library (http://pandas.pydata.org/)
import pandas as pd
# Beautiful Soup (http://www.crummy.com/software/BeautifulSoup/)
from bs4 import BeautifulSoup
# TextBlob (http://textblob.readthedocs.org/en/dev/)
from textblob import TextBlob
from textblob.sentiments import NaiveBayesAnalyzer
# Bokeh plotting library (http://bokeh.pydata.org/en/latest/)
from bokeh.io import show, save
from bokeh.plotting import figure
from bokeh.resources import CDN
from bokeh.embed import file_html
# Matplotlib for jpg
import matplotlib.pyplot as plt
# needs --> pip install python-wordpress-xmlrpc
from blogpost import BlogPost
# NLTK
import nltk.tokenize.punkt as punkt
from nltk.tokenize import sent_tokenize, word_tokenize
import nltk.data as nldata
# Numpy - handy arrays
import numpy as np

class BookAnalysis:

    def __init__(self, epub):
        '''
        Pulls all the chapter info from an epub file.
        '''
        ebook = Book(epub)
        self.make_book(ebook)

    def make_book(self,ebook):
        '''
        Breaks epub book into chapters and creates a dictionary called book_dict\n\n
        
        Takes in ebook which is a libepub Book type
        '''
        self.book_dict = collections.OrderedDict()
        for chapter in ebook.chapters:
            soup = BeautifulSoup(chapter.content, 'xml')
            try:
                if soup.h1.attrs['class'] == 'chapter0':
                    chapter_dict = dict()
                    string = str()
                    for tag in soup('p', class_=['indent', 'nonindent']):
                        string += str(tag.get_text()) + ' '
                    chapter_data = list()
                    chapter_num = soup.h1.attrs['id']
                    chapter_data.append(chapter_num)
                    chapter_author = soup.h1.get_text()
                    chapter_data.append(chapter_author)
                    chapter_data.append(string)
                    page_num = soup.a.attrs['id']
                    chapter_dict[page_num] = chapter_data
                    self.book_dict.update(chapter_dict)
                else:
                    pass
            except AttributeError:
                print('No h1 in {} --- Skipping'.format(soup.div.attrs))
                pass
        self.renumber_prologue()<commit_msg>replace curly quotes with straight ones<commit_after># coding: utf-8
# authors: dagrha, ngreeney
# Game of Thrones (Book 1) Sentiment Analysis.
'''This class creates a basic chart and a statistical summary table for a selected
chapter in the novel Game of Thrones (George R.R. Martin). Can upload to WordPress
site automatically.'''

# collections from the Standard Library
import collections
import logging

# libepub (https://github.com/jharjono/libepub/)
from libepub.book import Book
# pandas dataframe library (http://pandas.pydata.org/)
import pandas as pd
# Beautiful Soup (http://www.crummy.com/software/BeautifulSoup/)
from bs4 import BeautifulSoup
# TextBlob (http://textblob.readthedocs.org/en/dev/)
from textblob import TextBlob
from textblob.sentiments import NaiveBayesAnalyzer
# Bokeh plotting library (http://bokeh.pydata.org/en/latest/)
from bokeh.io import show, save
from bokeh.plotting import figure
from bokeh.resources import CDN
from bokeh.embed import file_html
# Matplotlib for jpg
import matplotlib.pyplot as plt
# needs --> pip install python-wordpress-xmlrpc
from blogpost import BlogPost
# NLTK
import nltk.tokenize.punkt as punkt
from nltk.tokenize import sent_tokenize, word_tokenize
import nltk.data as nldata
# Numpy - handy arrays
import numpy as np

class BookAnalysis:

    def __init__(self, epub):
        '''
        Pulls all the chapter info from an epub file.
        '''
        ebook = Book(epub)
        self.make_book(ebook)

    def make_book(self,ebook):
        '''
        Breaks epub book into chapters and creates a dictionary called book_dict\n\n
        
        Takes in ebook which is a libepub Book type
        '''
        self.book_dict = collections.OrderedDict()
        for chapter in ebook.chapters:
            soup = BeautifulSoup(chapter.content, 'xml')
            try:
                if soup.h1.attrs['class'] == 'chapter0':
                    chapter_dict = dict()
                    string = str()
                    for tag in soup('p', class_=['indent', 'nonindent']):
                        string += str(tag.get_text()) + ' '
                    string = string.replace('“','"')
                    string = string.replace('”','"')
                    chapter_data = list()
                    chapter_num = soup.h1.attrs['id']
                    chapter_data.append(chapter_num)
                    chapter_author = soup.h1.get_text()
                    chapter_data.append(chapter_author)
                    chapter_data.append(string)
                    page_num = soup.a.attrs['id']
                    chapter_dict[page_num] = chapter_data
                    self.book_dict.update(chapter_dict)
                else:
                    pass
            except AttributeError:
                print('No h1 in {} --- Skipping'.format(soup.div.attrs))
                pass
        self.renumber_prologue()
{'fraction_non_alphanumeric': 0.09023354564755838, 'fraction_numerical': 0.0028308563340410475, 'mean_word_length': 2.2339816933638446, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4354272', 'n_tokens_mistral': 1647, 'n_tokens_neox': 1569, 'n_words': 464}