<jupyter_start><jupyter_text>NLP in Finance: SEC FilingsIn this notebook, you will analyze 10-K, 8-K filings using NLP skills that we practiced today.Reference: https://github.com/Seungju182/AI-for-Trading/tree/master/Project_5_NLP_on_Financial_Statements We are going to use sec-edgar-downloader package. For details, please check the link below.https://pypi.org/project/sec-edgar-downloader/<jupyter_code>!pip install -U sec-edgar-downloader
from sec_edgar_downloader import Downloader
doc_type = '8-K'
ticker = 'AAPL'
after = '2017-01-01'
before = '2017-03-25'
dl = Downloader("./data")

# Get all 8-K filings for Apple (ticker: AAPL)
#dl.get("8-K", "AAPL")

# Get all 8-K filings for Apple, including filing amends (8-K/A)
#dl.get("8-K", "AAPL", include_amends=True)

# Get all 8-K filings for Apple after January 1, 2017 and before March 25, 2017
# Note: after and before strings must be in the form "YYYY-MM-DD"
dl.get(doc_type, ticker, after=after, before=before)<jupyter_output><empty_output><jupyter_text>Import libraries> Don't forget to download stopwords and wordnet<jupyter_code>import os

import time
import numpy as np
import pandas as pd
import nltk
import re
from bs4 import BeautifulSoup
nltk.download('stopwords')
nltk.download('wordnet')
os.listdir(os.getcwd())
txt_files = []

for dir, subdir, files in os.walk(os.getcwd()):
  for x in files:
    if x.endswith('.txt'):
      print(dir)
      print(x)
      txt_files.append(os.path.join(dir, x))
txt_files
text_data = []

for file in txt_files:
  with open(file, 'r', encoding='utf-8') as f:
    text = f.read()
    text_data.append(text)
    f.close()
text_data[0]
len(text_data[0])<jupyter_output><empty_output><jupyter_text>Preprocess<jupyter_code>def get_documents(text):
    """
    Extract the documents from the text

    Parameters
    ----------
    text : str
        The text with the document strings inside

    Returns
    -------
    extracted_docs : list of str
        The document strings found in `text`
    """
    
    regex_start = re.compile(r'<DOCUMENT>')
    regex_end = re.compile(r'</DOCUMENT>')
    
    start_idx = [x.end() for x in re.finditer(regex_start, text)]
    end_idx = [x.start() for x in re.finditer(regex_end, text)]
    
    result = []
    for start, end in zip(start_idx, end_idx):
        result.append(text[start:end])
    
    return result
get_documents(text_data[0])
len(get_documents(text_data[1]))
def get_document_type(doc):
    """
    Return the document type lowercased

    Parameters
    ----------
    doc : str
        The document string

    Returns
    -------
    doc_type : str
        The document type lowercased
    """
    
    match = re.search(r'<TYPE>[^\n]+', doc)
    
    return match.group()[len('<TYPE>'):].lower()
len(text_data)
# Something's wrong!
len(text_data[1])
data = []

# We should only select documents with the type we want
for docs in text_data:
  for doc in get_documents(docs):
    if get_document_type(doc) == doc_type.lower():
      data.append(doc)

data<jupyter_output><empty_output><jupyter_text>Clean upNow let's clean up our text<jupyter_code>def remove_html_tags(text):
    text = BeautifulSoup(text, 'html.parser').get_text()
    
    return text


def clean_text(text):
    text = text.lower()
    text = remove_html_tags(text)
    
    return text
cleaned_data = [clean_text(text) for text in data]
cleaned_data[0]<jupyter_output><empty_output><jupyter_text>Lemmatize and remove stopwords<jupyter_code>from nltk.stem import WordNetLemmatizer
from nltk.corpus import wordnet


def lemmatize_words(words):
    """
    Lemmatize words 

    Parameters
    ----------
    words : list of str
        List of words

    Returns
    -------
    lemmatized_words : list of str
        List of lemmatized words
    """
    
    return [WordNetLemmatizer().lemmatize(w, pos='v') for w in words]
word_pattern = re.compile('\w+')

cleaned_data = [lemmatize_words(word_pattern.findall(text)) for text in cleaned_data]
len(cleaned_data)
len(cleaned_data[0])
from nltk.corpus import stopwords
lemma_stopwords = lemmatize_words(stopwords.words('english'))
final_words = []

for data in cleaned_data:
  final = [word for word in data if word not in lemma_stopwords]
  final_words.append(final)
len(final_words)
len(final_words[0])<jupyter_output><empty_output><jupyter_text>We just lemmatized and removed stopwords. From now on, you can play with this text data. Code below is a brief example of sentimental analysis.<jupyter_code>pos = pd.read_csv('https://raw.githubusercontent.com/RaghuveerRao/Sentiment-Analysis/master/LM_pos_words.txt', squeeze=True).str.lower()
neg = pd.read_csv('https://raw.githubusercontent.com/RaghuveerRao/Sentiment-Analysis/master/LM_neg_words.txt', squeeze=True).str.lower()
neg
pos = lemmatize_words(list(pos.values))
neg = lemmatize_words(list(neg.values))
for doc in final_words:
  
  num_pos = len([word for word in doc if word in pos])
  num_neg = len([word for word in doc if word in neg])

  if num_pos > num_neg:
    print("Positive")
  elif num_pos < num_neg:
    print("Negative")
  else:
    print("Neutral")<jupyter_output>Negative
Positive
Positive
Negative
{'fraction_non_alphanumeric': 0.1213884041109172, 'fraction_numerical': 0.009889470622454915, 'mean_word_length': 3.893738140417457, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 30, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '94046', 'license': 'MIT', 'n_tokens_mistral': 1849, 'n_tokens_neox': 1773, 'n_words': 452, 'path': 'Day 13/13_3_nlp_in_finance.ipynb', 'repo_name': 'SLCFLAB/Data-Science-Python', 'size': '5157'}