<commit_before>import json
import re

from bs4 import BeautifulSoup

from glob import glob
from pyspark import SparkConf, SparkContext
from pyspark.sql import SparkSession
from pyspark.ml.feature import CountVectorizer
from pyspark.ml.feature import StopWordsRemover
from pyspark.ml.feature import Word2Vec
from pyspark.ml.clustering import KMeans


def readguide(guide):
    with open(guide, 'r') as myfile:
        return myfile.read()

def extract_text(content_str):
    content = json.loads(content_str)

    soup = BeautifulSoup(content[2], "lxml")

    # kill all script and style elements
    for script in soup(["script", "style"]):
        script.extract()

    headerdiv = soup.find("div", {"id": "s-lg-guide-header-info"})
    maindiv = soup.find("div", {"id": "s-lg-guide-main"})

    if headerdiv and maindiv:
        txt = " ".join(headerdiv.strings).replace('\n', ' ') + " ".join(maindiv.strings).replace('\n', ' ')
        # remove non-alpha/numeric
        regex = re.compile('[^a-zA-Z]')
        txt = regex.sub(" ", txt)
        # remove whitespace
        txt = ' '.join(txt.split())
        if isinstance(txt, str):
            return [content[0], content[1], txt.split(" ")]
        else:
            return
    else:
        return

conf = SparkConf().setMaster("local").setAppName("My App")
sc = SparkContext(conf=conf)
spark = SparkSession(sc)

guides = glob("data/guides/*")

print("---------- ", type(guides))

# create an RDD consisting of a list of paths to .json files
guidesRDD = sc.parallelize(guides[0:10])
#guidesRDD = sc.parallelize(guides)

print("---------- guidesRDD.first():  ", guidesRDD.first())
print("---------- guidesRDD.count():  ", guidesRDD.count())

# create an RDD containing html documents
documentsRDD = guidesRDD.map(lambda guide: readguide(guide))

print("---------- documentsRDD.first():  ", documentsRDD.first())
print("---------- documentsRDD.count():  ", documentsRDD.count())

# create RDD containg plain text derived from html, guide id, and page_id
textsRDD = documentsRDD.map(lambda document: extract_text(document))

print("---------- textsRDD.count():  ", textsRDD.count())
print("---------- textsRDD.first():  ", textsRDD.first())
print("---------- textsRDD.first():  ", type(textsRDD.first()))

textsDFrame = textsRDD.toDF(["guide_id", "page_id", "words"])

textsDFrame.show(10)

texts = textsDFrame.collect()

print(" -------------- len(texts): ", len(texts))
print(" -------------- type(texts): ", type(texts))
#print(" -------------- print(texts): ", texts)

# save to parquet
textsDFrame.select("guide_id", "page_id", "words").write.save("data/libguides_bow.parquet", format="parquet")

"""


"""
"""
# Trains a k-means model.
kmeans = KMeans().setK(2).setSeed(1)
model = kmeans.fit(vectoredDFrame)

# Evaluate clustering by computing Within Set Sum of Squared Errors.
wssse = model.computeCost(res)
print("Within Set Sum of Squared Errors = " + str(wssse))

# Shows the result.
centers = model.clusterCenters()
print("Cluster Centers: ")
for center in centers:
    print(center)
"""
"""
texts = textsRDD.collect()

print(" -------------- len(texts): ", len(texts))
print(" -------------- type(texts): ", type(texts))
#print(" -------------- print(texts): ", texts)

joblib.dump(texts, 'data/libguides.pkl')

with open('data/libguides.json', 'w') as outfile:
    json.dump(texts, outfile)
"""
<commit_msg>inserted empty strings to guides that contained no text.<commit_after>import json
import re

from bs4 import BeautifulSoup

from glob import glob
from pyspark import SparkConf, SparkContext
from pyspark.sql import SparkSession
from pyspark.ml.feature import CountVectorizer
from pyspark.ml.feature import StopWordsRemover
from pyspark.ml.feature import Word2Vec
from pyspark.ml.clustering import KMeans

def readguide(guide):
    with open(guide, 'r') as myfile:
        return myfile.read()

def extract_text(content_str):
    content = json.loads(content_str)

    soup = BeautifulSoup(content[2], "lxml")

    # kill all script and style elements
    for script in soup(["script", "style"]):
        script.extract()

    headerdiv = soup.find("div", {"id": "s-lg-guide-header-info"})
    maindiv = soup.find("div", {"id": "s-lg-guide-main"})

    if headerdiv and maindiv:
        txt = " ".join(headerdiv.strings).replace('\n', ' ') + " ".join(maindiv.strings).replace('\n', ' ')
        # remove non-alpha/numeric
        regex = re.compile('[^a-zA-Z]')
        txt = regex.sub(" ", txt)
        # remove whitespace
        txt = ' '.join(txt.split())
        if isinstance(txt, str):
            return [content[0], content[1], txt.split(" ")]
        else:
            # if not a string, insert a space to make it one to avoid null content
            return [content[0], content[1], " "]
    else:
        # return empty string to avoid null content
        return [content[0], content[1], " "]

conf = SparkConf().setMaster("local").setAppName("My App")
sc = SparkContext(conf=conf)
spark = SparkSession(sc)

guides = glob("data/guides/*")

# create an RDD consisting of a list of paths to .json files
#guidesRDD = sc.parallelize(guides[0:10])
guidesRDD = sc.parallelize(guides)

# create an RDD containing html documents
documentsRDD = guidesRDD.map(lambda guide: readguide(guide))

# create RDD containg plain text derived from html, guide id, and page_id
textsRDD = documentsRDD.map(lambda document: extract_text(document))

textsDFrame = textsRDD.toDF(["guide_id", "page_id", "words"])

textsDFrame.show(10)

textsDFrame.printSchema()

# save to parquet
textsDFrame.select("guide_id", "page_id", "words").write.save("data/libguides_bow.parquet", format="parquet")

{'fraction_non_alphanumeric': 0.17338995046001415, 'fraction_numerical': 0.0046001415428167025, 'mean_word_length': 3.500796178343949, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3348475', 'n_tokens_mistral': 1875, 'n_tokens_neox': 1819, 'n_words': 458}