<jupyter_start><jupyter_text>`SpaCy` and `pandas`<jupyter_code>import os
import spacy

# init spacy

nlp = spacy.load("en_core_web_sm")
type(nlp)
example = "This is a sentence written in English."
doc = nlp(example)
type(doc)
for token in doc:
    print(token.text, token.pos_, token.tag_, token.lemma_)<jupyter_output>This DET DT this
is AUX VBZ be
a DET DT a
sentence NOUN NN sentence
written VERB VBN write
in ADP IN in
English PROPN NNP English
. PUNCT . .<jupyter_text>__Reading data with `pandas`__<jupyter_code>import pandas as pd
infile = os.path.join('..', 'data', 'fake_or_real_news.csv')

data = pd.read_csv(infile)

# removing a column
del data['Unnamed: 0']

# 5 random records
data.sample(5)

# count the amount of records for every unique value in "label" col
data["label"].value_counts()<jupyter_output><empty_output><jupyter_text>__Filter on columns__<jupyter_code># all return new df where all entries are True / False based on whether "label" == "FAKE"
data['label'] == "FAKE"

# Filter based on condition in brackets
fake_news_df = data[data["label"] == "FAKE"]
real_news_df = data[data["label"] == "REAL"]

# Only fake news left
fake_news_df["label"].value_counts()<jupyter_output><empty_output><jupyter_text>__Counting features in data__<jupyter_code>adj_count = 0

# process texts in batch
for doc in nlp.pipe(fake_news_df["title"], batch_size=500):
    for token in doc:
        if token.tag_ == "JJ":
            adj_count += 1
            
adj_count<jupyter_output><empty_output><jupyter_text>Sentiment with `SpaCy`<jupyter_code>import os
import pandas as pd
import spacy
from spacytextblob import
nlp<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11285455642727821, 'fraction_numerical': 0.004828002414001207, 'mean_word_length': 5.352490421455939, 'pattern_counts': {'":': 1, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '277821', 'license': 'MIT', 'n_tokens_mistral': 621, 'n_tokens_neox': 599, 'n_words': 166, 'path': 'notebooks/spaCy and pandas.ipynb', 'repo_name': 'NormalReedus/cds-language-analytics', 'size': '1657'}