<jupyter_start><jupyter_text>Google Trends Data using [pytrends](https://github.com/GeneralMills/pytrends)---<jupyter_code># === Connect to Google === #
from pytrends.request import TrendReq

# Mountain Time offset is -7:00 = 420 minutes
pytrends = TrendReq(hl="en-US", tz=420)
# === Other imports === #
import pandas as pd
import numpy as np
import os
import time
# === Build payload === #

# Base word to include in query
recycle = "recycle"

# List of materials to query
keywords = [
    'crayons',
]

# Concatenate each keyword with the base word
concat_kw = [" ".join([recycle, kw]) for kw in keywords]

print(concat_kw)
# === Build payload, Pt. II === #
pytrends.build_payload(
    concat_kw,
)
# === Return dataframe of interest over time === #
df1 = pytrends.interest_over_time()
df1.head()
df1.tail()
# === Export dataframe with date index only === #
df_dates_blank = df1.drop(columns=["recycle crayons", "isPartial"])
df_dates_blank.head()
df_dates_blank.to_csv("blank_date_index.csv")<jupyter_output><empty_output><jupyter_text>--- AutomateThe goal here is to...- Iterate through the full list of keywords- Send off 5 at a time to Google Trends- Concatenate the resulting data into a single dataframe<jupyter_code># === Import items into list === #

# Open text file and read, splitting on the newline character
with open("earth911_items_list.txt", "r") as f:
    recycle_list = f.read().splitlines()
len(recycle_list)
recycle_list[45:60]
# === Test out slicing list into 5 at a time === #

s_prev = 0

for s in range(0, 336, 5):
    # Slice from prev value of s to current
    recycle_list[s_prev:s]
    # Set current value of s to use in next iteration
    s_prev = s
def build_trend_df(base_query: str, kw_list: list, df) -> None:
    """
    Google Trends for 5 keywords at a time and add to dataframe.
    
    Parameters
    ----------
    base_query : string
        Query onto which each keyword is appended.
    kw_list : list
        List of keywords.
    df : pd.DataFrame
        Pre-existing dataframe onto which trends data will be added.
    
    Returns
    -------
    None
        Trends data is added to pre-existing dataframe, not returned.
    """

    s_prev = 45  # Used to keep track of previous loop value
    # Build lists of 5 items at a time
    for s in range(50, 336, 5):
        five_items_list = kw_list[s_prev:s]  # Slice from prev value of s to current

        # Concatenate each keyword with the base word - returns new list
        concat_queries = [" ".join([base_query, kw]) for kw in five_items_list]
#         print(concat_queries)

        # Build the payload and run the query for current 5 items
        pytrends.build_payload(concat_queries)
        df_return = pytrends.interest_over_time()

        # Add current results to OG dataframe
        if df_return.empty:
            print(f"No results for {s_prev}-{s}.")
        else:
            for item in concat_queries:
                if item in df_return.columns:
                    df[item] = df_return[item]

        # Space out the requests and give visual progress status
        print(f"Completed items {s_prev}-{s}.")
        time.sleep(2)
        s_prev = s  # Set current value of s to use in next iteration
# === Create OG Dataframe === #
df_og = df1.drop(columns=["recycle crayons", "isPartial"])
df_og
# === Run the function! === #
base_q1 = "recycle"
base_q2 = "how to recycle"

build_trend_df(base_q1, recycle_list, df_og)
# === Check out the results === #
df_og.describe()
# df_og.head()
# === Export to csv for posterity === #
df_og.to_csv("earth911_items_pytrends_all.csv")
# === New df to filter out 0-value columns === #
df2 = df_og.copy()
# === Get list of 0-value columns === #

zero_value_cols = []

for col in df2.columns:
    if df2[col].mean() == 0.0:
        zero_value_cols.append(col)
# === Drop 0-value columns === #
df3 = df2.drop(columns=zero_value_cols)
df3.shape
# === Result is 78 columns - Check em out === #
pd.options.display.max_rows = 100
df3.describe().T
# === Export pruned df to csv (for posterity!) === #
df3.to_csv("earth911_items_pytrends.csv")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11867280213126664, 'fraction_numerical': 0.016468878663114556, 'mean_word_length': 3.2142857142857144, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '623643', 'license': 'MIT', 'n_tokens_mistral': 1421, 'n_tokens_neox': 1341, 'n_words': 435, 'path': 'notebooks/scrapers/google_trends/01-google-trends-pytrends.ipynb', 'repo_name': 'tobias-fyi/neurecycle', 'size': '4129'}