<jupyter_start><jupyter_text>Download link informationhttps://developer.nrel.gov/docs/solar/nsrdb/psm3-download/<jupyter_code>for year in years:
    for zip_code in tqdm(zip_codes.keys()):
        req_str = (
            f"https://developer.nrel.gov/api/solar/nsrdb_psm3_download.csv?"
            + f'wkt=POINT({zip_codes[zip_code]["lon"]}%20{zip_codes[zip_code]["lat"]})'
            + f"&names={year}"
            + f'&leap_day={cfg_vars["leap_year"]}'
            + f'&interval={cfg_vars["interval"]}'
            + f'&utc={cfg_vars["utc"]}'
            + f'&full_name={cfg_vars["name"]}'
            + f'&email={cfg_vars["email"]}'
            + f'&affiliation={cfg_vars["affiliation"]}'
            + f'&mailing_list={cfg_vars["mailing_list"]}'
            + f'&reason={cfg_vars["use"]}'
            + f'&api_key={cfg_vars["key"]}'
            + f'&attributes={cfg_vars["attrs"]}'
        )

        logger.info(f"{req_str}\n")

        # sleep so we don't overrun the rate NREL limit
        sleep(2)
        try:
            df_raw = pd.read_csv(req_str, nrows=nrows)
            logger.info("reg_str successful.")
        except:
            logger.error(f"Error requesting\n{req_str}\n")

        # query and extract the first 2 lines to get metadata:
        df_meta = df_raw.iloc[0].copy()
        # display(df_meta)

        row1_cols = df_raw.iloc[1]
        new_header = [item.replace(" ", "_") if isinstance(item, str) else item for item in row1_cols]

        df_data = df_raw.iloc[1:].copy()
        df_data.columns = new_header
        df_data.drop(1, axis=0, inplace=True)
        df_data = df_data.loc[:, df_data.columns.notnull()].copy()
        df_data.reset_index(drop=True, inplace=True)

        df_data.insert(0, "date_time", "")

        df_data["date_time"] = pd.to_datetime(
            df_data["Year"].astype(str)
            + "-"
            + df_data["Month"].astype(str)
            + "-"
            + df_data["Day"].astype(str)
            + " "
            + df_data["Hour"].astype(str)
            + ":"
            + df_data["Minute"].astype(str)
            + ":"
            + "00"
        )

        # df_data.drop(["Year", "Month", "Day", "Hour", "Minute"], axis=1, inplace=True)
        df_data.drop(["Minute"], axis=1, inplace=True)
        df_data.insert(1, "zipcode", zip_code)
        df_data.insert(2, "location_id", df_meta["Location ID"])

        data_names = [
            (df_data, "nsrdb_" + str(zip_code) + "_" + str(year) + ".csv"),
            (df_meta, "nsrdb_meta_" + str(zip_code) + "_" + str(year) + ".csv"),
        ]

        try:
            for item in data_names:
                item[0].to_csv("~/_UMSI/697/data/nsrdb_raw/" + item[1], index=True)
                logger.info(f"{item[1]} successfully written.\n")
        except:
            logger.error("Error writing .csv raw file(s)")

        try:
            cursor.execute(queries.select_zip_year, {"zipcode": zip_code, "year": year})
            count = cursor.fetchone()
            # print(count)

            if (count[0] == "8760") or (count[0] == "8784"):
                logger.warning(f"data for {year}, {zip_code} already present\n")
            else:
                df_data.to_sql("nsrdb", conn, if_exists="append", index=False, method="multi")
                logger.info(f"data for {year}, {zip_code} written to {db_file}:{db_table1}\n")
        except:
            logger.error("Error writing to nsrdb\n")

        llltze_params = {
            "loc_id": df_meta["Location ID"],
            "lat": df_meta["Latitude"],
            "lon": df_meta["Longitude"],
            "elev": df_meta["Elevation"],
            "tz": df_meta["Time Zone"],
            "zipcode": zip_code,
        }
        logger.info(f"{llltze_params}\n")

        cursor.execute(queries.update_gzc_llltze, llltze_params)
        conn.commit()
        
        cursor2.execute(queries.update_gzc_llltze, llltze_params)
        conn2.commit()        

        cursor.execute(queries.select_zipcode, {"zipcode": zip_code})
        logger.info(f"gzc: {cursor.fetchall()}\n")
conn.close()
conn2.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16553890768487192, 'fraction_numerical': 0.009666505558240696, 'mean_word_length': 2.164373088685015, 'pattern_counts': {'":': 11, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '391954', 'license': 'MIT', 'n_tokens_mistral': 1451, 'n_tokens_neox': 1415, 'n_words': 205, 'path': 'archives/gm/notebooks/nsrdb_download.ipynb', 'repo_name': 'gamyers/solar-697', 'size': '4138'}