<jupyter_start><jupyter_text>Using API from https://github.com/4chan/4chan-API/blob/master/pages/User_images_and_static_content.md<jupyter_code>import os
import sys
import pandas as pd
import numpy as np
import urllib, json, os, datetime, time, requests
from unidecode import unidecode
import unicodecsv as csv
ROOT_DIR = "../../"
with open(ROOT_DIR + "config/config-scraping.json") as f:
    cfg = json.load(f)
sys.path.insert(0, os.path.abspath(ROOT_DIR + 'src/scraping'))
import scraping as scrape
save_dir = ROOT_DIR + cfg["tmp_dir"]
board = "wg"
data = scrape.scrape_new_data(board, save_dir)
df = scrape.extract_board_posts(board, save_dir, data)
scrape.download_images(df, board, save_dir)<jupyter_output><empty_output><jupyter_text>- Download thumbnails for image analysis- Maintain original image resolution for resolution visualization- Should probably hold off on downloading any images until after metadata filtering is done- Add nsfw tag- Tags for image recognition?-  Need to create and maintain a csv for every scrape so we don't have images downloaded with no corresponding csv data.- Add a datetime object column for when the image was originally scraped<jupyter_code>def remove_encoding(string):
    if pd.isnull(string):
        return None
    try:
        return string.encode('utf-8').decode("ascii", "ignore")
    except UnicodeEncodeError:
        return None
# str_cols = ["now", "name", "sub", "com"]
# for col in str_cols:
#     df[col] = df[col].str.replace('\\', '')
df["name"] = df["name"].apply(remove_encoding)
if not os.path.exists(save_dir + "csvs"):
    os.mkdir(save_dir + "csvs")
df.to_csv(save_dir + "csvs/" + board + "_metadata.csv", index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11778425655976676, 'fraction_numerical': 0.0023323615160349854, 'mean_word_length': 5.0, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '119360', 'license': 'MIT', 'n_tokens_mistral': 568, 'n_tokens_neox': 545, 'n_words': 163, 'path': 'notebooks/scraping/wg_4chan_scrape_coden.ipynb', 'repo_name': 'codencoding/4chan-image-analysis', 'size': '1715'}