<jupyter_start><jupyter_text>Image Scraper for Online Image Boards  Set Up<jupyter_code>import sys
import os
import requests
from re import sub, finditer
from bs4 import BeautifulSoup
from io import BytesIO
from PIL import Image<jupyter_output><empty_output><jupyter_text>Gets images from URL and saves to destination<jupyter_code>def download_file(url, dest_dir, filename=None):
    if(filename is None):
        filename = urlsplit(url).path.split("/")[-1]
        
    request = requests.get(url)
    image = Image.open(BytesIO(request.content))
    
    if image.mode in ('RGBA', 'LA'):
        #background = Image.new(image.mode[:-1], image.size, fill_color)
        #background.paste(image, image.split()[-1])
        background = img_alpha_to_colour(image)
        image = background
    
    image.save(os.path.join(dest_dir, filename))<jupyter_output><empty_output><jupyter_text>Handles images that have no alpha layer<jupyter_code>def img_alpha_to_colour(image, color=(255, 255, 255)):
    image.load()  # needed for split()
    background = Image.new('RGB', image.size, color)
    background.paste(image, mask=image.split()[3])  # 3 is the alpha channel
    return background<jupyter_output><empty_output><jupyter_text>Takes HTML code as input, creates list, appends img elements to list<jupyter_code>def get_file_urls(soup):
    file_urls = []
    
    # fix href errors
    for anchor in soup.find_all(attrs={'class':'fileThumb'}):
        file_urls.append(sub("//", "https://", anchor.get('href')))
    return file_urls<jupyter_output><empty_output><jupyter_text>Iterate through posts, find img filenames<jupyter_code>def get_filenames(soup):
    filenames = []

    for anchor in soup.find_all(attrs={'class': 'fileText'}):
        filenames.append(anchor.get_text().split(" ")[1])
        print(filenames.append(anchor.get_text().split(" ")[1]))
    return filenames<jupyter_output><empty_output><jupyter_text>Primary Function<jupyter_code>def main(url):
    print("┌─────────────────────────────────────────────────────────────────────┐")
    print("│           Image Scraping Tool built by <NAME>           │")
    print("└─────────────────────────────────────────────────────────────────────┘")
    print("Downloading images from " + url + " ...")
    
    dest_dir = os.getcwd()
    
    html = requests.get(url).text
    soup = BeautifulSoup(html, "html.parser")

    file_urls = get_file_urls(soup)
    filenames = get_filenames(soup)

    for i in range(len(file_urls)):
        print("Downloading file: " + filenames[i])
        download_file(file_urls[i], dest_dir, filenames[i])

    print("┌─────────────────────────────────────────────────────────────────────┐")
    print("│             Scrape completed. Your files are now ready.             │")
    print("└─────────────────────────────────────────────────────────────────────┘")<jupyter_output><empty_output><jupyter_text>Running the Script<jupyter_code>main("https://boards.4channel.org/g/thread/76818171")<jupyter_output>┌─────────────────────────────────────────────────────────────────────┐
│           Image Scraping Tool built by <NAME>           │
└─────────────────────────────────────────────────────────────────────┘
Downloading images from https://boards.4channel.org/g/thread/76818171 ...
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
Downloading file: 241.png
Downloading file: 241.png
Downloading file: 1555427192637.png
Downloading file: 1555427192637.png
Downloading file: yNb7BPF.png
Downloading file: yNb7BPF.png
Downloading file: 21686342_275610329600215_(...).jpg
Downloading file: 21686342_275610329600215_(...).jpg
Downloading file: kanna
{'fraction_non_alphanumeric': 0.21144024514811033, 'fraction_numerical': 0.02936670071501532, 'mean_word_length': 4.635971223021583, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 30, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '477781', 'license': 'MIT', 'n_tokens_mistral': 1490, 'n_tokens_neox': 1286, 'n_words': 290, 'path': 'image_scraper.ipynb', 'repo_name': 'dietofworms/img-scraper', 'size': '3940'}