<commit_before>#!/usr/bin/env python3
import io
import os
import random
from configparser import ConfigParser
from glob import glob
from zipfile import ZipFile

from jinja2 import Template
from PIL import Image

S = 500
M = 1000
L = 2000
TEMPLATE_NAME = "template.html"

if os.path.exists(TEMPLATE_NAME):
    template_path = TEMPLATE_NAME
else:
    template_path = os.path.join(os.path.dirname(__file__), TEMPLATE_NAME)
T = Template(open(template_path, "r").read())


def create_website(root="."):
    """
    Walk the directory tree, rename images, and generate indexes.
    """

    for (dir, dirs, files) in os.walk(root):
        if dir.startswith("./."):
            continue
        if dir.startswith("./static"):
            continue
        if os.path.basename(dir) == "thumbnails":
            continue
        if not any([file.lower().endswith(".jpg") for file in files]):
            continue

        os.makedirs(os.path.join(dir, "thumbnails"), exist_ok=True)
        photos = rename_images(dir)
        generate_index(dir, photos)


def rename_images(dir):
    """
    Rename images sequentially.
    """

    inifile = os.path.join(dir, "photog.ini")
    options = read_inifile(inifile)

    # Store Exif DateTimeOriginal
    photos = []
    for image in glob("*.jpg", root_dir=dir):
        basename = image.split(".", maxsplit=1)[0]
        im = Image.open(os.path.join(dir, image))
        date = im._getexif()[36867]
        photos.append(
            {
                "basename": basename,
                "date": date,
            }
        )

        # Rename
        for alt in glob(f"{basename}.*", root_dir=dir):
            ext = alt.split(".", maxsplit=1)[1]
            src = os.path.join(dir, alt)
            dst = os.path.join(dir, f"_{alt}")
            os.rename(src, dst)

    # Sort
    if options.get("sort") == "random":
        random.shuffle(photos)
    else:
        photos.sort(key=lambda p: p["date"])
        if options.get("sort") == "descending":
            photos.reverse()

    # Re-rename
    for counter, image in enumerate(photos):
        basename = image["basename"]
        for alt in glob(f"_{basename}.*", root_dir=dir):
            ext = alt.split(".", maxsplit=1)[1]
            src = os.path.join(dir, alt)
            dst = os.path.join(dir, f"{counter+1}.{ext}")
            os.rename(src, dst)
        image["basename"] = str(counter + 1)

    return photos


def generate_index(dir, photos):
    """
    Generate index.html
    """

    inifile = os.path.join(dir, "photog.ini")
    options = read_inifile(inifile)
    zippath = os.path.join(dir, "all.zip")
    if options.get("zip", True):
        zipfile = ZipFile(zippath, "w")
    elif os.path.exists(zippath):
        os.remove(zippath)

    for image in photos:
        basename = image["basename"]
        filename = f"{basename}.jpg"
        path = os.path.join(dir, filename)
        small_thumbnail = os.path.join("thumbnails", f"{basename} (small).jpg")
        medium_thumbnail = os.path.join("thumbnails", f"{basename} (medium).jpg")
        large_thumbnail = os.path.join("thumbnails", f"{basename} (large).jpg")
        im = Image.open(path)
        original_width, original_height = im.size
        exif = im.info["exif"]

        # Generate S, M and L thumbnails
        im.thumbnail((L, 99999))
        im.save(os.path.join(dir, large_thumbnail), quality=95, exif=exif)
        im.thumbnail((M, 99999))
        im.save(os.path.join(dir, medium_thumbnail), quality=95, exif=exif)
        im.thumbnail((S, 99999))
        im.save(os.path.join(dir, small_thumbnail), quality=95, exif=exif)

        # Add original and large thumbnail to zip archive
        if options.get("zip", True):
            zipfile.write(path, os.path.join("print", filename))
            zipfile.write(
                os.path.join(dir, large_thumbnail), os.path.join("web", filename)
            )

        image.update(
            {
                "small": small_thumbnail,
                "medium": medium_thumbnail,<commit_msg>Smarter generation of thumbnails<commit_after>#!/usr/bin/env python3
import io
import os
import random
from configparser import ConfigParser
from glob import glob
from zipfile import ZipFile

from jinja2 import Template
from PIL import Image

S = 500
M = 1000
L = 2000
TEMPLATE_NAME = "template.html"

if os.path.exists(TEMPLATE_NAME):
    template_path = TEMPLATE_NAME
else:
    template_path = os.path.join(os.path.dirname(__file__), TEMPLATE_NAME)
T = Template(open(template_path, "r").read())


def create_website(root="."):
    """
    Walk the directory tree, rename images, and generate indexes.
    """

    for (dir, dirs, files) in os.walk(root):
        if dir.startswith("./."):
            continue
        if dir.startswith("./static"):
            continue
        if os.path.basename(dir) == "thumbnails":
            continue
        if not any([file.lower().endswith(".jpg") for file in files]):
            continue

        os.makedirs(os.path.join(dir, "thumbnails"), exist_ok=True)
        photos = rename_images(dir)
        generate_index(dir, photos)


def rename_images(dir):
    """
    Rename images sequentially.
    """

    inifile = os.path.join(dir, "photog.ini")
    options = read_inifile(inifile)

    # Store Exif DateTimeOriginal
    photos = []
    for image in glob("*.jpg", root_dir=dir):
        basename = image.split(".", maxsplit=1)[0]
        im = Image.open(os.path.join(dir, image))
        date = im._getexif()[36867]
        photos.append(
            {
                "basename": basename,
                "date": date,
            }
        )

        # Rename
        for alt in glob(f"{basename}.*", root_dir=dir):
            ext = alt.split(".", maxsplit=1)[1]
            src = os.path.join(dir, alt)
            dst = os.path.join(dir, f"_{alt}")
            os.rename(src, dst)

    # Sort
    if options.get("sort") == "random":
        random.shuffle(photos)
    else:
        photos.sort(key=lambda p: p["date"])
        if options.get("sort") == "descending":
            photos.reverse()

    # Re-rename
    for counter, image in enumerate(photos):
        basename = image["basename"]
        for alt in glob(f"_{basename}.*", root_dir=dir):
            ext = alt.split(".", maxsplit=1)[1]
            src = os.path.join(dir, alt)
            dst = os.path.join(dir, f"{counter+1}.{ext}")
            os.rename(src, dst)
        image["basename"] = str(counter + 1)

    return photos


def generate_index(dir, photos):
    """
    Generate index.html
    """

    inifile = os.path.join(dir, "photog.ini")
    options = read_inifile(inifile)
    zippath = os.path.join(dir, "all.zip")
    if options.get("zip", True):
        zipfile = ZipFile(zippath, "w")
    elif os.path.exists(zippath):
        os.remove(zippath)

    for image in photos:
        basename = image["basename"]
        filename = f"{basename}.jpg"
        path = os.path.join(dir, filename)
        small_thumbnail = os.path.join("thumbnails", f"{basename} (small).jpg")
        medium_thumbnail = os.path.join("thumbnails", f"{basename} (medium).jpg")
        large_thumbnail = os.path.join("thumbnails", f"{basename} (large).jpg")
        im = Image.open(path)
        original_width, original_height = im.size
        exif = im.info["exif"]

        try:
            update_file = os.path.getmtime(os.path.join(dir, large_thumbnail)) < os.path.getmtime(path)
        except:
            update_file = True
        if update_file:
            # Generate S, M and L thumbnails
            im.thumbnail((L, 99999))
            im.save(os.path.join(dir, large_thumbnail), quality=95, exif=exif)
            im.thumbnail((M, 99999))
            im.save(os.path.join(dir, medium_thumbnail), quality=95, exif=exif)
            im.thumbnail((S, 99999))
            im.save(os.path.join(dir, small_thumbnail), quality=95, exif=exif)

        # Add original and large thumbnail to zip archive
        if options.get("zip", True):
            zipfile.write(path, os.path.join("print", filename))
            zipfile.write(
                os.path.join(dir, large_thumbnail), os.path.join("web", filename)
            )

        image.update(
            {
                "small": small_thumbnail,
                "medium": medium_thumbnail,
{'fraction_non_alphanumeric': 0.14069081718618365, 'fraction_numerical': 0.011313034059453605, 'mean_word_length': 2.2677939441604402, 'pattern_counts': {'":': 14, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1830471', 'n_tokens_mistral': 2788, 'n_tokens_neox': 2667, 'n_words': 572}