<commit_before>import os
import posixpath
import urllib.parse
import aiohttp
from PIL import Image

IDOL_IMAGES_PATH = 'idol_images/'
OUTPUT_PATH = 'scout_output/'
CIRCLE_DISTANCE = 10


async def create_image(
        idol_circle_urls: list, num_rows: int, output_filename: str) -> str:
    """
    Creates a stitched together image of idol circles.

    :param idol_circle_urls: urls of idol circle images to be stitched together.
    :param num_rows: Number of rows to use in the image
    :param output_filename: name of output image file

    :return: path pointing to created image
    """
    image_filepaths = []  # list of image filepaths

    # Save images that do not exists
    for image_url in idol_circle_urls:
        url_path = urllib.parse.urlsplit(image_url).path
        filename = posixpath.basename(url_path)
        image_filepaths.append(IDOL_IMAGES_PATH + filename)

        await download_image_from_url(image_url, IDOL_IMAGES_PATH + filename)

    # Load images
    circle_images = []
    for image_filepath in image_filepaths:
        circle_images.append(Image.open(image_filepath))

    image = await _build_image(circle_images, num_rows, 10, 10)

    image.save(OUTPUT_PATH + output_filename, 'PNG')
    return OUTPUT_PATH + output_filename


async def download_image_from_url(url: str, path: str) -> str:
    """
    Downloads an image from a url and saves it to a specified location.

    :param url: url of image
    :param path: path where image will be saved to

    :return: path of saved image
    """
    # Create directories for storing images if they do not exist
    _ensure_path_exists(IDOL_IMAGES_PATH)
    _ensure_path_exists(OUTPUT_PATH)

    if not os.path.exists(path):
        print('Saving ' + url + ' to ' + path)
        fp = open(path, 'wb')
        response = await aiohttp.get(url)
        image = await response.read()
        fp.write(image)
        fp.close()

    return path


async def _build_image(circle_images: list, num_rows: int,
        x_distance: int, y_distance: int) -> object:
    """
    Stitches together a list of images to an output image.

    :param circle_images: list of image object being stitched together
    :param num_rows: number of rows to lay the image out in
    :param: x_distance: x spacing between each image
    :param y_distane: y spacing between each image

    :return: ouput image object
    """
    # Compute required height and width
    circle_width, circle_height = circle_images[0].size
    out_height = num_rows * (circle_height + y_distance)
    out_width = ((len(circle_images) + 1) * (circle_width + x_distance)) // 2

    image = Image.new('RGBA', (out_width, out_height))

    circle_rows = []
    for row_index in range(0, num_rows):
        circle_rows.append([])

    i = 0
    for circle_image in circle_images:
        circle_rows[i % len(circle_rows)].append(circle_image)
        i += 1

    x = 0
    y = 0
    for row_index in range(0, len(circle_rows)):
        x = 0

        # Offset row
        if (row_index + 1) % 2 == 0:
            x += circle_width // 2

        for col_index in range(0, len(circle_rows[row_index])):
            image.paste(circle_rows[row_index][col_index], (x, y))
            x += circle_width + x_distance

        y += circle_height + y_distance

    return image


def _ensure_path_exists(path: str):
    """
    Makes sure a path exists. Creates new directory if it does not.

    :param path: path being checked
    """
    try:
        os.mkdir(path)
    except OSError as Exception:
        return
<commit_msg>Updated create_image to be more reabable(I think?)<commit_after>import urllib.parse
from collections import deque
from pathlib import Path
from typing import List, Sequence, Tuple

from PIL import Image
from aiohttp import ClientSession

IDOL_IMAGES_PATH = Path('idol_images')
OUTPUT_PATH = Path('scout_output')
CIRCLE_DISTANCE = 10


async def create_image(
        idol_circle_urls: list, num_rows: int, output_filename: str) -> Path:
    """
    Creates a stitched together image of idol circles.

    :param idol_circle_urls: urls of idol circle images to be stitched together.
    :param num_rows: Number of rows to use in the image
    :param output_filename: name of output image file
    :return: path pointing to created image
    """
    image_filepaths = []  # list of image filepaths
    # Save images that do not exists
    session = ClientSession()
    for image_url in idol_circle_urls:
        url_path = urllib.parse.urlsplit(image_url).path
        file_path = IDOL_IMAGES_PATH.joinpath(Path(url_path).name)
        image_filepaths.append(file_path)
        await download_image_from_url(image_url, file_path, session)
    session.close()
    # Load images
    circle_images = [Image.open(str(i)) for i in image_filepaths]
    image = _build_image(circle_images, num_rows, 10, 10)
    output_path = OUTPUT_PATH.joinpath(output_filename)
    image.save(str(output_path), 'PNG')
    return output_path


async def download_image_from_url(
        url: str, path: Path, session: ClientSession) -> Path:
    """
    Downloads an image from a url and saves it to a specified location.

    :param url: url of image
    :param path: path where image will be saved to
    :param session: the aiohttp ClientSession
    :return: path of saved image
    """
    # Create directories for storing images if they do not exist
    if not IDOL_IMAGES_PATH.is_dir():
        IDOL_IMAGES_PATH.mkdir(parents=True, exist_ok=True)
    if not OUTPUT_PATH.is_dir():
        OUTPUT_PATH.mkdir(parents=True, exist_ok=True)
    async with session.get(url) as r:
        if r.status == 200 and not path.is_file():
            print('Saving ' + url + ' to ' + str(path))
            image = await r.read()
            path.write_bytes(image)
    return path


def _build_image(circle_images: list, num_rows: int,
                 x_padding: int, y_padding: int) -> Image:
    """
    Stitches together a list of images to an output image.

    :param circle_images: list of image object being stitched together
    :param num_rows: number of rows to lay the image out in
    :param x_padding: x spacing between each image
    :param y_padding: y spacing between each row
    :return: ouput image object
    """
    sizes = [circle.size for circle in circle_images]
    positions, x, y = compute_pos(sizes, num_rows, x_padding, y_padding)
    image_queue = deque(circle_images)
    img = Image.new('RGBA', (x, y))
    for row in positions:
        for c in row:
            img.paste(image_queue.popleft(), c)
    return img


def compute_pos(
        sizes: List[Tuple[int]], num_rows: int,
        x_padding: int, y_padding: int) -> tuple:
    """
    Compute all images positions from the list of images and number of rows.
    :param sizes: A list of sizes for all images.
    :param num_rows: the number of rows.
    :param x_padding: x spacing between each image
    :param y_padding: y spacing between each row
    :return: Positions for all images, the total x size, the total y size
    """
    total_x, total_y = 0, 0
    rows = split(sizes, num_rows)
    res = []
    row_x_sizes = []

    for row in rows:
        row_x_sizes.append(
            sum([i[0] for i in row]) + x_padding * (len(row) - 1))
        row_y = max([i[1] for i in row])
        res.append(compute_row(row, x_padding, total_y))
        total_y += row_y + y_padding
        total_x = max(row_x_sizes)
    i = 0
    for row, row_x, row_sizes in zip(res, row_x_sizes, rows):
        actual = row_sizes[-1][0] + row[-1][0]
        diff = round((total_x - actual) / 2)
        if diff > 0:
            res[i] = [(x + diff, y) for x, y in row]
        i += 1
    return res, total_x, total_y - y_padding


def compute_row(
        row_sizes: List[Tuple[int]],
        x_padding: int, y_pos: int) -> List[Tuple[int]]:
    """
    Compute the positions for a single row.
    :param row_sizes: the list of image sizes in that row.
    :param x_padding: the x padding in between images.
    :param y_pos: the y position of that row.
    :return: A list of (x, y) positions for that row.
    """
    res = []
    x = 0
    for size in row_sizes:
        res.append((x, y_pos))
        x += size[0] + x_padding
    return res


def split(in_: Sequence, chunks: int) -> List[List]:
    """
    Split a sequence into roughly equal chunks.
    :param in_: the input sequence.
    :param chunks: the number of chunks.
    :return: the sequence split up into chunks.
    """
    k, m = divmod(len(in_), chunks)
    return [
        in_[i * k + min(i, m):(i + 1) * k + min(i + 1, m)]
        for i in range(chunks)
    ]


def _build_image_old(circle_images: list, num_rows: int,
                     x_distance: int, y_distance: int):
    """
    Stitches together a list of images to an output image.
    :param circle_images: list of image object being stitched together
    :param num_rows: number of rows to lay the image out in
    :param: x_distance: x spacing between each image
    :param y_distance: y spacing between each image
    :return: ouput image object
    """
    # Compute required height and width
    circle_width, circle_height = circle_images[0].size
    out_height = num_rows * (circle_height + y_distance)
    out_width = ((len(circle_images) + 1) * (circle_width + x_distance)) // 2

    image = Image.new('RGBA', (out_width, out_height))

    circle_rows = []
    for row_index in range(0, num_rows):
        circle_rows.append([])

    i = 0
    for circle_image in circle_images:
        circle_rows[i % len(circle_rows)].append(circle_image)
        i += 1

    x = 0
    y = 0
    for row_index in range(0, len(circle_rows)):
        x = 0

        # Offset row
        if (row_index + 1) % 2 == 0:
            x += circle_width // 2

        for col_index in range(0, len(circle_rows[row_index])):
            image.paste(circle_rows[row_index][col_index], (x, y))
            x += circle_width + x_distance

        y += circle_height + y_distance

    return image


if __name__ == '__main__':
    from os import listdir
    from random import choice
    from time import time

    images = [p for p in listdir('./idol_images') if p.endswith('.png')]
    image_filepaths = ['idol_images/' + choice(images) for _ in range(11)]
    circle_images = [Image.open(str(i)) for i in image_filepaths]
    s = time()
    for _ in range(5000):
        image = _build_image(circle_images, 3, 10, 10)
    e = time()
    print('new avg time')
    print((e - s) / 5000)
    s = time()
    for _ in range(5000):
        image = _build_image_old(circle_images, 3, 10, 10)
    e = time()
    print('old avg time')
    print((e - s) / 5000)

    image = _build_image(circle_images, 3, 10, 10)
    output_path = OUTPUT_PATH.joinpath('test.png')
    image.save(str(output_path), 'PNG')

{'fraction_non_alphanumeric': 0.09511045383122285, 'fraction_numerical': 0.008780848507255754, 'mean_word_length': 2.7779329608938546, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '779842', 'n_tokens_mistral': 3573, 'n_tokens_neox': 3419, 'n_words': 1125}