<commit_before>import dataset
import sys
import numpy as np
import keras
import os
import logging
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, Reshape, BatchNormalization, Lambda
from keras.layers import Conv2D, MaxPooling2D
from keras import backend as K
from keras.regularizers import l2
from keras.callbacks import ModelCheckpoint, TensorBoard, Callback
from haikunator import Haikunator
from PIL import Image, ImageDraw
from logging import FileHandler
import subprocess
import argparse

haikunator = Haikunator()


class LandmarkPreview(Callback):
    def __init__(self, out_dir,
                 batch_size=10, validation_gen=None):
        super().__init__()
        self.out_dir = out_dir
        self.batch_size = batch_size
        self.validation_gen = validation_gen
        os.makedirs(out_dir, exist_ok=True)

    def draw_landmarks(self, image, landmarks, r=1, fill_color=(255,0,0,100)):
        draw = ImageDraw.Draw(image)
        for row in landmarks:
            x, y = row
            draw.ellipse((x-r, y-r, x+r, y+r), fill=fill_color)
        
    def on_epoch_end(self, epoch, logs=None):
        logs = logs or {}
        if self.validation_data:<commit_msg>opt and bugfix hopefully<commit_after>import dataset
import sys
import numpy as np
import keras
import os
import logging
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, Reshape, BatchNormalization, Lambda
from keras.layers import Conv2D, MaxPooling2D
from keras import backend as K
from keras.regularizers import l2
from keras.callbacks import ModelCheckpoint, TensorBoard, Callback, EarlyStopping, ReduceLROnPlateau
from haikunator import Haikunator
from PIL import Image, ImageDraw
from logging import FileHandler
import subprocess
import argparse

haikunator = Haikunator()


class LandmarkPreview(Callback):
    def __init__(self, out_dir,
                 batch_size=10, validation_gen=None):
        super().__init__()
        self.out_dir = out_dir
        self.batch_size = batch_size
        self.validation_gen = validation_gen
        os.makedirs(out_dir, exist_ok=True)

    def draw_landmarks(self, image, landmarks, r=1, fill_color=(255,0,0,100)):
        draw = ImageDraw.Draw(image)
        for row in landmarks:
            x, y = row
            draw.ellipse((x-r, y-r, x+r, y+r), fill=fill_color)
        
    def on_epoch_end(self, epoch, logs=None):
        logs = logs or {}
        if self.validation_data:
{'fraction_non_alphanumeric': 0.07553234230614704, 'fraction_numerical': 0.01124949779027722, 'mean_word_length': 3.5272727272727273, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '430687', 'n_tokens_mistral': 788, 'n_tokens_neox': 768, 'n_words': 228}