<commit_before># routines to support generating panda3d models

import subprocess
import cv2
import math
import numpy as np
import os
import scipy.spatial

def make_textures(src_dir, analysis_dir, image_list, resolution=512):
    dst_dir = os.path.join(analysis_dir, 'models')
    if not os.path.exists(dst_dir):
        print("Notice: creating models directory =", dst_dir)
        os.makedirs(dst_dir)
    for image in image_list:
        src = os.path.join(src_dir, image.name)
        dst = os.path.join(dst_dir, image.name)
        if not os.path.exists(dst):
            subprocess.run(['convert', '-resize', '%dx%d!' % (resolution, resolution), src, dst])
        
def make_textures_opencv(src_dir, analysis_dir, image_list, resolution=512):
    dst_dir = os.path.join(analysis_dir, 'models')
    if not os.path.exists(dst_dir):
        print("Notice: creating texture directory =", dst_dir)
        os.makedirs(dst_dir)
    for image in image_list:
        src = image.image_file
        dst = os.path.join(dst_dir, image.name + '.JPG')
        print(src, '->', dst)
        if not os.path.exists(dst):
            src = cv2.imread(src, flags=cv2.IMREAD_ANYCOLOR|cv2.IMREAD_ANYDEPTH|cv2.IMREAD_IGNORE_ORIENTATION)
            height, width = src.shape[:2]
            # downscale image first
            method = cv2.INTER_AREA  # cv2.INTER_AREA
            scale = cv2.resize(src, (0,0),
                               fx=resolution/float(width),
                               fy=resolution/float(height),
                               interpolation=method)
            do_equalize = False
            if do_equalize:
                # convert to hsv color space
                hsv = cv2.cvtColor(scale, cv2.COLOR_BGR2HSV)
                hue,sat,val = cv2.split(hsv)
                # adaptive histogram equalization on 'value' channel
                clahe = cv2.createCLAHE(clipLimit=3.0, tileGridSize=(8,8))
                aeq = clahe.apply(val)
                # recombine
                hsv = cv2.merge((hue,sat,aeq))
                # convert back to rgb
                result = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
            else:
                result = scale
            cv2.imwrite(dst, result)
            print("Texture %dx%d %s" % (resolution, resolution, dst))
            
def generate_from_grid(proj, group, ref_image=False, src_dir=".",
                       analysis_dir=".", resolution=512 ):
    # make the textures if needed
    make_textures_opencv(src_dir, analysis_dir, proj.image_list, resolution)
    
    for name in group:
        image = proj.findImageByName(name)
        if len(image.grid_list) == 0:
            continue

        root, ext = os.path.splitext(image.name)
        name = os.path.join( analysis_dir, "models", root + ".egg" )
        print("EGG file name:", name)

        f = open(name, "w")
        f.write("<CoordinateSystem> { Z-Up }\n\n")
        # f.write("<Texture> tex { \"" + image.name + ".JPG\" }\n\n")
        f.write("<Texture> tex { \"dummy.jpg\" }\n\n")
        f.write("<VertexPool> surface {\n")

        # this is contructed in a weird way, but we generate the 2d
        # iteration in the same order that the original grid_list was
        # constucted so it works.
        width, height = proj.cam.get_image_params()
        steps = int(math.sqrt(len(image.grid_list))) - 1
        n = 1
        nan_list = []<commit_msg>Make an actual "dummy.jpg" image so the windows version of pandas doesn't complain (the warning message scrolling in a dos cmd shell slows loading way down!)<commit_after># routines to support generating panda3d models

import subprocess
import cv2
import math
import numpy as np
import os
import scipy.spatial

def make_textures(src_dir, analysis_dir, image_list, resolution=512):
    dst_dir = os.path.join(analysis_dir, 'models')
    if not os.path.exists(dst_dir):
        print("Notice: creating models directory =", dst_dir)
        os.makedirs(dst_dir)
    for image in image_list:
        src = os.path.join(src_dir, image.name)
        dst = os.path.join(dst_dir, image.name)
        if not os.path.exists(dst):
            subprocess.run(['convert', '-resize', '%dx%d!' % (resolution, resolution), src, dst])
        
def make_textures_opencv(src_dir, analysis_dir, image_list, resolution=512):
    dst_dir = os.path.join(analysis_dir, 'models')
    if not os.path.exists(dst_dir):
        print("Notice: creating texture directory =", dst_dir)
        os.makedirs(dst_dir)
    for image in image_list:
        src = image.image_file
        dst = os.path.join(dst_dir, image.name + '.JPG')
        print(src, '->', dst)
        if not os.path.exists(dst):
            src = cv2.imread(src, flags=cv2.IMREAD_ANYCOLOR|cv2.IMREAD_ANYDEPTH|cv2.IMREAD_IGNORE_ORIENTATION)
            height, width = src.shape[:2]
            # downscale image first
            method = cv2.INTER_AREA  # cv2.INTER_AREA
            scale = cv2.resize(src, (0,0),
                               fx=resolution/float(width),
                               fy=resolution/float(height),
                               interpolation=method)
            do_equalize = False
            if do_equalize:
                # convert to hsv color space
                hsv = cv2.cvtColor(scale, cv2.COLOR_BGR2HSV)
                hue,sat,val = cv2.split(hsv)
                # adaptive histogram equalization on 'value' channel
                clahe = cv2.createCLAHE(clipLimit=3.0, tileGridSize=(8,8))
                aeq = clahe.apply(val)
                # recombine
                hsv = cv2.merge((hue,sat,aeq))
                # convert back to rgb
                result = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
            else:
                result = scale
            cv2.imwrite(dst, result)
            print("Texture %dx%d %s" % (resolution, resolution, dst))
    # make the dummy.jpg image from the first texture
    src = os.path.join(dst_dir, image_list[0].image_file)
    dst = os.path.join(dst_dir, "dummy.jpg")
    print("Dummy:", src, dst)
    if not os.path.exists(dst):
        src = cv2.imread(src, flags=cv2.IMREAD_ANYCOLOR|cv2.IMREAD_ANYDEPTH|cv2.IMREAD_IGNORE_ORIENTATION)
        height, width = src.shape[:2]
        # downscale image first
        method = cv2.INTER_AREA  # cv2.INTER_AREA
        resolution = 64
        dummy = cv2.resize(src, (0,0),
                           fx=resolution/float(width),
                           fy=resolution/float(height),
                           interpolation=method)
        cv2.imwrite(dst, dummy)
        print("Texture %dx%d %s" % (resolution, resolution, dst))
    
            
def generate_from_grid(proj, group, ref_image=False, src_dir=".",
                       analysis_dir=".", resolution=512 ):
    # make the textures if needed
    make_textures_opencv(src_dir, analysis_dir, proj.image_list, resolution)
    
    for name in group:
        image = proj.findImageByName(name)
        if len(image.grid_list) == 0:
            continue

        root, ext = os.path.splitext(image.name)
        name = os.path.join( analysis_dir, "models", root + ".egg" )
        print("EGG file name:", name)

        f = open(name, "w")
        f.write("<CoordinateSystem> { Z-Up }\n\n")
        # f.write("<Texture> tex { \"" + image.name + ".JPG\" }\n\n")
        f.write("<Texture> tex { \"dummy.jpg\" }\n\n")
        f.write("<VertexPool> surface {\n")

        # this is contructed in a weird way, but we generate the 2d
        # iteration in the same order that the original grid_list was
        # constucted so it works.
        width, height = proj.cam.get_image_params()
        steps = int(math.sqrt(len(image.grid_list))) - 1
        n = 1
        nan_list = []
{'fraction_non_alphanumeric': 0.11510605276771857, 'fraction_numerical': 0.01189860320744956, 'mean_word_length': 2.0870259481037925, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '476921', 'n_tokens_mistral': 2503, 'n_tokens_neox': 2404, 'n_words': 565}