<jupyter_start><jupyter_text>Generate images<jupyter_code>#imports
import os
#restrict gpu usage
os.environ["CUDA_VISIBLE_DEVICES"]="0"

from PIL import Image
import cv2
import bezier
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib
import matplotlib.pyplot as plt
from matplotlib.pyplot import figure
from matplotlib.patches import Rectangle

# setting theme
#px = 1/plt.rcParams['figure.dpi']
#sns.set(style=None, rc={'figure.figsize':(256*px, 256*px)})

# generate random floating point values
from numpy.random import seed, rand, randint
from random import gauss
image_height = image_width = 400
# for 600x600, inches = 8.334,8.334
# for 144x144, inches = 2,2

def multi_line_datasetGen():
    for _ in range(500):
        fig = plt.gcf()
        fig.set_size_inches(5.56,5.56)
        ax = plt.subplot()

        ax.set_axis_off()

        plt.axis([0, image_height, 0, image_width])
        plt.tight_layout(pad=0)

        randLines = np.random.randint(0, 4)

        random_nodes = []
        curveLength = []
    
        
        for i in range(randLines):
            r_nodes = np.random.uniform(0,image_width,[2,2])
            con = np.concatenate((r_nodes[0], r_nodes[1]))
            random_nodes.append(con)

            curve = bezier.Curve(r_nodes, degree=1)
            curveLength.append(curve.length)

            curve.plot(num_pts=256, ax=ax)
        #plt.show()
        #print(fig)
            
        
        # Save just the portion _inside_ the second axis's boundaries
        fn = 'li_{}'.format(_)
        ax.figure.savefig('./workspace/mobile/images/multi_lines/imgs/{}.png'.format(fn))

        cap_data = np.array([fn, random_nodes, curveLength], dtype="object")
        xmlmaker2(cap_data)

        #raw.append(cap_data)

        plt.close('all')
        
        
multi_line_datasetGen()<jupyter_output>Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400.32x400.32)
Figure(400[...]<jupyter_text>Generate XML<jupyter_code># for xml generation

import xml.etree.cElementTree as ET

def xmlgen2(foln, fn, path, a_path, width, height, random_nodes, length):

    root = ET.Element("annotation")
    folder = ET.SubElement(root, "folder").text = foln
    filename = ET.SubElement(root, "filename").text = fn+'.png'
    path = ET.SubElement(root, "path").text = path
    source = ET.SubElement(root, "source") 
    
    ET.SubElement(source, "database").text = "unknown"
    
    size = ET.SubElement(root, "size")

    ET.SubElement(size, "width").text = width
    ET.SubElement(size, "height").text = height
    
    segmented = ET.SubElement(root, "segmented").text = "0"
    _object = ET.SubElement(root, "_object")
    
    ET.SubElement(_object, "name").text = fn
    ET.SubElement(_object, "pose").text = "Unspecified"
    
    lengths = ET.SubElement(_object, "lengths")
    
    for i,j in enumerate(length):
        ET.SubElement(lengths, "l_{}".format(i)).text = str(j)
        
    ET.SubElement(_object, "truncated").text = "0"
    ET.SubElement(_object, "difficult").text = "0"
    
    bndboxs = ET.SubElement(_object, "bndboxs")
    
    for i,j in enumerate(random_nodes):
        xmin, ymin, xmax, ymax = j
        
        hook = ET.SubElement(bndboxs, "b_{}".format(i))
        ET.SubElement(hook, "xmin").text = str(xmin)
        ET.SubElement(hook, "ymin").text = str(ymin)
        ET.SubElement(hook, "xmax").text = str(xmax)
        ET.SubElement(hook, "ymax").text = str(ymax)
    
    tree = ET.ElementTree(root)
    tree.write(a_path)

# get all raw images 
def xmlmaker2(arr):
        fn, random_nodes, length = arr
        '''
        x, y = random_nodes
        xmin = str(x[0])
        ymin = str(x[1])
        xmax = str(y[0])
        ymax = str(y[1])
        '''
        width = str(image_width)
        height = str(image_height)

        foln = "./workspace/mobile/images/multi_lines/"
        fn = fn
        path = "{}/imgs/{}.png".format(foln, fn)
        a_path = "{}/imgs/{}.xml".format(foln, fn)
        xmlgen2(foln, fn, path, a_path, width, height, random_nodes, length)
        
#xmlmaker()<jupyter_output><empty_output><jupyter_text>Partition dataset<jupyter_code># Imports

import os
import re
from shutil import copyfile
import argparse
import math
import random

# Partition images

def partition_images(source, dest, ratio=0.1, copy_xml=True):
    copy_xml = True
    source = source
    dest = dest
    train_dir = os.path.join(dest, 'train')
    test_dir = os.path.join(dest, 'test')

    if not os.path.exists(train_dir):
        os.makedirs(train_dir)
    if not os.path.exists(test_dir):
        os.makedirs(test_dir)

    images = [f for f in os.listdir(source)
              if re.search(r'([a-zA-Z0-9\s_\\.\-\(\):])+(.jpg|.jpeg|.png)$', f)]

    num_images = len(images)
    num_test_images = math.ceil(ratio*num_images)

    for i in range(num_test_images):
        idx = random.randint(0, len(images)-1)
        filename = images[idx]
        copyfile(os.path.join(source, filename),
                 os.path.join(test_dir, filename))
        if copy_xml:
            xml_filename = os.path.splitext(filename)[0]+'.xml'
            copyfile(os.path.join(source, xml_filename),
                     os.path.join(test_dir,xml_filename))
        images.remove(images[idx])

    for filename in images:
        copyfile(os.path.join(source, filename),
                 os.path.join(train_dir, filename))
        if copy_xml:
            xml_filename = os.path.splitext(filename)[0]+'.xml'
            copyfile(os.path.join(source, xml_filename),
                     os.path.join(train_dir, xml_filename))

partition_images("./workspace/mobile/images/multi_lines/imgs", "./workspace/mobile/images/multi_lines")<jupyter_output><empty_output><jupyter_text>XML to CSV<jupyter_code>import os
import glob
import pandas as pd
import io
import xml.etree.ElementTree as ET

def xml_to_csv2(path):
    xml_list = []
    for xml_file in glob.glob(path + '/*.xml'):
        tree = ET.parse(xml_file)
        root = tree.getroot()
        #print(root.findall('_object'))
        for member in root.findall('_object'):
            # if no line
            if(not member[2]):
                value = (member[0].text,
                     int(float(root.find('size')[0].text)),
                     int(float(root.find('size')[1].text)),
                     str(0),
                     str(0),
                     str(0),
                     str(0),
                     str(0),
                     str(0)
                     )
                
                xml_list.append(value)
            else:
                for i,j in enumerate(member[2]):
                    value = (member[0].text,
                         int(float(root.find('size')[0].text)),
                         int(float(root.find('size')[1].text)),
                         member[2][i].text,
                         str(1),
                         float(member[5][i][0].text),
                         float(member[5][i][1].text),
                         float(member[5][i][2].text),
                         float(member[5][i][3].text)
                         )
                    
                    xml_list.append(value)
    column_name = ['filename', 'width', 'height', 'length', 
                   'class', 'xmin', 'xmax', 'ymin', 'ymax']
    xml_df = pd.DataFrame(xml_list, columns=column_name)
    
    return xml_df

for i in ["test", "train"]:
    examples = xml_to_csv2("workspace/mobile/images/multi_lines/{}".format(i))
    #print(examples)

    examples.to_csv("workspace/mobile/images/multi_lines/{}.csv".format(i), index=None)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14403819282720073, 'fraction_numerical': 0.06462505822077318, 'mean_word_length': 2.4661016949152543, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 32}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '403166', 'license': 'Apache-2.0', 'n_tokens_mistral': 3346, 'n_tokens_neox': 2977, 'n_words': 513, 'path': 'TESTS/preprocessor.ipynb', 'repo_name': 'alexander-g/bezierobjdet', 'size': '8588'}