<jupyter_start><jupyter_text>データフレームの準備<jupyter_code>import glob
import os
import json
import pandas as pd
import shutil
import json
import numpy as np
from tqdm import tqdm
_df_dict = {}
_df_dict['train'] = pd.read_csv('datasets/COCOdata/train.csv')
_df_dict['test'] = pd.read_csv('datasets/COCOdata/test.csv')
_df_dict['val'] = pd.read_csv('datasets/COCOdata/val.csv')
_df_dict['train'][:5]<jupyter_output><empty_output><jupyter_text>XMLファイルの作成<jupyter_code>from xml.etree.ElementTree import *
import xml.etree.ElementTree as ET
import xml.dom.minidom as md
from PIL import Image
# 画像のサイズとチャネル数を取得
def get_image_info(path):
    img = Image.open(path)
    return img.width, img.height, img.getbands()
# ディレクトリとファイル名のみに分割
def file_dir_split(file_path):
    split_path = file_path.rsplit('/',1)
    dir_path = split_path[0]
    file_name = split_path[1]
    return file_name, dir_path
# VOCアノテーションXML ファイルを作成
def create_xml(ann_xml):
    annotation = Element('annotation')

    folder = SubElement(annotation, 'folder')
    folder.text = ann_xml['folder']

    path = SubElement(annotation, 'path')
    path.text = ann_xml['path']

    source = SubElement(annotation, 'source')
    database = SubElement(source, 'database')
    database.text = ann_xml['source']['database']

    size = SubElement(annotation, 'size')
    width = SubElement(size, 'width')
    height = SubElement(size, 'height')
    depth = SubElement(size, 'depth')
    width.text = ann_xml['size']['width']
    height.text = ann_xml['size']['height']
    depth.text = ann_xml['size']['depth']

    segmented = SubElement(annotation, 'segmented')
    segmented.text = ann_xml['segmented']

    for i, ann_object in enumerate(ann_xml['objects']):
        object = SubElement(annotation, 'object')

        name = SubElement(object, 'name')
        name.text = ann_object['name']

        pose = SubElement(object, 'pose')
        pose.text = ann_object['pose']

        truncated = SubElement(object, 'truncated')
        truncated.text = ann_object['truncated']

        difficult = SubElement(object, 'difficult')
        difficult.text = ann_object['difficult']

        bndbox = SubElement(object, 'bndbox')
        xmin = SubElement(bndbox, 'xmin')
        xmin.text = ann_object['bndbox']['xmin']
        ymin = SubElement(bndbox, 'ymin')
        ymin.text = ann_object['bndbox']['ymin']
        xmax = SubElement(bndbox, 'xmax')
        xmax.text = ann_object['bndbox']['xmax']
        ymax = SubElement(bndbox, 'ymax')
        ymax.text = ann_object['bndbox']['ymax']

    file_name, dir_path = file_dir_split(ann_xml['path'])
    file_name = file_name.split('.')[0]
    out_file_path = f'{dir_path}/{file_name}.xml'
    document = md.parseString(ET.tostring(annotation, 'utf-8'))
    with open(out_file_path, 'w') as f:
        document.writexml(f, encoding='utf-8', newl='\n', indent='', addindent='  ')
base_dir = 'datasets/COCOdata/COCOdata/Annotations_baobab'
# XMLファイルに変換
for dataset_kind, df in df_dict.items():
    file_names = df['file_name'].unique()
    i = 0
    for file_name in tqdm(file_names):
        if dataset_kind == 'train':
            if i < 56970:
                i += 1
                continue
        ann_xml = {}
        ann_xml['folder'] = dataset_kind
        ann_xml['filename'] = file_name
        ann_xml['path'] = f'{base_dir}/{ann_xml["filename"]}'
        ann_xml['source'] = {
            "database": 'Unknown'
        }
        width = df[df['file_name'] == file_name]['width'].unique().item()
        height= df[df['file_name'] == file_name]['height'].unique().item()
        depth = 3
        ann_xml['size'] = {
            "width": str(width),
            "height": str(height),
            "depth": str(depth)
        }
        ann_xml['segmented'] = '0'
        ann_xml['objects'] = []
        for index, row in df[df['file_name'] == file_name].iterrows():
            xmin = int(row['xmin'])
            ymin = int(row['ymin'])
            xmax = int(row['xmax'])
            ymax = int(row['ymax'])
            if xmin >= xmax:
                continue
                xmax += 1
                if xmax > width:
                    xmax = width
                    xmin -= 1
                print(dataset_kind)
                print(file_name)
                print(xmin, xmax)
            if ymin >= ymax:
                continue
                ymax += 1
                if ymax > height:
                    ymax = height
                    ymin -= 1
                print(dataset_kind)
                print(file_name)
                print(ymin, ymax)
            bndbox = {
                "xmin": str(xmin),
                "ymin": str(ymin),
                "xmax": str(xmax),
                "ymax": str(ymax)
            }
            object = {
                "name": row['name'],
                "pose": 'Unspecified',
                "truncated": '0',
                "difficult": '0',
                "bndbox": bndbox
            }
            ann_xml['objects'].append(object)
        create_xml(ann_xml)
# データセットリストファイル train.txt, test.txt, val.txtを作成
for dataset_kind in dataset_kinds:
    xml_dir = f'{base_dir}/{dataset_kind}'
    xml_files = glob_file_names(xml_dir, 'xml')
    with open(f'{base_dir}/{dataset_kind}.txt', 'w') as f:
        for file in xml_files:
            file_name, _ = file_dir_split(file)
            file_name = file_name.split('.')[0]
            f.write(f'{file_name}\n')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14170780764307916, 'fraction_numerical': 0.004205522033278479, 'mean_word_length': 2.35995085995086, 'pattern_counts': {'":': 13, '<': 10, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 33}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '547042', 'license': 'MIT', 'n_tokens_mistral': 1881, 'n_tokens_neox': 1830, 'n_words': 333, 'path': 'anno_data_conv/MSCOCO_to_VOC_format-fixed.ipynb', 'repo_name': 'ChairoHack/ssd.pytorch', 'size': '5469'}