initial commit

Dockerfile

@@ -0,0 +1,31 @@
+# read the doc: https://huggingface.co/docs/hub/spaces-sdks-docker
+# you will also find guides on how best to write your Dockerfile
+
+FROM python:3.9
+
+WORKDIR /code
+
+COPY ./requirements.txt /code/requirements.txt
+
+RUN apt-get update && apt-get upgrade -y && apt-get install -y libgl1-mesa-dev
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+RUN mim install mmcv-full==1.7.0
+RUN pip install mmdet mmpose
+
+# Set up a new user named "user" with user ID 1000
+RUN useradd -m -u 1000 user
+
+# Switch to the "user" user
+USER user
+
+# Set home to the user's home directory
+ENV HOME=/home/user \
+    PATH=/home/user/.local/bin:$PATH
+
+# Set the working directory to the user's home directory
+WORKDIR $HOME/app
+
+# Copy the current directory contents into the container at $HOME/app setting the owner to the user
+COPY --chown=user . $HOME/app
+
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

abgr.py

@@ -0,0 +1,47 @@
+import gradio as gr
+import huggingface_hub
+import onnxruntime as rt
+import numpy as np
+import cv2
+
+providers = ['CUDAExecutionProvider', 'CPUExecutionProvider']
+model_path = huggingface_hub.hf_hub_download("skytnt/anime-seg", "isnetis.onnx")
+rmbg_model = rt.InferenceSession(model_path, providers=providers)
+
+def get_mask(img, s=1024):
+    img = (img / 255).astype(np.float32)
+    h, w = h0, w0 = img.shape[:-1]
+    h, w = (s, int(s * w / h)) if h > w else (int(s * h / w), s)
+    ph, pw = s - h, s - w
+    img_input = np.zeros([s, s, 3], dtype=np.float32)
+    img_input[ph // 2:ph // 2 + h, pw // 2:pw // 2 + w] = cv2.resize(img, (w, h))
+    img_input = np.transpose(img_input, (2, 0, 1))
+    img_input = img_input[np.newaxis, :]
+    mask = rmbg_model.run(None, {'img': img_input})[0][0]
+    mask = np.transpose(mask, (1, 2, 0))
+    mask = mask[ph // 2:ph // 2 + h, pw // 2:pw // 2 + w]
+    mask = cv2.resize(mask, (w0, h0))[:, :, np.newaxis]
+    return mask
+
+
+def remove_bg(img):
+    mask = get_mask(img)
+    img = (mask * img + 255 * (1 - mask)).astype(np.uint8)
+    mask = (mask * 255).astype(np.uint8)
+    img = np.concatenate([img, mask], axis=2, dtype=np.uint8)
+    mask = mask.repeat(3, axis=2)
+    return mask, img
+
+def split_image(img):
+    mask = get_mask(img)
+    inv_mask = 1 - mask
+    fg = (mask * img + 255 * inv_mask).astype(np.uint8)
+    bg = (inv_mask * img + 255 * mask).astype(np.uint8)
+    mask = (mask * 255).astype(np.uint8)
+    inv_mask = (inv_mask * 255).astype(np.uint8)
+    fg = np.concatenate([fg, mask], axis=2, dtype=np.uint8)
+    bg = np.concatenate([bg, inv_mask], axis=2, dtype=np.uint8)
+    mask = mask.repeat(3, axis=2)
+    return mask, fg, bg
+
+

external/coco.py

@@ -0,0 +1,181 @@
+dataset_info = dict(
+    dataset_name='coco',
+    paper_info=dict(
+        author='Lin, Tsung-Yi and Maire, Michael and '
+        'Belongie, Serge and Hays, James and '
+        'Perona, Pietro and Ramanan, Deva and '
+        r'Doll{\'a}r, Piotr and Zitnick, C Lawrence',
+        title='Microsoft coco: Common objects in context',
+        container='European conference on computer vision',
+        year='2014',
+        homepage='http://cocodataset.org/',
+    ),
+    keypoint_info={
+        0:
+        dict(name='nose', id=0, color=[51, 153, 255], type='upper', swap=''),
+        1:
+        dict(
+            name='left_eye',
+            id=1,
+            color=[51, 153, 255],
+            type='upper',
+            swap='right_eye'),
+        2:
+        dict(
+            name='right_eye',
+            id=2,
+            color=[51, 153, 255],
+            type='upper',
+            swap='left_eye'),
+        3:
+        dict(
+            name='left_ear',
+            id=3,
+            color=[51, 153, 255],
+            type='upper',
+            swap='right_ear'),
+        4:
+        dict(
+            name='right_ear',
+            id=4,
+            color=[51, 153, 255],
+            type='upper',
+            swap='left_ear'),
+        5:
+        dict(
+            name='left_shoulder',
+            id=5,
+            color=[0, 255, 0],
+            type='upper',
+            swap='right_shoulder'),
+        6:
+        dict(
+            name='right_shoulder',
+            id=6,
+            color=[255, 128, 0],
+            type='upper',
+            swap='left_shoulder'),
+        7:
+        dict(
+            name='left_elbow',
+            id=7,
+            color=[0, 255, 0],
+            type='upper',
+            swap='right_elbow'),
+        8:
+        dict(
+            name='right_elbow',
+            id=8,
+            color=[255, 128, 0],
+            type='upper',
+            swap='left_elbow'),
+        9:
+        dict(
+            name='left_wrist',
+            id=9,
+            color=[0, 255, 0],
+            type='upper',
+            swap='right_wrist'),
+        10:
+        dict(
+            name='right_wrist',
+            id=10,
+            color=[255, 128, 0],
+            type='upper',
+            swap='left_wrist'),
+        11:
+        dict(
+            name='left_hip',
+            id=11,
+            color=[0, 255, 0],
+            type='lower',
+            swap='right_hip'),
+        12:
+        dict(
+            name='right_hip',
+            id=12,
+            color=[255, 128, 0],
+            type='lower',
+            swap='left_hip'),
+        13:
+        dict(
+            name='left_knee',
+            id=13,
+            color=[0, 255, 0],
+            type='lower',
+            swap='right_knee'),
+        14:
+        dict(
+            name='right_knee',
+            id=14,
+            color=[255, 128, 0],
+            type='lower',
+            swap='left_knee'),
+        15:
+        dict(
+            name='left_ankle',
+            id=15,
+            color=[0, 255, 0],
+            type='lower',
+            swap='right_ankle'),
+        16:
+        dict(
+            name='right_ankle',
+            id=16,
+            color=[255, 128, 0],
+            type='lower',
+            swap='left_ankle')
+    },
+    skeleton_info={
+        0:
+        dict(link=('left_ankle', 'left_knee'), id=0, color=[0, 255, 0]),
+        1:
+        dict(link=('left_knee', 'left_hip'), id=1, color=[0, 255, 0]),
+        2:
+        dict(link=('right_ankle', 'right_knee'), id=2, color=[255, 128, 0]),
+        3:
+        dict(link=('right_knee', 'right_hip'), id=3, color=[255, 128, 0]),
+        4:
+        dict(link=('left_hip', 'right_hip'), id=4, color=[51, 153, 255]),
+        5:
+        dict(link=('left_shoulder', 'left_hip'), id=5, color=[51, 153, 255]),
+        6:
+        dict(link=('right_shoulder', 'right_hip'), id=6, color=[51, 153, 255]),
+        7:
+        dict(
+            link=('left_shoulder', 'right_shoulder'),
+            id=7,
+            color=[51, 153, 255]),
+        8:
+        dict(link=('left_shoulder', 'left_elbow'), id=8, color=[0, 255, 0]),
+        9:
+        dict(
+            link=('right_shoulder', 'right_elbow'), id=9, color=[255, 128, 0]),
+        10:
+        dict(link=('left_elbow', 'left_wrist'), id=10, color=[0, 255, 0]),
+        11:
+        dict(link=('right_elbow', 'right_wrist'), id=11, color=[255, 128, 0]),
+        12:
+        dict(link=('left_eye', 'right_eye'), id=12, color=[51, 153, 255]),
+        13:
+        dict(link=('nose', 'left_eye'), id=13, color=[51, 153, 255]),
+        14:
+        dict(link=('nose', 'right_eye'), id=14, color=[51, 153, 255]),
+        15:
+        dict(link=('left_eye', 'left_ear'), id=15, color=[51, 153, 255]),
+        16:
+        dict(link=('right_eye', 'right_ear'), id=16, color=[51, 153, 255]),
+        17:
+        dict(link=('left_ear', 'left_shoulder'), id=17, color=[51, 153, 255]),
+        18:
+        dict(
+            link=('right_ear', 'right_shoulder'), id=18, color=[51, 153, 255])
+    },
+    joint_weights=[
+        1., 1., 1., 1., 1., 1., 1., 1.2, 1.2, 1.5, 1.5, 1., 1., 1.2, 1.2, 1.5,
+        1.5
+    ],
+    sigmas=[
+        0.026, 0.025, 0.025, 0.035, 0.035, 0.079, 0.079, 0.072, 0.072, 0.062,
+        0.062, 0.107, 0.107, 0.087, 0.087, 0.089, 0.089
+    ])

external/default_runtime.py

@@ -0,0 +1,20 @@
+checkpoint_config = dict(interval=10)
+
+log_config = dict(
+    interval=50,
+    hooks=[
+        dict(type='TextLoggerHook'),
+        # dict(type='TensorboardLoggerHook')
+        # dict(type='PaviLoggerHook') # for internal services
+    ])
+
+log_level = 'INFO'
+load_from = None
+resume_from = None
+dist_params = dict(backend='nccl')
+workflow = [('train', 1)]
+
+# disable opencv multithreading to avoid system being overloaded
+opencv_num_threads = 0
+# set multi-process start method as `fork` to speed up the training
+mp_start_method = 'fork'

external/faster_rcnn_r50_fpn_coco.py

@@ -0,0 +1,182 @@
+checkpoint_config = dict(interval=1)
+# yapf:disable
+log_config = dict(
+    interval=50,
+    hooks=[
+        dict(type='TextLoggerHook'),
+        # dict(type='TensorboardLoggerHook')
+    ])
+# yapf:enable
+dist_params = dict(backend='nccl')
+log_level = 'INFO'
+load_from = None
+resume_from = None
+workflow = [('train', 1)]
+# optimizer
+optimizer = dict(type='SGD', lr=0.02, momentum=0.9, weight_decay=0.0001)
+optimizer_config = dict(grad_clip=None)
+# learning policy
+lr_config = dict(
+    policy='step',
+    warmup='linear',
+    warmup_iters=500,
+    warmup_ratio=0.001,
+    step=[8, 11])
+total_epochs = 12
+
+model = dict(
+    type='FasterRCNN',
+    pretrained='torchvision://resnet50',
+    backbone=dict(
+        type='ResNet',
+        depth=50,
+        num_stages=4,
+        out_indices=(0, 1, 2, 3),
+        frozen_stages=1,
+        norm_cfg=dict(type='BN', requires_grad=True),
+        norm_eval=True,
+        style='pytorch'),
+    neck=dict(
+        type='FPN',
+        in_channels=[256, 512, 1024, 2048],
+        out_channels=256,
+        num_outs=5),
+    rpn_head=dict(
+        type='RPNHead',
+        in_channels=256,
+        feat_channels=256,
+        anchor_generator=dict(
+            type='AnchorGenerator',
+            scales=[8],
+            ratios=[0.5, 1.0, 2.0],
+            strides=[4, 8, 16, 32, 64]),
+        bbox_coder=dict(
+            type='DeltaXYWHBBoxCoder',
+            target_means=[.0, .0, .0, .0],
+            target_stds=[1.0, 1.0, 1.0, 1.0]),
+        loss_cls=dict(
+            type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+        loss_bbox=dict(type='L1Loss', loss_weight=1.0)),
+    roi_head=dict(
+        type='StandardRoIHead',
+        bbox_roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        bbox_head=dict(
+            type='Shared2FCBBoxHead',
+            in_channels=256,
+            fc_out_channels=1024,
+            roi_feat_size=7,
+            num_classes=80,
+            bbox_coder=dict(
+                type='DeltaXYWHBBoxCoder',
+                target_means=[0., 0., 0., 0.],
+                target_stds=[0.1, 0.1, 0.2, 0.2]),
+            reg_class_agnostic=False,
+            loss_cls=dict(
+                type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+            loss_bbox=dict(type='L1Loss', loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_pre=2000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_pre=1000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.05,
+            nms=dict(type='nms', iou_threshold=0.5),
+            max_per_img=100)
+        # soft-nms is also supported for rcnn testing
+        # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05)
+    ))
+
+dataset_type = 'CocoDataset'
+data_root = 'data/coco'
+img_norm_cfg = dict(
+    mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True)
+train_pipeline = [
+    dict(type='LoadImageFromFile'),
+    dict(type='LoadAnnotations', with_bbox=True),
+    dict(type='Resize', img_scale=(1333, 800), keep_ratio=True),
+    dict(type='RandomFlip', flip_ratio=0.5),
+    dict(type='Normalize', **img_norm_cfg),
+    dict(type='Pad', size_divisor=32),
+    dict(type='DefaultFormatBundle'),
+    dict(type='Collect', keys=['img', 'gt_bboxes', 'gt_labels']),
+]
+test_pipeline = [
+    dict(type='LoadImageFromFile'),
+    dict(
+        type='MultiScaleFlipAug',
+        img_scale=(1333, 800),
+        flip=False,
+        transforms=[
+            dict(type='Resize', keep_ratio=True),
+            dict(type='RandomFlip'),
+            dict(type='Normalize', **img_norm_cfg),
+            dict(type='Pad', size_divisor=32),
+            dict(type='DefaultFormatBundle'),
+            dict(type='Collect', keys=['img']),
+        ])
+]
+data = dict(
+    samples_per_gpu=2,
+    workers_per_gpu=2,
+    train=dict(
+        type=dataset_type,
+        ann_file=f'{data_root}/annotations/instances_train2017.json',
+        img_prefix=f'{data_root}/train2017/',
+        pipeline=train_pipeline),
+    val=dict(
+        type=dataset_type,
+        ann_file=f'{data_root}/annotations/instances_val2017.json',
+        img_prefix=f'{data_root}/val2017/',
+        pipeline=test_pipeline),
+    test=dict(
+        type=dataset_type,
+        ann_file=f'{data_root}/annotations/instances_val2017.json',
+        img_prefix=f'{data_root}/val2017/',
+        pipeline=test_pipeline))
+evaluation = dict(interval=1, metric='bbox')

external/hrnet_w48_coco_256x192.py

@@ -0,0 +1,169 @@
+_base_ = [
+    'default_runtime.py',
+    'coco.py'
+]
+evaluation = dict(interval=10, metric='mAP', save_best='AP')
+
+optimizer = dict(
+    type='Adam',
+    lr=5e-4,
+)
+optimizer_config = dict(grad_clip=None)
+# learning policy
+lr_config = dict(
+    policy='step',
+    warmup='linear',
+    warmup_iters=500,
+    warmup_ratio=0.001,
+    step=[170, 200])
+total_epochs = 210
+channel_cfg = dict(
+    num_output_channels=17,
+    dataset_joints=17,
+    dataset_channel=[
+        [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16],
+    ],
+    inference_channel=[
+        0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16
+    ])
+
+# model settings
+model = dict(
+    type='TopDown',
+    pretrained='https://download.openmmlab.com/mmpose/'
+    'pretrain_models/hrnet_w48-8ef0771d.pth',
+    backbone=dict(
+        type='HRNet',
+        in_channels=3,
+        extra=dict(
+            stage1=dict(
+                num_modules=1,
+                num_branches=1,
+                block='BOTTLENECK',
+                num_blocks=(4, ),
+                num_channels=(64, )),
+            stage2=dict(
+                num_modules=1,
+                num_branches=2,
+                block='BASIC',
+                num_blocks=(4, 4),
+                num_channels=(48, 96)),
+            stage3=dict(
+                num_modules=4,
+                num_branches=3,
+                block='BASIC',
+                num_blocks=(4, 4, 4),
+                num_channels=(48, 96, 192)),
+            stage4=dict(
+                num_modules=3,
+                num_branches=4,
+                block='BASIC',
+                num_blocks=(4, 4, 4, 4),
+                num_channels=(48, 96, 192, 384))),
+    ),
+    keypoint_head=dict(
+        type='TopdownHeatmapSimpleHead',
+        in_channels=48,
+        out_channels=channel_cfg['num_output_channels'],
+        num_deconv_layers=0,
+        extra=dict(final_conv_kernel=1, ),
+        loss_keypoint=dict(type='JointsMSELoss', use_target_weight=True)),
+    train_cfg=dict(),
+    test_cfg=dict(
+        flip_test=True,
+        post_process='default',
+        shift_heatmap=True,
+        modulate_kernel=11))
+
+data_cfg = dict(
+    image_size=[192, 256],
+    heatmap_size=[48, 64],
+    num_output_channels=channel_cfg['num_output_channels'],
+    num_joints=channel_cfg['dataset_joints'],
+    dataset_channel=channel_cfg['dataset_channel'],
+    inference_channel=channel_cfg['inference_channel'],
+    soft_nms=False,
+    nms_thr=1.0,
+    oks_thr=0.9,
+    vis_thr=0.2,
+    use_gt_bbox=False,
+    det_bbox_thr=0.0,
+    bbox_file='data/coco/person_detection_results/'
+    'COCO_val2017_detections_AP_H_56_person.json',
+)
+
+train_pipeline = [
+    dict(type='LoadImageFromFile'),
+    dict(type='TopDownGetBboxCenterScale', padding=1.25),
+    dict(type='TopDownRandomShiftBboxCenter', shift_factor=0.16, prob=0.3),
+    dict(type='TopDownRandomFlip', flip_prob=0.5),
+    dict(
+        type='TopDownHalfBodyTransform',
+        num_joints_half_body=8,
+        prob_half_body=0.3),
+    dict(
+        type='TopDownGetRandomScaleRotation', rot_factor=40, scale_factor=0.5),
+    dict(type='TopDownAffine'),
+    dict(type='ToTensor'),
+    dict(
+        type='NormalizeTensor',
+        mean=[0.485, 0.456, 0.406],
+        std=[0.229, 0.224, 0.225]),
+    dict(type='TopDownGenerateTarget', sigma=2),
+    dict(
+        type='Collect',
+        keys=['img', 'target', 'target_weight'],
+        meta_keys=[
+            'image_file', 'joints_3d', 'joints_3d_visible', 'center', 'scale',
+            'rotation', 'bbox_score', 'flip_pairs'
+        ]),
+]
+
+val_pipeline = [
+    dict(type='LoadImageFromFile'),
+    dict(type='TopDownGetBboxCenterScale', padding=1.25),
+    dict(type='TopDownAffine'),
+    dict(type='ToTensor'),
+    dict(
+        type='NormalizeTensor',
+        mean=[0.485, 0.456, 0.406],
+        std=[0.229, 0.224, 0.225]),
+    dict(
+        type='Collect',
+        keys=['img'],
+        meta_keys=[
+            'image_file', 'center', 'scale', 'rotation', 'bbox_score',
+            'flip_pairs'
+        ]),
+]
+
+test_pipeline = val_pipeline
+
+data_root = 'data/coco'
+data = dict(
+    samples_per_gpu=32,
+    workers_per_gpu=2,
+    val_dataloader=dict(samples_per_gpu=32),
+    test_dataloader=dict(samples_per_gpu=32),
+    train=dict(
+        type='TopDownCocoDataset',
+        ann_file=f'{data_root}/annotations/person_keypoints_train2017.json',
+        img_prefix=f'{data_root}/train2017/',
+        data_cfg=data_cfg,
+        pipeline=train_pipeline,
+        dataset_info={{_base_.dataset_info}}),
+    val=dict(
+        type='TopDownCocoDataset',
+        ann_file=f'{data_root}/annotations/person_keypoints_val2017.json',
+        img_prefix=f'{data_root}/val2017/',
+        data_cfg=data_cfg,
+        pipeline=val_pipeline,
+        dataset_info={{_base_.dataset_info}}),
+    test=dict(
+        type='TopDownCocoDataset',
+        ann_file=f'{data_root}/annotations/person_keypoints_val2017.json',
+        img_prefix=f'{data_root}/val2017/',
+        data_cfg=data_cfg,
+        pipeline=test_pipeline,
+        dataset_info={{_base_.dataset_info}}),
+)

faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:047c8118fc5ca88ba5ae1fab72f2cd6b070501fe3af2f3cba5cfa9a89b44b03e
+size 167287506

fileservice.py

@@ -0,0 +1,31 @@
+from fastapi import FastAPI, Request, Response
+
+filenames = ["js/poseTweek.js","js/geometry.js","js/poseAnimator.js", "js/limb.js"]
+contents = '\n'.join([f"<script type='text/javascript' src='{x}'></script>" for x in filenames])
+
+app = FastAPI()
+
+@app.middleware("http")
+async def insert_js(request: Request, call_next):
+    path = request.scope['path']  # get the request route
+    response = await call_next(request)
+    
+    if path == "/":
+        response_body = ""
+        async for chunk in response.body_iterator:
+            response_body += chunk.decode()
+
+        some_javascript = contents
+
+        response_body = response_body.replace("</head>", some_javascript + "</head>")
+
+        del response.headers["content-length"]
+
+        return Response(
+            content=response_body,
+            status_code=response.status_code, 
+            headers=dict(response.headers),
+            media_type=response.media_type
+        )
+
+    return response

hrnet_w48_coco_256x192-b9e0b3ab_20200708.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9e0b3ab0439cb68e166c7543e59d2587cd8d7e9acf5ea62a8378eeb82fb50e5
+size 255011654

js/geometry.js

@@ -0,0 +1,45 @@
+function dot2D(vectorA, vectorB) { return vectorA[0] * vectorB[0] + vectorA[1] * vectorB[1]; }
+function cross2D(vectorA, vectorB) { return vectorA[0] * vectorB[1] - vectorA[1] * vectorB[0]; }
+function magnitude2D(vector) { return Math.sqrt(dot2D(vector, vector)); }
+function perpendicular2D(v) { return [v[1], -v[0]]; }
+function reverse2D(v) { return [-v[0], -v[1]]; }
+function normalize2D(v) {
+    let length = Math.sqrt(dot2D(v, v));
+    if (length < 0.0001) { return [0.0, 0.0]; }
+    return [v[0] / length, v[1] / length];
+}
+function signedAngle(lhs, rhs) {return Math.atan2(cross2D(lhs, rhs), dot2D(lhs, rhs));}
+function angleBetween(lhs, rhs) {return Math.abs(signedAngle(lhs, rhs));}
+function positiveAngle(lhs, rhs) {
+    var a = signedAngle(lhs, rhs);
+    return a < 0 ? a + Math.PI * 2 : a;
+}
+
+function getSide(A, B, C, D) {
+    const BA = [A[0] - B[0], A[1] - B[1]];
+    const BD = [D[0] - B[0], D[1] - B[1]];
+    const BC = [C[0] - B[0], C[1] - B[1]];
+    const theta0 = positiveAngle(BA, BD);
+    const theta1 = positiveAngle(BA, BC);
+
+    if (theta0 == 0 || theta0 == theta1) {
+        return 0;
+    } else if (theta0 < theta1) {
+        return 1;
+    } else {
+        return -1
+    }
+}
+
+function slerp2D(ca, cb, t) {
+    const calen = magnitude2D(ca);
+    const cblen = magnitude2D(cb);
+    const lent = calen + (cblen - calen) * t;
+    const cq = [ca[0] * lent / calen, ca[1] * lent / calen];
+
+    const angle = signedAngle(ca, cb) * t;
+    const xt = cq[0] * Math.cos(angle) - cq[1] * Math.sin(angle);
+    const yt = cq[0] * Math.sin(angle) + cq[1] * Math.cos(angle);
+
+    return [xt, yt];
+}

js/layeredCanvas.js

@@ -0,0 +1,128 @@
+export class LayeredCanvas {
+    constructor(c) {
+        console.log("initializeLayeredCanvas");
+        this.canvas = c;
+        this.context = canvas.getContext('2d');
+
+        this.canvas.addEventListener('mousedown', this.handleMouseDown.bind(this));
+        this.canvas.addEventListener('mousemove', this.handleMouseMove.bind(this));
+        this.canvas.addEventListener('mouseup', this.handleMouseUp.bind(this));
+        this.canvas.addEventListener('mouseleave', this.handleMouseLeave.bind(this));
+
+        this.layers = [];
+    }
+
+    cleanup() {
+        this.canvas.removeEventListener('mousedown', this.handleMouseDown.bind(this));
+        this.canvas.removeEventListener('mousemove', this.handleMouseMove.bind(this));
+        this.canvas.removeEventListener('mouseup', this.handleMouseUp.bind(this));
+        this.canvas.removeEventListener('mouseleave', this.handleMouseLeave.bind(this));
+    }
+
+    getCanvasSize() {
+        return [this.canvas.width, this.canvas.height];
+    }
+    
+    getCanvasPosition(event) {
+        const rect = this.canvas.getBoundingClientRect();
+        const x = Math.floor(event.clientX - rect.left);
+        const y = Math.floor(event.clientY - rect.top);
+        return [x, y];
+    }
+    
+    handleMouseDown(event) {
+        const p = this.getCanvasPosition(event);
+      
+        for (let i = this.layers.length - 1; i >= 0; i--) {
+            const layer = this.layers[i];
+            if (layer.accepts(p)) {
+                layer.mouseDown(p);
+                this.draggingLayer = layer;
+                this.dragStart = p;
+                break;
+            }
+        }
+    }
+      
+    handleMouseMove(event) {
+        this.mouseCursor = this.getCanvasPosition(event);
+        if (this.draggingLayer) {
+            this.draggingLayer.mouseMove(this.getCanvasPosition(event)); // 念のため別の実体
+        }
+        this.render();
+    }
+    
+    handleMouseUp(event) {
+        if (this.draggingLayer) {
+            this.draggingLayer.mouseUp(this.getCanvasPosition(event));
+            this.draggingLayer = null;
+        }
+    }
+      
+    handleMouseLeave(event) {
+        this.mouseCursor = [-1,-1];
+        if (this.draggingLayer) {
+            this.handleMouseUp(event);
+        }
+    }
+      
+    render() {
+        for (let i = 0; i < this.layers.length; i++) {
+            const layer = this.layers[i];
+            layer.render(this.canvas, this.context);
+        }
+    }
+    
+    redraw() {
+        this.render();
+    }
+    
+    addLayer(layer) {
+        this.layers.push(layer);
+    }
+    
+}
+
+
+let mouseSequence = { // mixin
+    mouseDown(p) {
+        this.mouseHandler = this.mouse(p);
+    },
+    mouseMove(p) {
+        if (this.mouseHandler) {
+            this.mouseHandler.next(p);
+        }
+    },
+    mouseUp(p) {
+        if (this.mouseHandler) {
+            this.mouseHandler.next(null);
+            this.mouseHandler = null;
+        }
+    },
+/*
+    sample mouse handler
+    *mouse(p) {
+        while (p = yield) {
+            console.log("mouse", p);
+        }
+    }
+*/
+};
+
+export function sequentializeMouse(layerClass) {
+    layerClass.mouseDown = mouseSequence.mouseDown;
+    layerClass.mouseMove = mouseSequence.mouseMove;
+    layerClass.mouseUp = mouseSequence.mouseUp;
+}
+
+export class Layer {
+    constructor() {}
+
+    reserveRender() { renderReserved = true; }
+
+    accepts(point) { return false; }
+    mouseDown(point) {}
+    mouseMove(point) {}
+    mouseUp(point) {}
+    render(canvas, ctx) {}
+}

js/limb.js

@@ -0,0 +1,49 @@
+const limbSeq = [
+    [1, 2], [2, 3], [3, 4], // 右腕
+    [1, 5], [5, 6], [6, 7], // 左腕
+    [1, 8], [8, 9], [9, 10],  // 右胴→右脚
+    [1, 11], [11, 12], [12, 13], // 左胴→左脚
+    [1, 0], // 首
+    [0, 14], [14, 16], // 右目
+    [0, 15], [15, 17] // 左目
+  ];
+
+let parentNodeIndices = Array(18).fill(-1);
+let childNodeIndices = Array(18).fill(-1);
+let outEdgeIndices = Array(18).fill(-1); 
+let inEdgeIndices = Array(18).fill(-1); 
+
+function initializeLimb() {
+    limbSeq.forEach((limb, i) => {
+        parentNodeIndices[limb[1]] = limb[0];
+        childNodeIndices[limb[0]] = limb[1];
+        outEdgeIndices[limb[0]] = i;
+        inEdgeIndices[limb[1]] = i;
+    });
+}
+  
+function findParentNodeIndex(nodeIndex) {
+    return parentNodeIndices[nodeIndex];
+}
+  
+function findChildNodeIndex(nodeIndex) {
+    return childNodeIndices[nodeIndex];
+}
+  
+function findOutEdgeIndex(nodeIndex) {
+    return outEdgeIndices[nodeIndex];
+}
+  
+function findInEdgeIndex(nodeIndex) {
+    return inEdgeIndices[nodeIndex];
+}
+  
+function findNextEdgeIndex(edgeIndex) {
+    return findOutEdgeIndex(limbSeq[edgeIndex][1]);
+}
+  
+function findPrevEdgeIndex(edgeIndex) {
+    return findInEdgeIndex(limbSeq[edgeIndex][0]);
+}
+  
+initializeLimb();

js/poseAnimator.js

@@ -0,0 +1,379 @@
+const monitor = [];
+function eachMonitor(p, f) {
+  monitor.forEach((m) => {
+    if (m[0] === p[0] && m[1] === p[1]) {
+      f(m);
+    }
+  });
+}
+function isMonitored(p) {
+  let result = false;
+  eachMonitor(p, (m) => {
+    result = true;
+  });
+  return result;
+}
+
+
+// 2D projection
+function worldToLocal(pointA, pointB, pointC) {
+  const vectorAB = [pointB[0] - pointA[0], pointB[1] - pointA[1]];
+  const vectorAC = [pointC[0] - pointA[0], pointC[1] - pointA[1]];
+  const dot1 = dot2D(vectorAB, vectorAC);
+  const dot2 = dot2D(vectorAB, vectorAB);
+  const localX = dot1 / dot2;
+  const pointD = [pointA[0] + localX * vectorAB[0], pointA[1] + localX * vectorAB[1]];
+
+  // localY = distance from pointC to pointD
+  const vectorCD = [pointD[0] - pointC[0], pointD[1] - pointC[1]];
+  let localY = Math.sqrt(dot2D(vectorCD, vectorCD));
+
+  // if pointC is on the right side of vectorAB, localY is negative
+  const cross = cross2D(vectorAB, vectorAC);
+  if (0 < cross) {
+      localY = -localY;
+  }
+
+  return [localX, localY];
+}
+
+function distPointFromSeg(pointA, pointB, pointC, lx, ly) {
+  if (lx < 0) {
+    const vectorAC = [pointC[0] - pointA[0], pointC[1] - pointA[1]];
+    return Math.sqrt(dot2D(vectorAC, vectorAC));
+  }
+  if (1.0 < lx) {
+    const vectorBC = [pointC[0] - pointB[0], pointC[1] - pointB[1]];
+    return Math.sqrt(dot2D(vectorBC, vectorBC));
+  }
+  return Math.abs(ly);
+}
+
+function localToWorld([pointA, pointB], pointC) { // C is local coordinate
+  const vectorAB = [pointB[0] - pointA[0], pointB[1] - pointA[1]];
+  // pointD = pointA + C.x * vectorAB(垂線との交点)
+  const pointD = [pointA[0] + pointC[0] * vectorAB[0], pointA[1] + pointC[0] * vectorAB[1]];
+  const v = perpendicular2D(vectorAB);
+  const vLength = Math.sqrt(dot2D(v, v));
+  if (vLength < 0.0001) { return pointD; }
+
+  const newLength = pointC[1];
+  const factor = newLength / vLength;
+  return [pointD[0] + factor * v[0], pointD[1] + factor * v[1]];
+}
+
+function makeEdgeSegments(pose) {
+  return limbSeq.map((segment) => {
+    const p0 = pose[segment[0]];
+    const p1 = pose[segment[1]];
+    return [p0, p1];
+  });
+}
+
+function makeNormalizeEdgeSegments(edges) {
+  return edges.map((edge) => normalize2D([edge[1][0] - edge[0][0], edge[1][1] - edge[0][1]]));
+}
+
+const fieldDepth = 5;
+const FO0i = 0;
+const FO0t = 1;
+const FO0u = 2;
+const FO0d = 3;
+
+// c current
+// p prev
+// n next
+// 0 primary
+// 1 secondary
+// e edge
+// s segment
+// i in
+// o out
+// t time
+// l local
+// q point
+// m main
+// d distance
+// w weight
+
+function calcJointRange(
+  edgeSegments, normalizedEdgeSegments, 
+  edgeSegmentsForSideDecision,
+  ep, ec, m) {
+  // parent, child
+  const np = normalizedEdgeSegments[ep];
+  const nc = normalizedEdgeSegments[ec];
+  let pnp = perpendicular2D(np);
+  let pnc = perpendicular2D(nc);
+  let side = getSide(edgeSegmentsForSideDecision[ep][0], edgeSegmentsForSideDecision[ep][1], edgeSegmentsForSideDecision[ec][1], m);
+  if (side < 0) {
+    pnp = reverse2D(pnp);
+    pnc = reverse2D(pnc);
+  }
+  return [pnp, pnc];
+}
+
+function calcJointRangeWithOrder(oldEdgeSegments, oldNormalizedEdgeSegments, newEdgeSegments, e0, m, order) {
+  if (order < 0) {
+    const e1 = findPrevEdgeIndex(e0);
+    if (e1 < 0) { return null; }
+    const [pn1, pn0] = calcJointRange(oldEdgeSegments, oldNormalizedEdgeSegments, newEdgeSegments,e1, e0, m);
+    const oldPivot = oldEdgeSegments[e0][0];
+    const newPivot = newEdgeSegments[e0][0];
+    return [pn0, pn1, oldPivot, newPivot, e1];
+  } else if (0 < order) {
+    const e1 = findNextEdgeIndex(e0);
+    if (e1 < 0) { return null; }
+    const [pn0, pn1] = calcJointRange(oldEdgeSegments, oldNormalizedEdgeSegments, newEdgeSegments, e0, e1, m);
+    const oldPivot = oldEdgeSegments[e0][1];
+    const newPivot = newEdgeSegments[e0][1];
+    return [pn0, pn1, oldPivot, newPivot, e1];
+  } else {
+    return null;
+  }
+}
+
+function buildWeightMap(size, pose) {
+  const threshold = 512;
+
+  const [w, h] = size;
+
+  // この状態で、canvasに対して、  
+  // 同じ大きさの3次元配列[x,y,z]を作成する
+  // x,yは画像の2次元座標
+
+  // 各Pixelに対して、
+  // そのPixelに最も近いエッジのindexを[0]に、
+  // そのときのエッジ座標系におけるtを[1]に記録する
+  // どの骨格とも近くないものは[0]=255とする
+
+  const field = new Float32Array(w * h * fieldDepth)
+  function fieldIndex(x, y) {
+    return (y * w + x) * fieldDepth;
+  }
+  
+  const edgeSegments = makeEdgeSegments(pose);
+  const normalizedEdgeSegments = makeNormalizeEdgeSegments(edgeSegments);
+
+  // 各Pixelに対して、一番近いエッジを探す
+  // またそのエッジ座標系におけるtを保存する
+  for (let y = 0 ; y < h ; y++) {
+    for (let x = 0 ; x < w ; x++) {
+      const m = [x, y];
+      let minDist = threshold;
+      const fidx = fieldIndex(x, y);
+      field[fidx+FO0i] = -1;
+      for (let i = 0 ; i < limbSeq.length ; i++) {
+        // 線分から���への距離を計算
+        const s = edgeSegments[i];
+        const lq = worldToLocal(s[0], s[1], m);
+        const dist = distPointFromSeg(s[0], s[1], m, lq[0], lq[1]);
+        if (dist < minDist) {
+          minDist = dist;
+          field[fidx+FO0i] = i;
+          field[fidx+FO0t] = lq[0];
+          field[fidx+FO0u] = lq[1];
+          field[fidx+FO0d] = dist;
+        }
+      }
+    }
+  }
+
+  return field;
+}
+
+function renderField(canvas2, size, field) {
+  function blend(c0, w0, c1, w1) {
+    const r = (c0[0] * w0 + c1[0] * w1);
+    const g = (c0[1] * w0 + c1[1] * w1);
+    const b = (c0[2] * w0 + c1[2] * w1);
+    const a = (c0[3] * w0 + c1[3] * w1);
+    return [r, g, b, a];
+  }
+
+  const [w, h] = size;
+  function fieldIndex(x, y) {
+    return (y * w + x) * fieldDepth;
+  }
+
+  canvas2.width = w;
+  canvas2.height = h;
+  const ctx2 = canvas2.getContext('2d');
+
+  bindings = ctx2.createImageData(w, h);
+  const data = bindings.data;
+  for (let y = 0 ; y < h ; y++) {
+    for (let x = 0 ; x < w ; x++) {
+      const fidx = fieldIndex(x, y);
+      const oi = (y * w + x) * 4;
+
+      let c = [0, 0, 0, 64];
+
+      const e0 = field[fidx+FO0i];
+      if (0 <= e0) { 
+        c = colors[e0];
+        c[3] = 255;
+        // c[3] = 255 - field[fidx+FO0d] * 4;
+      }
+      data[oi + 0] = c[0];
+      data[oi + 1] = c[1];
+      data[oi + 2] = c[2];
+      data[oi + 3] = c[3];
+    }
+  }
+  ctx2.putImageData(bindings, 0, 0);
+
+  return field;
+}
+
+function makeImageDataFromPicture(canvas3, size, picture) {
+  const [w, h] = size;
+  canvas3.width = w;
+  canvas3.height = h;
+
+  const ctx3 = canvas3.getContext("2d");
+  // ctx3.drawImage(picture, sampleOffset[0], sampleOffset[1]);
+  ctx3.drawImage(picture, 0, 0);
+
+  return new ImageData(new Uint8ClampedArray(ctx3.getImageData(0, 0, w, h).data), w, h);
+}
+
+function animatePicture(canvas4, size, oldPose, newPose, srcImageData, initialPoseField) {
+  const srcData = srcImageData.data;
+
+  const [w, h] = size;
+
+  const ctx4 = canvas4.getContext("2d");
+  canvas4.width = w;
+  canvas4.height = h;
+  const dstCtx = ctx4;
+  const dstImageData = dstCtx.createImageData(w, h);
+  const dstData = dstImageData.data;
+
+  const field = buildWeightMap(size, newPose);
+  const canvas2 = document.getElementById("canvas2");
+  // renderField(canvas2, size, field);
+  drawBodyPoseTo(canvas2.getContext("2d"), [newPose]);
+
+  function fieldIndex(x, y) {
+    return (y * w + x) * fieldDepth;
+  }
+
+  const oldEdgeSegments = makeEdgeSegments(oldPose);
+  const oldNormalizedEdgeSegments = makeNormalizeEdgeSegments(oldEdgeSegments);
+  const newEdgeSegments = makeEdgeSegments(newPose);
+
+  function oldEdgePoint(edgeIndex, p) {
+    const v = oldEdgeSegments[edgeIndex];
+    return localToWorld(v, p);
+  }
+
+  function newEdgePoint(edgeIndex, p) {
+    const v = newEdgeSegments[edgeIndex];
+    return localToWorld(v, p);
+  }
+
+  debugLines = [];
+
+  for (let y = 0 ; y < h ; y++) {
+    for (let x = 0 ; x < w ; x++) {
+      // 各点に対して、fieldから近隣エッジ座標系でのローカル座標を取得する
+      const oi = (y * w + x) * 4;
+
+      const fidx = fieldIndex(x, y);
+      const e0 = field[fidx+FO0i];
+      if (e0 < 0) { 
+        dstData[oi + 0] = 0;
+        dstData[oi + 1] = 0;
+        dstData[oi + 2] = 0;
+        dstData[oi + 3] = 255;
+        continue; 
+      }
+
+      const lq = [field[fidx+FO0t], field[fidx+FO0u]];
+      let e1 = -1;
+      let order = 0; // 1 == forward, -1 == backward
+      if (lq[0] < 0.5) {
+        e1 = findPrevEdgeIndex(e0);
+      } else if (0.5 <= lq[0]) {
+        e1 = findNextEdgeIndex(e0);
+      }
+      if (0 <= e1) {
+        if (lq[0] < 0) {
+          order = -1;
+        } else if (1 < lq[0]) {
+          order = 1;
+        }
+      }
+
+      let iq = oldEdgePoint(e0, lq);
+
+      if (0 != order) {
+        // サイド判定は新ポーズで行う
+        const mNew = [x,y];
+        const jointRange = calcJointRangeWithOrder(
+          oldEdgeSegments, oldNormalizedEdgeSegments, 
+          newEdgeSegments,
+          e0, mNew, order);
+        const [pn0, pn1, oldPivot, newPivot, _] = jointRange;
+
+        const pivotMNew = [x - newPivot[0], y - newPivot[1]];
+        const [w0, w1]= [angleBetween(pn0, pivotMNew), angleBetween(pivotMNew, pn1)];
+
+        let d0 = field[fidx+FO0d];
+        let mt = slerp2D(pn0, pn1, w0 / (w0 + w1));
+        iq = [oldPivot[0] + mt[0] * d0, oldPivot[1] + mt[1] * d0];
+        eachMonitor([x,y], () => {
+          console.log('animate', x, y, ix, iy, e1, w0, w1, pn0, pn1);
+          debugLines.push([oldPivot, pn0]);
+          debugLines.push([oldPivot, pn1]);
+        });
+      }
+
+      iq = [Math.round(iq[0]), Math.round(iq[1])];
+
+      var initialOwner = initialPoseField[fieldIndex(iq[0], iq[1]) + FO0i];
+      if (0 <= initialOwner && initialOwner != e0 && initialOwner != e1) {
+        // ほかの領土
+        dstData[oi + 0] = 0;
+        dstData[oi + 1] = 0;
+        dstData[oi + 2] = 0;
+        dstData[oi + 3] = 0;
+      } else {
+        // canvas3の[ix,iy]の色を取得して、canvas4の[x,y]に描画する
+        const si = (iq[1] * w + iq[0]) * 4;
+        dstData[oi + 0] = srcData[si + 0];
+        dstData[oi + 1] = srcData[si + 1];
+        dstData[oi + 2] = srcData[si + 2];
+        dstData[oi + 3] = srcData[si + 3];
+      }
+    }
+  }
+
+  dstCtx.putImageData(dstImageData, 0, 0);
+}
+
+function handleMicroscopeMouseMove(e) {
+  const [x,y] = mouseCursor;
+
+  const imageData = ctx.getImageData(x - 2, y - 2, 5, 5);
+  const data = imageData.data;
+
+  const microscope = document.getElementById('microscope');
+  const ctx2 = microscope.getContext('2d');
+  ctx2.clearRect(0, 0, microscope.width, microscope.height);
+  for (let i = 0 ; i < 5 ; i++) {
+    for (let j = 0 ; j < 5 ; j++) {
+      const idx = (i * 5 + j) * 4;
+      ctx2.fillStyle = `rgba(${data[idx]}, ${data[idx+1]}, ${data[idx+2]}, ${data[idx+3]})`;
+      ctx2.fillRect(j * 10, i * 10, 10, 10);
+    }
+  }
+}
+
+function initMicroscope() {
+  // canvas.addEventListener('mousemove', handleMicroscopeMouseMove);
+  // poseData[0][6] = [329, 148];
+  // poseData[0][7] = [329, 194];
+}
+

js/poseTweek.js

@@ -0,0 +1,862 @@
+var canvas = null;
+var ctx = null;
+var offscreen = null;
+var initialPoseField = null;
+var canvasBg = null;
+var debugLines = [];
+
+const wheelDisplayTime = 500;
+
+const colors = [[255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0], [170, 255, 0], [85, 255, 0], [0, 255, 0],
+[0, 255, 85], [0, 255, 170], [0, 255, 255], [0, 170, 255], [0, 85, 255], [0, 0, 255], [85, 0, 255],
+[170, 0, 255], [255, 0, 255], [255, 0, 170], [255, 0, 85]];
+
+function cutOffLimb(pose, cutOffIndex) {
+  console.log(`cutOffLimb: ${cutOffIndex}`);
+  // 末端ノードの座標を削除する
+  var newPose = deepCopy(pose);
+  for (let i = 0; i < 18; i++) {
+    if (newPose[i] == null) {continue;}
+    // ルートまで検索し、その間にcuttOffIndexがあれば削除
+    var curr = i;
+    while (curr !== 1) {
+      console.log(`checking: ${i} -> ${curr}`);
+      let parent = findParentNodeIndex(curr);
+      if (parent === cutOffIndex) {
+        console.log(`cutOffLimb: ${i} -> ${cutOffIndex}`);
+        newPose[i] = null;
+        break;
+      }
+      curr = parent;
+    }
+  }
+  return newPose;
+}
+
+function repairPose(sourcePose) {
+  // TODO: ループには対応してないかも
+  var pose = sourcePose;
+  var newPose = new Array(18)
+  for (var k = 0; k < 3; k++) {
+    var processed = 0; // イテレーション用
+    for (let i = 0; i < 18; i++) {
+      if (pose[i] == null) {
+        let parent = findParentNodeIndex(i);
+        if (parent === -1) {continue;} // あり得ない
+        if (pose[parent] == null) {
+          console.log(`repair failed(A): ${i} -> parent loss`);
+          continue;
+        }
+
+        // サンプルデータから引っ張ってくる
+        var v = sampleCandidateSource[i].map((x, j) => x - sampleCandidateSource[parent][j]);
+        newPose[i] = pose[parent].map((x, j) => x + v[j]);
+        console.log(`repaired: ${i} -> ${newPose[newPose.length - 1]}`);
+        processed++;
+      } else {
+        newPose[i] = pose[i].map(x => x);
+      }
+    }
+    if (processed === 0) {break;}
+    pose = newPose;
+  }
+  return newPose;
+}
+
+function deepCopy(arr) {
+  return JSON.parse(JSON.stringify(arr));
+}
+
+function distSq(p0, p1) {
+  return (p0[0] - p1[0]) ** 2 + (p0[1] - p1[1]) ** 2;
+}
+
+// poseDataの形式：[[[x1, y1], [x2, y2], ...],[[x3, y3], [x4, y4], ...], ...]
+// 各要素が人間
+// 人間の各要素が関節
+
+function poseDataToCandidateAndSubset(poseData) {
+  let candidate = [];
+  let subset = [];
+  for (let i = 0; i < poseData.length; i++) {
+    let person = poseData[i];
+    let subsetElement = [];
+    for (let j = 0; j < person.length; j++) {
+      candidate.push(person[j]);
+      subsetElement.push(candidate.length - 1);
+    }
+    subset.push(subsetElement);
+  }
+  return [candidate, subset];
+}
+
+// サンプルデータ  
+const sampleOffset = [0, -70];
+const sampleCandidateSource = [[235, 158],[234, 220],[193, 222],[138, 263],[89, 308],[276, 220],[325, 264],[375, 309],[207, 347],[203, 433],[199, 523],[261, 347],[262, 430],[261, 522],[227, 148],[245, 148],[208, 158],[258, 154]].map((p) => [p[0] + sampleOffset[0], p[1] + sampleOffset[1]]);
+const sampleSubsetElementSource = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17];
+
+// const sampleCandidateSource = [[618.00, 0.00], [618.00, 44.00], [304.00, 81.00], [482.00, 96.00], [66.00, 270.00], [171.00, 280.00], [618.00, 82.00], [307.00, 112.00], [460.00, 143.00], [0.00, 301.00], [65.00, 301.00], [172.00, 303.00], [584.00, 86.00], [275.00, 119.00], [420.00, 139.00], [0.00, 301.00], [41.00, 301.00], [144.00, 303.00], [544.00, 131.00], [348.00, 139.00], [262.00, 160.00], [0.00, 337.00], [52.00, 339.00], [130.00, 348.00], [570.00, 175.00], [283.00, 177.00], [78.00, 338.00], [172.00, 380.00], [651.00, 78.00], [338.00, 111.00], [505.00, 144.00], [92.00, 301.00], [198.00, 305.00], [661.00, 132.00], [349.00, 156.00], [541.00, 179.00], [106.00, 336.00], [203.00, 348.00], [305.00, 159.00], [665.00, 160.00], [563.00, 192.00], [80.00, 343.00], [181.00, 385.00], [614.00, 205.00], [291.00, 220.00], [432.00, 320.00], [152.00, 372.00], [43.00, 380.00], [0.00, 386.00], [623.00, 281.00], [306.00, 290.00], [92.00, 357.00], [509.00, 434.00], [304.00, 357.00], [622.00, 368.00], [47.00, 394.00], [0.00, 395.00], [142.00, 405.00], [535.00, 565.00], [655.00, 200.00], [337.00, 217.00], [467.00, 322.00], [191.00, 372.00], [83.00, 375.00], [344.00, 282.00], [655.00, 282.00], [103.00, 343.00], [237.00, 368.00], [22.00, 377.00], [0.00, 379.00], [460.00, 459.00], [305.00, 352.00], [638.00, 355.00], [0.00, 401.00], [110.00, 412.00], [411.00, 570.00], [608.00, 0.00], [608.00, 40.00], [297.00, 75.00], [469.00, 84.00], [0.00, 261.00], [58.00, 263.00], [165.00, 275.00], [625.00, 0.00], [625.00, 39.00], [309.00, 74.00], [486.00, 83.00], [71.00, 264.00], [180.00, 276.00], [599.00, 0.00], [599.00, 44.00], [284.00, 80.00], [440.00, 93.00], [48.00, 271.00], [0.00, 272.00], [157.00, 277.00], [634.00, 0.00], [633.00, 41.00], [319.00, 77.00], [79.00, 269.00], [190.00, 277.00]];
+// const sampleSubsetElementSource = [1.00,6.00,12.00,18.00,24.00,28.00,33.00,39.00,43.00,49.00,54.00,59.00,65.00,72.00,77.00,84.00,90.00,97.00,32.98,18.00],[5.00,11.00,17.00,23.00,27.00,32.00,37.00,42.00,46.00,-1.00,-1.00,62.00,67.00,-1.00,82.00,88.00,95.00,100.00,25.45,15.00],[4.00,10.00,16.00,22.00,26.00,31.00,36.00,41.00,47.00,51.00,57.00,63.00,66.00,74.00,81.00,87.00,93.00,99.00,26.97,18.00],[3.00,8.00,14.00,19.00,25.00,30.00,35.00,40.00,45.00,52.00,58.00,61.00,70.00,75.00,79.00,86.00,92.00,-1.00,30.45,17.00],[2.00,7.00,13.00,20.00,-1.00,29.00,34.00,38.00,44.00,50.00,53.00,60.00,64.00,71.00,78.00,85.00,91.00,98.00,27.89,17.00],[0.00,-1.00,-1.00,-1.00,-1.00,-1.00,-1.00,-1.00,-1.00,-1.00,-1.00,-1.00,-1.00,-1.00,76.00,83.00,-1.00,96.00,3.33,4.00];
+
+function makePoseFromCandidateAndSubsetElement(candidate, subsetElement) {
+  var pose = [];
+  for (let j = 0 ; j < 18; j++) {
+    let i = subsetElement[j];
+    pose.push(i < 0 || candidate[i] == null ? null : candidate[i].map((x)=>x));
+  }
+  return pose;
+}
+
+function makePoseDataFromCandidateAndSubset(candidate, subset) {
+  return subset.map(subsetElement => makePoseFromCandidateAndSubsetElement(candidate, subsetElement));
+}
+
+function addPerson() {
+  var dx = Math.random() * 100;
+  var dy = Math.random() * 100;
+
+  poseData.push(
+    makePoseFromCandidateAndSubsetElement(
+      sampleCandidateSource.map(point => [point[0] + dx, point[1] + dy]),
+      sampleSubsetElementSource));
+
+  addHistory();    
+  Redraw();
+}
+
+function removePerson(personIndex) {
+  poseData.splice(personIndex, 1);
+  addHistory();
+  Redraw();
+}
+
+function repairPerson(personIndex) {
+  poseData[personIndex] = repairPose(poseData[personIndex]);
+  addHistory();
+  Redraw();
+}
+
+function cutOffPersonLimb(personIndex, limbIndex) {
+  poseData[personIndex] = cutOffLimb(poseData[personIndex], limbIndex);
+  console.log(poseData[personIndex]);
+  console.log(poseData);
+  addHistory();
+  Redraw();
+}
+
+// ドラッグ中の各キーが押されているかどうかのフラグ
+var keyDownFlags = {};
+// マウスカーソル
+var mouseCursor = [-1, -1];
+
+function cross(lhs, rhs) {return lhs[0] * rhs[1] - lhs[1] * rhs[0];}
+function dot(lhs, rhs) {return lhs[0] * rhs[0] + lhs[1] * rhs[1];}
+function directedAngleTo(lhs, rhs) {return Math.atan2(cross(lhs, rhs), dot(lhs, rhs));}
+
+function isMouseOnCanvas() {
+  // mouseCursorがcanvasの範囲内にあるかどうかを判定
+  var rect = canvas.getBoundingClientRect();
+  var f = 0 <= mouseCursor[0] && mouseCursor[0] <= rect.width && 0 <= mouseCursor[1] && mouseCursor[1] <= rect.height;
+  return f;
+}
+
+function clearCanvas() {
+  var w = canvas.width;
+  var h = canvas.height;
+  ctx.fillStyle = 'black';
+  ctx.fillRect(0, 0, w, h);
+}
+
+function drawBackground() {
+  if (canvasBg != null) {
+    ctx.drawImage(canvasBg, 0, 0);
+  }
+}
+
+function resizeCanvas(width, height) {
+  canvas.width = width ? width : canvas.width;
+  canvas.height = height ? height : canvas.height;
+  Redraw();
+}
+
+function calculateCenter(shape) {
+  var center = shape.reduce(function(acc, point) {
+    if (point === null) {
+      acc[0] += point[0];
+      acc[1] += point[1];
+    }
+    return acc;
+  }, [0, 0]);
+  center[0] /= shape.length;
+  center[1] /= shape.length;
+  return center;
+}
+
+// v2d -> v3d
+function rotateX(vector, angle) {
+  var x = vector[0];
+  var y = vector[1];
+  var z = 0;
+
+  // X軸に対して回転する
+  var x1 = x;
+  var y1 = y * Math.cos(angle) - z * Math.sin(angle);
+  var z1 = y * Math.sin(angle) + z * Math.cos(angle);
+
+  return [x1, y1, z1];
+}
+
+// v2d -> v3d
+function rotateY(vector, angle) {
+  var x = vector[0];
+  var y = vector[1];
+  var z = 0;
+
+  // Y軸に対して回転する
+  var x1 = x * Math.cos(angle) + z * Math.sin(angle);
+  var y1 = y;
+  var z1 = -x * Math.sin(angle) + z * Math.cos(angle);
+
+  return [x1, y1, z1];  
+}
+
+// v3d -> v2d
+function perspectiveProjection(vector, cameraDistance) {
+  var x = vector[0];
+  var y = vector[1];
+  var z = vector[2];
+
+  if (z === 0) {
+    return [x, y];
+  }
+
+  var scale = cameraDistance / (cameraDistance - z);
+  var x1 = x * scale;
+  var y1 = y * scale;
+
+  return [x1, y1];
+}  
+
+// v2d -> v3d
+function rotateAndProject(f, p, c, angle) {
+  var v = [p[0] - c[0], p[1] - c[1]];
+  var v1 = f(v, angle);
+  var v2 = perspectiveProjection(v1, 500);
+  return [v2[0] + c[0], v2[1] + c[1]];
+}
+
+function drawPicture() {
+  ctx.globalAlpha = 1.0;
+  // if (picture != null) {
+    //var [w, h] = [canvas.width, canvas.height];
+    //var [pw, ph] = [picture.width, picture.height];
+    //var [x, y] = [(w-pw)/2, (h-ph)/2];
+    // ctx.drawImage(picture, sampleOffset[0], sampleOffset[1]);
+    // ctx.drawImage(picture, 0, 0);
+  // }
+
+  ctx.globalAlpha = 1.0;
+  if (offscreen != null) {
+    ctx.drawImage(offscreen, 0, 0);
+  }
+}
+
+function drawBodyPose() {
+  drawBodyPoseTo(ctx, poseData);
+}
+
+function drawBodyPoseTo(ctx, poseData) {
+  let stickWidth = 4;
+  let imageSize = Math.min(canvas.width, canvas.height);
+  stickWidth *= imageSize / 512;
+
+  ctx.globalAlpha = 0.6;
+
+  // edge
+  for (let i = 0; i < poseData.length; i++) {
+    const pose = poseData[i];
+
+    for (let j = 0; j < 17; j++) {
+      const p = pose[limbSeq[j][0]];
+      const q = pose[limbSeq[j][1]];
+      if (p == null || q == null) continue;
+      const [X0, Y0] = p;
+      const [X1, Y1] = q;
+      let angle = Math.atan2(Y1 - Y0, X1 - X0);
+      let magnitude = ((X0 - X1) ** 2 + (Y0 - Y1) ** 2) ** 0.5
+      let polygon = new Path2D();
+      polygon.ellipse((X0+X1)/2, (Y0+Y1)/2, magnitude / 2, stickWidth, angle, 0, 2 * Math.PI);
+      //ctx.fillStyle = `rgb(${colors[j].join(',')})`;
+      ctx.fillStyle = 'gray';
+      ctx.fill(polygon);
+    }
+  }
+
+  ctx.globalAlpha = 1.0;
+
+  // node
+  for (let i = 0; i < poseData.length; i++) {
+    const pose = poseData[i];
+
+    ctx.font = '12px serif';
+    for (let j = 0; j < 18; j++) {
+      const p = pose[j];
+      if (p == null) continue;
+      const [x, y] = p;
+      ctx.beginPath();
+      ctx.arc(x, y, stickWidth, 0, 2 * Math.PI);
+      ctx.fillStyle = `rgb(${colors[j].join(',')})`;
+      ctx.fill();
+      // ctx.fillStyle = 'rgb(255,255,255)'
+      // ctx.fillText(j, x-3, y+4);
+    }
+  }
+}
+
+let lastWheeling = 0;
+
+function drawUI() {
+  if (keyDownFlags['Space'] || keyDownFlags['BracketLeft'] || keyDownFlags['BracketRight'] || 
+      new Date().getTime() - lastWheeling < wheelDisplayTime) {
+    ctx.beginPath();
+    ctx.lineWidth=4;
+    ctx.arc(mouseCursor[0], mouseCursor[1], dragRange, 0, 2 * Math.PI);
+    ctx.strokeStyle = 'rgb(255,255,255)';
+    ctx.stroke();
+  }
+
+  if (isDragging && (dragMode == "rotate" || dragMode == "rotate2")) {
+    ctx.beginPath();
+    ctx.lineWidth=1;
+    ctx.strokeStyle = 'rgb(255,255,255)';
+    ctx.moveTo(dragStart[0], dragStart[1]);
+    ctx.lineTo(dragStart[0]+rotateBaseVector[0], dragStart[1]+rotateBaseVector[1]);
+    ctx.stroke();
+  }
+
+  ctx.globalAlpha = 0.1;
+
+  for (let i = 0 ; i < debugLines.length ; i++) {
+    const p = debugLines[i];
+    const l = 50;
+    ctx.beginPath();
+    ctx.lineWidth= 2;
+    ctx.strokeStyle = `rgb(${colors[i].join(',')})`;
+    ctx.moveTo(p[0][0], p[0][1]);
+    ctx.lineTo(p[0][0]+p[1][0] * l, p[0][1]+p[1][1] * l);
+    ctx.stroke();
+  }
+  ctx.globalAlpha = 1.0;
+
+  let operationTextFlags = {
+    "Space": "Range Move",
+    "AltLeft": "Body Move",
+    "AltRight": "Body Move",
+    "ControlLeft": "Scale",
+    "ControlRight": "Scale",
+    "ShiftLeft": "Rotate",
+    "ShiftRight": "Rotate",
+    "KeyQ": "CutOff",
+    "KeyD": "Delete",
+    "KeyX": "X-Axis",
+    "KeyC": "Y-Axis",
+    "KeyR": "Repair",
+  }
+
+  // operationTextFlagsに含まれるものがkeyDownFlagsに含まれるばあい、そのキーの文字列を取得
+  let activeOperations = Object.keys(operationTextFlags).filter(key => keyDownFlags[key]);
+  if (activeOperations.length > 0) {
+    // 左上に表示
+    ctx.font = '20px serif';
+    ctx.fillStyle = 'rgb(255,255,255)';
+    ctx.fillText(operationTextFlags[activeOperations[0]], 10, 30);
+  }
+
+  // draw mouse cursor position at right bottom
+  ctx.font = '20px serif';
+  ctx.fillStyle = 'rgb(255,255,255)';
+  ctx.fillText(`(${mouseCursor[0]}, ${mouseCursor[1]})`, canvas.width-200, canvas.height-10);
+
+}
+
+function Redraw() {
+  clearCanvas();
+  drawBackground();
+  drawPicture();
+  drawBodyPose();
+  drawUI();
+}
+
+function getNearestNode(p) {
+    let minDistSq = Infinity;
+    let personIndex = -1;
+    let nodeIndex = -1;
+    for (let i = 0; i < poseData.length; i++) {
+      const pose = poseData[i];
+      for (let j = 0; j < pose.length; j++) {
+        const q = pose[j];
+        if (q == null) continue;
+        const d = distSq(p, q);
+        if (d < minDistSq) {
+          minDistSq = d;
+          personIndex = i;
+          nodeIndex = j;
+        }
+      }
+    }
+    return [personIndex, nodeIndex, Math.sqrt(minDistSq)];
+}
+
+let dragRange = 64;
+let dragRangeDelta = 16;
+
+// ドラッグ中に座標を保持するための変数
+let isDragging = false;
+let dragStart = [0, 0];
+let dragPersonIndex = -1;
+let dragMarks = [];
+let dragMode = "";
+let rotateBaseVector = null;
+let history = [];
+let historyIndex = 0;
+
+function clearHistory() {
+  history = [];
+  historyIndex = 0;  
+}
+
+function addHistory() {
+  history = history.slice(0, historyIndex);
+  history.push(JSON.parse(JSON.stringify(poseData)));
+  historyIndex = history.length;
+}
+
+function undo() {
+  if (1 < historyIndex) {
+    historyIndex--;
+    poseData = deepCopy(history[historyIndex-1]);
+    Redraw();
+  }   
+}
+
+function redo() {
+  if (historyIndex < history.length) {
+    historyIndex++;
+    poseData = deepCopy(history[historyIndex-1]);
+    Redraw();
+  }   
+}
+
+function fetchLatestPoseData() {
+  return history[historyIndex-1];
+}
+
+function getCanvasPosition(event) {
+  const rect = canvas.getBoundingClientRect();
+  const x = Math.floor(event.clientX - rect.left);
+  const y = Math.floor(event.clientY - rect.top);
+  return [x, y];
+}
+
+function forEachMarkedNodes(fn) {
+  for (let i = 0; i < dragMarks.length; i++) {
+    for (let j = 0; j < dragMarks[i].length; j++) {
+      if (dragMarks[i][j]) {
+        fn(i, j, poseData[i][j]);
+      }
+    }
+  }
+}
+
+// Canvas要素上でマウスが押された場合に呼び出される関数
+function handleMouseDown(event) {
+  const p = getCanvasPosition(event);
+  const [personIndex, nodeIndex, minDist] = getNearestNode(p);
+
+  if (keyDownFlags["KeyD"]) {removePerson(personIndex);return;}
+  if (keyDownFlags["KeyR"]) {repairPerson(personIndex);return;}
+
+  if (keyDownFlags["KeyQ"] && minDist < 16) {
+    console.log("pressed KeyQ");
+    cutOffPersonLimb(personIndex, nodeIndex);
+    return;
+  }
+
+  // ドラッグ処理の開始
+  dragStart = p;
+  dragMarks = poseData.map(pose => pose.map(node => false));
+
+  if (event.altKey || event.ctrlKey || event.shiftKey ||
+      keyDownFlags["KeyX"] || keyDownFlags["KeyC"]) {
+    // dragMarksを設定
+    dragMarks[personIndex] = 
+      poseData[personIndex].map((node) => node != null);
+      isDragging = true;
+    if (event.altKey) {
+      dragMode = "move";
+    } else if (event.ctrlKey) {
+      dragMode = "scale";
+    } else if (event.shiftKey) {
+      dragMode = "rotate";
+      rotateBaseVector = [0, 0];
+    } else if (keyDownFlags["KeyX"]) {
+      dragMode = "rotateX";
+    } else if (keyDownFlags["KeyC"]) {
+      dragMode = "rotateY";
+    }
+  } else if (keyDownFlags["Space"]) {
+    dragMarks[personIndex] = 
+      poseData[personIndex].map(
+        (node) => node != null && distSq(p, node) < dragRange ** 2);
+    isDragging = dragMarks[personIndex].some((mark) => mark);
+    dragMode = "move";
+  } else if (minDist < 16) {
+    dragMarks[personIndex][nodeIndex] = true;
+    isDragging = true;
+    dragMode = "move";
+  }
+}
+
+// Canvas要素上でマウスが動いた場合に呼び出される関数
+function handleMouseMove(event) {
+  mouseCursor = getCanvasPosition(event);
+  if (isDragging) {
+    const p = getCanvasPosition(event);
+    const dragOffset = [p[0] - dragStart[0], p[1] - dragStart[1]];
+    const latestPoseData = fetchLatestPoseData();
+
+    if (dragMode == "scale") {
+      // 拡大縮小
+      let xScale = 1 + dragOffset[0] / canvas.width;
+      let yScale = 1 + dragOffset[0] / canvas.height;
+      forEachMarkedNodes((i, j, node) => {
+        const lp = latestPoseData[i][j];
+        node[0] = (lp[0] - dragStart[0]) * xScale + dragStart[0];
+        node[1] = (lp[1] - dragStart[1]) * yScale + dragStart[1];
+      });
+    } else if (dragMode == "rotate") {
+      rotateBaseVector = dragOffset;
+      if (!event.shiftKey) {
+        dragMode = "rotate2";
+      }
+    } else if (dragMode == "rotate2") {
+      // 回転
+      let angle = directedAngleTo(rotateBaseVector, dragOffset);
+      forEachMarkedNodes((i, j, node) => {
+        const lp = latestPoseData[i][j];
+        let x = lp[0] - dragStart[0];
+        let y = lp[1] - dragStart[1];
+        let sin = Math.sin(angle);
+        let cos = Math.cos(angle);
+        node[0] = x * cos - y * sin + dragStart[0];
+        node[1] = x * sin + y * cos + dragStart[1];
+      });
+    } else if (dragMode == "rotateX") {
+      const center = dragStart;
+      const angle = dragOffset[1] / -40;
+      forEachMarkedNodes((i, j, node) => {
+        const lp = latestPoseData[i][j];
+        const np = rotateAndProject(rotateX, lp, center, angle);
+        node[0] = np[0];
+        node[1] = np[1];
+      });
+    } else if (dragMode == "rotateY") {
+      const center = dragStart;
+      const angle = dragOffset[0] / 40;
+      forEachMarkedNodes((i, j, node) => {
+        const lp = latestPoseData[i][j];
+        const np = rotateAndProject(rotateY, lp, center, angle);
+        node[0] = np[0];
+        node[1] = np[1];
+      });
+    } else if (dragMode == "move") {
+      // 移動
+      forEachMarkedNodes((i, j, node) => {
+        const lp = latestPoseData[i][j];
+        node[0] = lp[0] + dragOffset[0];
+        node[1] = lp[1] + dragOffset[1];
+      });
+    }
+
+    // animatePicture(document.getElementById("canvas4"),[canvas.width, canvas.height], history[0][0], poseData[0], pictureImageData, initialPoseField);
+  }
+
+  Redraw();
+}
+
+function handleMouseUp(event) {
+  console.profile("animate");
+  animatePicture(
+    document.getElementById("canvas4"),
+    [canvas.width, canvas.height], history[0][0], poseData[0], pictureImageData, initialPoseField);
+    console.profileEnd("animate");
+    isDragging = false; 
+  addHistory();
+  Redraw();
+}
+
+function handleMouseLeave(event) {
+  mouseCursor = [-1,-1];
+  if (isDragging) {
+    handleMouseUp(event);
+  }
+  keyDownFlags = {};
+}
+
+function ModifyDragRange(delta) { dragRange = Math.max(dragRangeDelta, Math.min(512, dragRange + delta)); }
+
+document.addEventListener('wheel', function(event) {
+  if (!isMouseOnCanvas()) {return;}
+  if (!event.altKey && !keyDownFlags['Space']) {return;}
+
+  event.preventDefault();
+  const deltaY = event.deltaY;
+  if (deltaY < 0) {ModifyDragRange(-dragRangeDelta);}
+  if (0 < deltaY) {ModifyDragRange(dragRangeDelta);}
+  lastWheeling = new Date().getTime();
+  Redraw();
+  window.setTimeout(function() { Redraw(); }, wheelDisplayTime+10);
+}, {passive: false});
+
+document.addEventListener("keydown", (event) => { 
+  if (!isMouseOnCanvas()) {return;}
+
+  if (event.code == "BracketLeft") { ModifyDragRange(-dragRangeDelta); }
+  if (event.code == "BracketRight") { ModifyDragRange(dragRangeDelta); }
+  keyDownFlags[event.code] = true; 
+  Redraw();
+  event.preventDefault();
+});
+document.addEventListener("keyup", (event) => { 
+  if (!isMouseOnCanvas()) {return;}
+
+  keyDownFlags[event.code] = false; 
+  if (event.ctrlKey && event.code == "KeyE") {
+    addPerson();
+  } else if (event.ctrlKey && event.code == "KeyZ") {
+    if (event.shiftKey) {
+      redo();
+    } else {
+      undo();
+    }
+  }
+  Redraw();
+  event.preventDefault();
+});
+
+function initializeEditor() {
+  console.log("initializeEditor");
+
+  canvas = document.getElementById('canvas');
+  ctx = canvas.getContext('2d');
+
+  canvas.addEventListener('mousedown', handleMouseDown);
+  canvas.addEventListener('mousemove', handleMouseMove);
+  canvas.addEventListener('mouseup', handleMouseUp);
+  canvas.addEventListener('mouseleave', handleMouseLeave);
+  poseData = [];
+  clearHistory();
+}
+
+function importPose(jsonData) {
+  if (jsonData != null) {
+    newPoseData = makePoseDataFromCandidateAndSubset(jsonData.candidate, jsonData.subset);
+    if (jsonData.width != null && jsonData.height != null) {
+      resizeCanvas(jsonData.width, jsonData.height);
+    }
+  } else {
+    newPoseData = makePoseDataFromCandidateAndSubset(sampleCandidateSource, [sampleSubsetElementSource]);
+  }
+  poseData = poseData.concat(newPoseData);
+  addHistory();
+  Redraw();
+}
+
+// crc32
+// CRC32を初期化
+function initCrc32Table() {
+  const crcTable = new Uint32Array(256);
+  for (let i = 0; i < 256; i++) {
+    let c = i;
+    for (let j = 0; j < 8; j++) {
+      c = (c & 1) ? (0xEDB88320 ^ (c >>> 1)) : (c >>> 1);
+    }
+    crcTable[i] = c;
+  }
+  return crcTable;
+}
+
+// データのCRC32を計算
+function getCrc32(data, crc=0) {
+  const crcTable = initCrc32Table();
+  crc = (crc ^ 0xFFFFFFFF) >>> 0;
+  for (let i = 0; i < data.length; i++) {
+    crc = crcTable[(crc ^ data[i]) & 0xFF] ^ (crc >>> 8);
+  }
+  return (crc ^ 0xFFFFFFFF) >>> 0;
+}
+
+function stringToUint8Array(str) {
+  var arr = new Uint8Array(str.length);
+  for (var i = 0; i < str.length; i++) {
+    arr[i] = str.charCodeAt(i);
+  }
+  return arr;
+}
+
+function base64ToUint8Array(base64Str) {
+  return stringToUint8Array(atob(base64Str));
+}
+
+function visitPng(png, type) {
+  var dataLength;
+  var chunkType;
+  var nextChunkPos;
+  var Signature = String.fromCharCode(137, 80, 78, 71, 13, 10, 26, 10);
+  var rpos = 0;
+  
+  // シグネチャの確認
+  if (String.fromCharCode.apply(null, png.subarray(rpos, rpos += 8)) !== Signature) {
+      throw new Error('invalid signature');
+  }
+  
+  // チャンクの探索
+  while (rpos < png.length) {
+      dataLength = (
+          (png[rpos++] << 24) |
+          (png[rpos++] << 16) |
+          (png[rpos++] <<  8) |
+          (png[rpos++]      )
+      ) >>> 0;
+  
+      nextChunkPos = rpos + dataLength + 8;
+  
+      chunkType = String.fromCharCode.apply(null, png.subarray(rpos, rpos += 4));
+      
+      if (chunkType === type) {
+        return [rpos - 8, dataLength, nextChunkPos];
+      }
+      
+      rpos = nextChunkPos;
+  }
+}
+
+function createChunk(type, data) {
+  var dataLength = data.length;
+  var chunk = new Uint8Array(4 + 4 + dataLength + 4);
+  var type = stringToUint8Array(type);
+  var pos = 0;
+  
+  // length
+  chunk[pos++] = (dataLength >> 24) & 0xff;
+  chunk[pos++] = (dataLength >> 16) & 0xff;
+  chunk[pos++] = (dataLength >>  8) & 0xff;
+  chunk[pos++] = (dataLength      ) & 0xff;
+  
+  // type
+  chunk[pos++] = type[0];
+  chunk[pos++] = type[1];
+  chunk[pos++] = type[2];
+  chunk[pos++] = type[3];
+  
+  // data
+  for (let i = 0; i < dataLength; ++i) {
+      chunk[pos++] = data[i];
+  }
+  
+  //crc
+  initCrc32Table();
+  let crc = getCrc32(type);
+  crc = getCrc32(data, crc);
+  chunk[pos++] = (crc >> 24) & 0xff;
+  chunk[pos++] = (crc >> 16) & 0xff;
+  chunk[pos++] = (crc >>  8) & 0xff;
+  chunk[pos++] = (crc      ) & 0xff;
+  
+  return chunk;
+}
+
+function insertChunk(destBuffer, sourceBuffer, rpos, chunk) {
+  var pos = 0;
+  
+  // IDAT チャンクの前までコピー
+  destBuffer.set(sourceBuffer.subarray(0, rpos), pos);
+  pos += rpos;
+  
+  // hoGe チャンクをコピー
+  destBuffer.set(chunk, pos);
+  pos += chunk.length;
+  
+  // IDAT チャンク以降をコピー
+  destBuffer.set(sourceBuffer.subarray(rpos), pos);
+}
+
+function mergeCanvasWithPose(keyword, content) {
+  const canvasUrl = canvas.toDataURL();
+
+  var insertion = stringToUint8Array(`${keyword}\0${content}`);
+  var chunk = createChunk("tEXt", insertion);
+  var sourceBuffer = base64ToUint8Array(canvasUrl.split(',')[1]);
+  var destBuffer = new Uint8Array(sourceBuffer.length + insertion.length + 12);
+
+  var [rpos, dataLength, nextChunkPos] = visitPng(sourceBuffer, "IHDR");
+  insertChunk(destBuffer, sourceBuffer, nextChunkPos, chunk);
+
+  var blob = new Blob([destBuffer], {type: "image/png"});
+  var url = URL.createObjectURL(blob);
+  return url;  
+}
+
+function savePose() {
+  var [candidate, subset] = poseDataToCandidateAndSubset(poseData);
+  let jsonData = {candidate: candidate, subset: subset};
+
+  var url = mergeCanvasWithPose("openpose", JSON.stringify(jsonData));
+
+  const createEl = document.createElement('a');
+  createEl.href = url;
+
+  // This is the name of our downloaded file
+  createEl.download = "pose.png";
+
+  createEl.click();
+  createEl.remove();
+
+  return jsonData;
+}
+
+function importPicture(image) {
+  var pic = new Image();
+  pic.onload = () => {
+    const size = [canvas.width, canvas.height];
+
+    const canvas3 = document.getElementById("canvas3");
+    pictureImageData = makeImageDataFromPicture(canvas3, size, pic);
+
+    const canvas4 = document.getElementById("canvas4");
+    offscreen = canvas4;
+
+    initialPoseField = buildWeightMap(size, poseData[0]);
+    animatePicture(canvas4, size, poseData[0], poseData[0], pictureImageData, initialPoseField);
+
+    Redraw();
+  };
+  pic.src = image;
+
+  initMicroscope();
+}
+
+function importBackground(image) {
+  if (image == null) {
+    canvasBg = null;
+    Redraw();
+    return;
+  }
+
+  let m = new Image();
+  m.src = image;
+  m.onload = function() {
+    canvasBg = m;
+    Redraw();
+  }
+}

main.py

@@ -0,0 +1,61 @@
+import gradio as gr
+import json as js
+import util
+from fileservice import app
+from fastapi.staticfiles import StaticFiles
+from pose import infer, draw
+from abgr import remove_bg, split_image
+
+def image_changed(image):
+  if image == None:
+    return "estimation", {}
+
+  print("make mask")
+  cvimage = util.pil2cv(image)
+  mask, fg, bg = split_image(cvimage[..., ::-1])
+
+  print("pose not found")
+  pose_result, _ = infer(cvimage)
+
+  candidate, subset = util.convert_to_openpose(pose_result)
+  candidateJson = util.candidate_to_json_string(candidate)
+  subsetJson = util.subset_to_json_string(subset)
+  jsonText = f'{{"candidate":{candidateJson}, "subset":{subsetJson}, "width":{image.width}, "height":{image.height}}}'
+  return f'{image.width}px x {image.height}px, {len(subset)} indivisual(s)', jsonText, mask, fg, bg
+
+with gr.Blocks(css="""button { min-width: 80px; }""") as demo:
+  with gr.Row():
+    with gr.Column(scale=1):
+      source = gr.Image(type="pil")
+      info = gr.Markdown("""info""")
+      btn = gr.Button("Import")  
+      mask = gr.Image()
+      frontImage = gr.Image(image_mode="RGBA")
+      backImage = gr.Image(image_mode="RGBA")
+      with gr.Accordion(label="Json", open=False):
+        json = gr.JSON(label="Json")
+    with gr.Column(scale=3):
+      gr.HTML('<canvas id="canvas" width="512" height="512"></canvas>')
+  with gr.Row():
+    gr.HTML('<canvas id="canvas2" width="512" height="512" style="display:none"></canvas>')
+    gr.HTML('<canvas id="canvas3" width="512" height="512" style="display:none"></canvas>')
+    gr.HTML('<canvas id="canvas4" width="512" height="512" style="display:none"></canvas>')
+    gr.HTML('<canvas id="microscope" width="50" height="50" style="display:none"></canvas>')
+
+  source.change(
+    fn = image_changed,
+    inputs = [source],
+    outputs = [info, json, mask, frontImage, backImage])
+
+  btn.click(
+    fn = None,
+    inputs = [frontImage, backImage, json],
+    outputs = [],
+    _js="(frontImage, backImage, json) => { initializeEditor(); importPose(json); importPicture(frontImage); importBackground(backImage); return []; }")
+
+  demo.load(fn=None, inputs=[], outputs=[], _js="() => { initializeEditor(); importPose(); importPicture('./static/sample2.png'); return []; }")
+
+print("mount")
+app.mount("/static", StaticFiles(directory="static"), name="static")
+app.mount("/js", StaticFiles(directory="js"), name="js")
+gr.mount_gradio_app(app, demo, path="/")

pose.py

@@ -0,0 +1,50 @@
+from mmpose.apis import (inference_top_down_pose_model, init_pose_model,
+                         process_mmdet_results, vis_pose_result)
+from mmpose.datasets import DatasetInfo
+from mmdet.apis import inference_detector, init_detector
+
+det_model = init_detector(
+    "./external/faster_rcnn_r50_fpn_coco.py", 
+    "./faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth",
+    device="cpu")
+pose_model = init_pose_model(
+    "./external/hrnet_w48_coco_256x192.py",
+    "./hrnet_w48_coco_256x192-b9e0b3ab_20200708.pth",
+    device="cpu")
+
+dataset = pose_model.cfg.data['test']['type']
+dataset_info = pose_model.cfg.data['test'].get('dataset_info', None)
+
+dataset_info = DatasetInfo(dataset_info)
+
+def infer(image):
+    mmdet_results = inference_detector(det_model, image)
+    person_results = process_mmdet_results(mmdet_results, 1)
+
+    pose_results, returned_outputs = inference_top_down_pose_model(
+        pose_model,
+        image,
+        person_results,
+        bbox_thr=0.3,
+        format='xyxy',
+        dataset=dataset,
+        dataset_info=dataset_info,
+        return_heatmap=False,
+        outputs=None)
+    # print(pose_results)
+    # print(returned_outputs)
+
+    return pose_results, returned_outputs
+
+def draw(image, results):
+    return vis_pose_result(
+        pose_model,
+        image,
+        results,
+        dataset=dataset,
+        dataset_info=dataset_info,
+        kpt_score_thr=0.3,
+        radius=4,
+        thickness=3,
+        show=False,
+        out_file=None)

requirements.txt

@@ -0,0 +1,12 @@
+fastapi==0.92.0
+gradio==3.18.0
+numpy==1.23.5
+opencv_python
+scipy
+torch
+torchvision
+openmim
+huggingface-hub
+onnxruntime-gpu
+
+

util.py

@@ -0,0 +1,63 @@
+import numpy as np
+import cv2
+
+def pil2cv(image):
+    ''' PIL型 -> OpenCV型 '''
+    new_image = np.array(image, dtype=np.uint8)
+    if new_image.ndim == 2:  # モノクロ
+        pass
+    elif new_image.shape[2] == 3:  # カラー
+        new_image = cv2.cvtColor(new_image, cv2.COLOR_RGB2BGR)
+    elif new_image.shape[2] == 4:  # 透過
+        new_image = cv2.cvtColor(new_image, cv2.COLOR_RGBA2BGRA)
+    return new_image
+
+def candidate_to_json_string(arr):
+    a = [f'[{x:.2f}, {y:.2f}]' for x, y, *_ in arr]
+    return '[' + ', '.join(a) + ']'
+
+# make subset to json
+def subset_to_json_string(arr):
+    arr_str = ','.join(['[' + ','.join([f'{num:.2f}' for num in row]) + ']' for row in arr])
+    return '[' + arr_str + ']'
+
+keypoint_index_mapping = [
+    0, 
+    17,
+    6,
+    8,
+    10,
+    5,
+    7,
+    9,
+    12,
+    14,
+    16,
+    11,
+    13,
+    15,
+    2,
+    1,
+    4,
+    3,
+]
+
+def convert_keypoints(keypoints):
+    return [keypoints[i] for i in keypoint_index_mapping]
+
+def convert_to_openpose(pose_result):
+    candidate = []
+    subset = []
+    for d in pose_result:
+        n = len(candidate)
+        if d['bbox'][4] < 0.9: 
+            continue
+        keypoints = d['keypoints'][:, :2].tolist()
+        midpoint = [(keypoints[5][0] + keypoints[6][0]) / 2, (keypoints[5][1] + keypoints[6][1]) / 2]
+        keypoints.append(midpoint)
+        candidate.extend(convert_keypoints(keypoints))
+        m = len(candidate)
+        subset.append([j for j in range(n, m)])
+
+    return candidate, subset
+