Update README.md
Browse files
README.md
CHANGED
@@ -3,4 +3,191 @@ library_name: transformers.js
|
|
3 |
tags:
|
4 |
- pose-estimation
|
5 |
license: agpl-3.0
|
6 |
-
---
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
3 |
tags:
|
4 |
- pose-estimation
|
5 |
license: agpl-3.0
|
6 |
+
---
|
7 |
+
|
8 |
+
YOLOv8l-pose with ONNX weights to be compatible with Transformers.js.
|
9 |
+
|
10 |
+
## Usage (Transformers.js)
|
11 |
+
|
12 |
+
If you haven't already, you can install the [Transformers.js](https://huggingface.co/docs/transformers.js) JavaScript library from [NPM](https://www.npmjs.com/package/@xenova/transformers) using:
|
13 |
+
```bash
|
14 |
+
npm i @xenova/transformers
|
15 |
+
```
|
16 |
+
|
17 |
+
**Example:** Perform pose-estimation w/ `Xenova/yolov8l-pose`.
|
18 |
+
|
19 |
+
```js
|
20 |
+
import { AutoModel, AutoProcessor, RawImage } from '@xenova/transformers';
|
21 |
+
|
22 |
+
// Load model and processor
|
23 |
+
const model_id = 'Xenova/yolov8l-pose';
|
24 |
+
const model = await AutoModel.from_pretrained(model_id);
|
25 |
+
const processor = await AutoProcessor.from_pretrained(model_id);
|
26 |
+
|
27 |
+
// Read image and run processor
|
28 |
+
const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/football-match.jpg';
|
29 |
+
const image = await RawImage.read(url);
|
30 |
+
const { pixel_values } = await processor(image);
|
31 |
+
|
32 |
+
// Set thresholds
|
33 |
+
const threshold = 0.3; // Remove detections with low confidence
|
34 |
+
const iouThreshold = 0.5; // Used to remove duplicates
|
35 |
+
const pointThreshold = 0.3; // Hide uncertain points
|
36 |
+
|
37 |
+
// Predict bounding boxes and keypoints
|
38 |
+
const { output0 } = await model({ images: pixel_values });
|
39 |
+
|
40 |
+
// Post-process:
|
41 |
+
const permuted = output0[0].transpose(1, 0);
|
42 |
+
// `permuted` is a Tensor of shape [ 8400, 56 ]:
|
43 |
+
// - 8400 potential detections
|
44 |
+
// - 56 parameters for each box:
|
45 |
+
// - 4 for the bounding box dimensions (x-center, y-center, width, height)
|
46 |
+
// - 1 for the confidence score
|
47 |
+
// - 17 * 3 = 51 for the pose keypoints: 17 labels, each with (x, y, visibilitiy)
|
48 |
+
|
49 |
+
// Example code to format it nicely:
|
50 |
+
const results = [];
|
51 |
+
const [scaledHeight, scaledWidth] = pixel_values.dims.slice(-2);
|
52 |
+
for (const [xc, yc, w, h, score, ...keypoints] of permuted.tolist()) {
|
53 |
+
if (score < threshold) continue;
|
54 |
+
|
55 |
+
// Get pixel values, taking into account the original image size
|
56 |
+
const x1 = (xc - w / 2) / scaledWidth * image.width;
|
57 |
+
const y1 = (yc - h / 2) / scaledHeight * image.height;
|
58 |
+
const x2 = (xc + w / 2) / scaledWidth * image.width;
|
59 |
+
const y2 = (yc + h / 2) / scaledHeight * image.height;
|
60 |
+
results.push({ x1, x2, y1, y2, score, keypoints })
|
61 |
+
}
|
62 |
+
|
63 |
+
|
64 |
+
// Define helper functions
|
65 |
+
function removeDuplicates(detections, iouThreshold) {
|
66 |
+
const filteredDetections = [];
|
67 |
+
|
68 |
+
for (const detection of detections) {
|
69 |
+
let isDuplicate = false;
|
70 |
+
let duplicateIndex = -1;
|
71 |
+
let maxIoU = 0;
|
72 |
+
|
73 |
+
for (let i = 0; i < filteredDetections.length; ++i) {
|
74 |
+
const filteredDetection = filteredDetections[i];
|
75 |
+
const iou = calculateIoU(detection, filteredDetection);
|
76 |
+
if (iou > iouThreshold) {
|
77 |
+
isDuplicate = true;
|
78 |
+
if (iou > maxIoU) {
|
79 |
+
maxIoU = iou;
|
80 |
+
duplicateIndex = i;
|
81 |
+
}
|
82 |
+
}
|
83 |
+
}
|
84 |
+
|
85 |
+
if (!isDuplicate) {
|
86 |
+
filteredDetections.push(detection);
|
87 |
+
} else if (duplicateIndex !== -1 && detection.score > filteredDetections[duplicateIndex].score) {
|
88 |
+
filteredDetections[duplicateIndex] = detection;
|
89 |
+
}
|
90 |
+
}
|
91 |
+
|
92 |
+
return filteredDetections;
|
93 |
+
}
|
94 |
+
|
95 |
+
function calculateIoU(detection1, detection2) {
|
96 |
+
const xOverlap = Math.max(0, Math.min(detection1.x2, detection2.x2) - Math.max(detection1.x1, detection2.x1));
|
97 |
+
const yOverlap = Math.max(0, Math.min(detection1.y2, detection2.y2) - Math.max(detection1.y1, detection2.y1));
|
98 |
+
const overlapArea = xOverlap * yOverlap;
|
99 |
+
|
100 |
+
const area1 = (detection1.x2 - detection1.x1) * (detection1.y2 - detection1.y1);
|
101 |
+
const area2 = (detection2.x2 - detection2.x1) * (detection2.y2 - detection2.y1);
|
102 |
+
const unionArea = area1 + area2 - overlapArea;
|
103 |
+
|
104 |
+
return overlapArea / unionArea;
|
105 |
+
}
|
106 |
+
|
107 |
+
const filteredResults = removeDuplicates(results, iouThreshold);
|
108 |
+
|
109 |
+
// Display results
|
110 |
+
for (const { x1, x2, y1, y2, score, keypoints } of filteredResults) {
|
111 |
+
console.log(`Found person at [${x1}, ${y1}, ${x2}, ${y2}] with score ${score.toFixed(3)}`)
|
112 |
+
for (let i = 0; i < keypoints.length; i += 3) {
|
113 |
+
const label = model.config.id2label[Math.floor(i / 3)];
|
114 |
+
const [x, y, point_score] = keypoints.slice(i, i + 3);
|
115 |
+
if (point_score < pointThreshold) continue;
|
116 |
+
console.log(` - ${label}: (${x.toFixed(2)}, ${y.toFixed(2)}) with score ${point_score.toFixed(3)}`);
|
117 |
+
}
|
118 |
+
}
|
119 |
+
```
|
120 |
+
|
121 |
+
<details>
|
122 |
+
|
123 |
+
<summary>See example output</summary>
|
124 |
+
|
125 |
+
```
|
126 |
+
Found person at [539.2378807067871, 41.92433733940124, 642.9805946350098, 334.98332471847533] with score 0.727
|
127 |
+
- nose: (445.67, 84.43) with score 0.976
|
128 |
+
- left_eye: (451.88, 76.89) with score 0.983
|
129 |
+
- right_eye: (440.39, 76.33) with score 0.888
|
130 |
+
- left_ear: (463.89, 81.68) with score 0.837
|
131 |
+
- left_shoulder: (478.95, 123.91) with score 0.993
|
132 |
+
- right_shoulder: (419.52, 123.44) with score 0.694
|
133 |
+
- left_elbow: (501.07, 180.46) with score 0.979
|
134 |
+
- left_wrist: (504.60, 238.34) with score 0.950
|
135 |
+
- left_hip: (469.53, 220.77) with score 0.985
|
136 |
+
- right_hip: (431.21, 222.54) with score 0.875
|
137 |
+
- left_knee: (473.45, 302.16) with score 0.972
|
138 |
+
- right_knee: (432.61, 302.91) with score 0.759
|
139 |
+
- left_ankle: (467.74, 380.37) with score 0.874
|
140 |
+
- right_ankle: (438.06, 381.94) with score 0.516
|
141 |
+
Found person at [0.59722900390625, 59.435689163208, 157.59026527404785, 370.3985949516296] with score 0.927
|
142 |
+
- nose: (56.99, 100.53) with score 0.959
|
143 |
+
- left_eye: (63.46, 94.19) with score 0.930
|
144 |
+
- right_eye: (51.11, 96.48) with score 0.846
|
145 |
+
- left_ear: (73.43, 97.84) with score 0.798
|
146 |
+
- right_ear: (46.36, 99.41) with score 0.484
|
147 |
+
- left_shoulder: (84.93, 134.17) with score 0.988
|
148 |
+
- right_shoulder: (41.60, 133.96) with score 0.976
|
149 |
+
- left_elbow: (96.33, 189.89) with score 0.959
|
150 |
+
- right_elbow: (24.60, 192.73) with score 0.879
|
151 |
+
- left_wrist: (104.79, 258.62) with score 0.928
|
152 |
+
- right_wrist: (7.89, 238.55) with score 0.830
|
153 |
+
- left_hip: (83.23, 234.45) with score 0.993
|
154 |
+
- right_hip: (53.89, 235.50) with score 0.991
|
155 |
+
- left_knee: (87.80, 326.73) with score 0.988
|
156 |
+
- right_knee: (49.44, 327.89) with score 0.982
|
157 |
+
- left_ankle: (100.93, 416.88) with score 0.925
|
158 |
+
- right_ankle: (44.52, 421.24) with score 0.912
|
159 |
+
Found person at [112.88127899169922, 13.998864459991454, 504.09095764160156, 533.4011061668397] with score 0.943
|
160 |
+
- nose: (122.64, 98.36) with score 0.366
|
161 |
+
- left_ear: (132.43, 77.58) with score 0.794
|
162 |
+
- left_shoulder: (196.67, 124.78) with score 0.999
|
163 |
+
- right_shoulder: (176.97, 142.00) with score 0.998
|
164 |
+
- left_elbow: (256.79, 196.00) with score 0.998
|
165 |
+
- right_elbow: (182.85, 279.47) with score 0.994
|
166 |
+
- left_wrist: (305.44, 270.10) with score 0.982
|
167 |
+
- right_wrist: (129.72, 281.09) with score 0.963
|
168 |
+
- left_hip: (275.59, 290.38) with score 1.000
|
169 |
+
- right_hip: (263.91, 310.60) with score 1.000
|
170 |
+
- left_knee: (237.89, 445.88) with score 0.998
|
171 |
+
- right_knee: (249.66, 477.34) with score 0.998
|
172 |
+
- left_ankle: (349.25, 438.70) with score 0.940
|
173 |
+
- right_ankle: (338.20, 586.62) with score 0.935
|
174 |
+
Found person at [424.730339050293, 67.2046113729477, 639.5703506469727, 493.03533136844635] with score 0.944
|
175 |
+
- nose: (416.55, 141.74) with score 0.991
|
176 |
+
- left_eye: (428.51, 130.99) with score 0.962
|
177 |
+
- right_eye: (408.83, 130.86) with score 0.938
|
178 |
+
- left_ear: (441.95, 133.48) with score 0.832
|
179 |
+
- right_ear: (399.56, 133.27) with score 0.652
|
180 |
+
- left_shoulder: (440.79, 193.75) with score 0.999
|
181 |
+
- right_shoulder: (372.38, 208.42) with score 0.998
|
182 |
+
- left_elbow: (453.56, 290.07) with score 0.995
|
183 |
+
- right_elbow: (350.56, 262.83) with score 0.992
|
184 |
+
- left_wrist: (482.36, 363.64) with score 0.995
|
185 |
+
- right_wrist: (398.84, 267.30) with score 0.993
|
186 |
+
- left_hip: (435.96, 362.27) with score 0.999
|
187 |
+
- right_hip: (388.40, 383.41) with score 0.999
|
188 |
+
- left_knee: (460.50, 425.60) with score 0.994
|
189 |
+
- right_knee: (403.19, 516.76) with score 0.992
|
190 |
+
- left_ankle: (459.31, 558.19) with score 0.893
|
191 |
+
- right_ankle: (426.29, 552.55) with score 0.868
|
192 |
+
```
|
193 |
+
</details>
|