Create README.md
Browse files
README.md
ADDED
@@ -0,0 +1,136 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
---
|
2 |
+
library_name: transformers.js
|
3 |
+
tags:
|
4 |
+
- pose-estimation
|
5 |
+
license: apache-2.0
|
6 |
+
---
|
7 |
+
|
8 |
+
|
9 |
+
https://github.com/open-mmlab/mmpose/tree/main/projects/rtmo with ONNX weights to be compatible with Transformers.js.
|
10 |
+
|
11 |
+
## Usage (Transformers.js)
|
12 |
+
|
13 |
+
If you haven't already, you can install the [Transformers.js](https://huggingface.co/docs/transformers.js) JavaScript library from [NPM](https://www.npmjs.com/package/@xenova/transformers) using:
|
14 |
+
```bash
|
15 |
+
npm i @xenova/transformers
|
16 |
+
```
|
17 |
+
|
18 |
+
**Example:** Perform pose-estimation w/ `Xenova/RTMO-s`.
|
19 |
+
|
20 |
+
```js
|
21 |
+
import { AutoModel, AutoProcessor, RawImage } from '@xenova/transformers';
|
22 |
+
|
23 |
+
// Load model and processor
|
24 |
+
const model_id = 'Xenova/RTMO-s';
|
25 |
+
const model = await AutoModel.from_pretrained(model_id);
|
26 |
+
const processor = await AutoProcessor.from_pretrained(model_id);
|
27 |
+
|
28 |
+
// Read image and run processor
|
29 |
+
const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/football-match.jpg';
|
30 |
+
const image = await RawImage.read(url);
|
31 |
+
const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image);
|
32 |
+
|
33 |
+
// Predict bounding boxes and keypoints
|
34 |
+
const { dets, keypoints } = await model({ input: pixel_values });
|
35 |
+
|
36 |
+
// Select the first image
|
37 |
+
const predicted_boxes = dets.tolist()[0];
|
38 |
+
const predicted_points = keypoints.tolist()[0];
|
39 |
+
const [height, width] = original_sizes[0];
|
40 |
+
const [resized_height, resized_width] = reshaped_input_sizes[0];
|
41 |
+
|
42 |
+
// Compute scale values
|
43 |
+
const xScale = width / resized_width;
|
44 |
+
const yScale = height / resized_height;
|
45 |
+
|
46 |
+
// Define thresholds
|
47 |
+
const point_threshold = 0.3;
|
48 |
+
const box_threshold = 0.3;
|
49 |
+
|
50 |
+
// Display results
|
51 |
+
for (let i = 0; i < predicted_boxes.length; ++i) {
|
52 |
+
const [xmin, ymin, xmax, ymax, box_score] = predicted_boxes[i];
|
53 |
+
if (box_score < box_threshold) continue;
|
54 |
+
|
55 |
+
const x1 = (xmin * xScale).toFixed(2);
|
56 |
+
const y1 = (ymin * yScale).toFixed(2);
|
57 |
+
const x2 = (xmax * xScale).toFixed(2);
|
58 |
+
const y2 = (ymax * yScale).toFixed(2);
|
59 |
+
|
60 |
+
console.log(`Found person at [${x1}, ${y1}, ${x2}, ${y2}] with score ${box_score.toFixed(3)}`)
|
61 |
+
const points = predicted_points[i]; // of shape [17, 3]
|
62 |
+
for (let id = 0; id < points.length; ++id) {
|
63 |
+
const label = model.config.id2label[id];
|
64 |
+
const [x, y, point_score] = points[id];
|
65 |
+
if (point_score < point_threshold) continue;
|
66 |
+
console.log(` - ${label}: (${(x * xScale).toFixed(2)}, ${(y * yScale).toFixed(2)}) with score ${point_score.toFixed(3)}`);
|
67 |
+
}
|
68 |
+
}
|
69 |
+
```
|
70 |
+
|
71 |
+
<details>
|
72 |
+
|
73 |
+
<summary>See example output</summary>
|
74 |
+
|
75 |
+
```
|
76 |
+
Found person at [423.33, 55.52, 644.28, 504.13] with score 0.988
|
77 |
+
- nose: (527.30, 117.12) with score 0.733
|
78 |
+
- left_eye: (541.79, 109.26) with score 0.554
|
79 |
+
- right_eye: (515.04, 107.59) with score 0.475
|
80 |
+
- left_shoulder: (563.30, 171.75) with score 1.000
|
81 |
+
- right_shoulder: (464.21, 159.75) with score 1.000
|
82 |
+
- left_elbow: (575.71, 238.04) with score 0.998
|
83 |
+
- right_elbow: (436.06, 218.10) with score 0.999
|
84 |
+
- left_wrist: (605.86, 303.35) with score 1.000
|
85 |
+
- right_wrist: (497.47, 220.82) with score 1.000
|
86 |
+
- left_hip: (540.97, 307.31) with score 1.000
|
87 |
+
- right_hip: (475.85, 318.78) with score 1.000
|
88 |
+
- left_knee: (578.63, 368.63) with score 1.000
|
89 |
+
- right_knee: (501.05, 442.49) with score 1.000
|
90 |
+
- left_ankle: (572.11, 464.96) with score 0.991
|
91 |
+
- right_ankle: (535.75, 441.52) with score 0.981
|
92 |
+
Found person at [89.97, 3.96, 517.81, 507.28] with score 0.966
|
93 |
+
- left_shoulder: (242.65, 111.06) with score 0.999
|
94 |
+
- right_shoulder: (228.79, 112.54) with score 0.999
|
95 |
+
- left_elbow: (321.84, 169.07) with score 0.999
|
96 |
+
- right_elbow: (225.76, 218.20) with score 1.000
|
97 |
+
- left_wrist: (351.73, 220.74) with score 0.999
|
98 |
+
- right_wrist: (160.19, 228.03) with score 1.000
|
99 |
+
- left_hip: (342.34, 246.81) with score 1.000
|
100 |
+
- right_hip: (360.05, 259.35) with score 0.999
|
101 |
+
- left_knee: (299.56, 377.97) with score 0.998
|
102 |
+
- right_knee: (313.81, 378.83) with score 0.976
|
103 |
+
- left_ankle: (443.84, 312.35) with score 0.983
|
104 |
+
- right_ankle: (424.74, 433.61) with score 0.823
|
105 |
+
Found person at [-0.53, 51.78, 153.65, 371.17] with score 0.769
|
106 |
+
- nose: (75.52, 85.67) with score 0.363
|
107 |
+
- left_shoulder: (121.54, 113.17) with score 1.000
|
108 |
+
- right_shoulder: (49.77, 117.60) with score 1.000
|
109 |
+
- left_elbow: (132.90, 147.02) with score 0.932
|
110 |
+
- right_elbow: (30.31, 156.42) with score 0.992
|
111 |
+
- left_wrist: (154.43, 162.08) with score 0.871
|
112 |
+
- right_wrist: (17.20, 196.43) with score 0.943
|
113 |
+
- left_hip: (105.61, 204.27) with score 0.999
|
114 |
+
- right_hip: (61.99, 203.66) with score 0.999
|
115 |
+
- left_knee: (114.70, 270.91) with score 1.000
|
116 |
+
- right_knee: (63.75, 275.33) with score 1.000
|
117 |
+
- left_ankle: (125.53, 342.00) with score 0.998
|
118 |
+
- right_ankle: (63.16, 344.07) with score 0.997
|
119 |
+
Found person at [519.40, 34.94, 650.11, 312.07] with score 0.488
|
120 |
+
- nose: (554.82, 76.58) with score 0.920
|
121 |
+
- left_eye: (563.12, 69.41) with score 0.666
|
122 |
+
- right_eye: (544.82, 70.01) with score 0.595
|
123 |
+
- left_shoulder: (596.60, 105.61) with score 0.999
|
124 |
+
- right_shoulder: (523.29, 107.31) with score 0.969
|
125 |
+
- left_elbow: (625.14, 151.30) with score 0.999
|
126 |
+
- right_elbow: (515.96, 147.59) with score 0.322
|
127 |
+
- left_wrist: (630.90, 196.91) with score 0.998
|
128 |
+
- right_wrist: (520.75, 181.83) with score 0.415
|
129 |
+
- left_hip: (583.24, 200.84) with score 0.998
|
130 |
+
- right_hip: (533.69, 200.01) with score 0.978
|
131 |
+
- left_knee: (583.79, 265.14) with score 0.934
|
132 |
+
- right_knee: (538.27, 262.98) with score 0.669
|
133 |
+
- left_ankle: (584.90, 309.76) with score 0.489
|
134 |
+
```
|
135 |
+
|
136 |
+
</details>
|