Xenova HF staff commited on
Commit
4367212
1 Parent(s): 84c9f3a

Update README.md

Browse files
Files changed (1) hide show
  1. README.md +188 -1
README.md CHANGED
@@ -3,4 +3,191 @@ library_name: transformers.js
3
  tags:
4
  - pose-estimation
5
  license: agpl-3.0
6
- ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3
  tags:
4
  - pose-estimation
5
  license: agpl-3.0
6
+ ---
7
+
8
+ YOLOv8l-pose with ONNX weights to be compatible with Transformers.js.
9
+
10
+ ## Usage (Transformers.js)
11
+
12
+ If you haven't already, you can install the [Transformers.js](https://huggingface.co/docs/transformers.js) JavaScript library from [NPM](https://www.npmjs.com/package/@xenova/transformers) using:
13
+ ```bash
14
+ npm i @xenova/transformers
15
+ ```
16
+
17
+ **Example:** Perform pose-estimation w/ `Xenova/yolov8l-pose`.
18
+
19
+ ```js
20
+ import { AutoModel, AutoProcessor, RawImage } from '@xenova/transformers';
21
+
22
+ // Load model and processor
23
+ const model_id = 'Xenova/yolov8l-pose';
24
+ const model = await AutoModel.from_pretrained(model_id);
25
+ const processor = await AutoProcessor.from_pretrained(model_id);
26
+
27
+ // Read image and run processor
28
+ const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/football-match.jpg';
29
+ const image = await RawImage.read(url);
30
+ const { pixel_values } = await processor(image);
31
+
32
+ // Set thresholds
33
+ const threshold = 0.3; // Remove detections with low confidence
34
+ const iouThreshold = 0.5; // Used to remove duplicates
35
+ const pointThreshold = 0.3; // Hide uncertain points
36
+
37
+ // Predict bounding boxes and keypoints
38
+ const { output0 } = await model({ images: pixel_values });
39
+
40
+ // Post-process:
41
+ const permuted = output0[0].transpose(1, 0);
42
+ // `permuted` is a Tensor of shape [ 8400, 56 ]:
43
+ // - 8400 potential detections
44
+ // - 56 parameters for each box:
45
+ // - 4 for the bounding box dimensions (x-center, y-center, width, height)
46
+ // - 1 for the confidence score
47
+ // - 17 * 3 = 51 for the pose keypoints: 17 labels, each with (x, y, visibilitiy)
48
+
49
+ // Example code to format it nicely:
50
+ const results = [];
51
+ const [scaledHeight, scaledWidth] = pixel_values.dims.slice(-2);
52
+ for (const [xc, yc, w, h, score, ...keypoints] of permuted.tolist()) {
53
+ if (score < threshold) continue;
54
+
55
+ // Get pixel values, taking into account the original image size
56
+ const x1 = (xc - w / 2) / scaledWidth * image.width;
57
+ const y1 = (yc - h / 2) / scaledHeight * image.height;
58
+ const x2 = (xc + w / 2) / scaledWidth * image.width;
59
+ const y2 = (yc + h / 2) / scaledHeight * image.height;
60
+ results.push({ x1, x2, y1, y2, score, keypoints })
61
+ }
62
+
63
+
64
+ // Define helper functions
65
+ function removeDuplicates(detections, iouThreshold) {
66
+ const filteredDetections = [];
67
+
68
+ for (const detection of detections) {
69
+ let isDuplicate = false;
70
+ let duplicateIndex = -1;
71
+ let maxIoU = 0;
72
+
73
+ for (let i = 0; i < filteredDetections.length; ++i) {
74
+ const filteredDetection = filteredDetections[i];
75
+ const iou = calculateIoU(detection, filteredDetection);
76
+ if (iou > iouThreshold) {
77
+ isDuplicate = true;
78
+ if (iou > maxIoU) {
79
+ maxIoU = iou;
80
+ duplicateIndex = i;
81
+ }
82
+ }
83
+ }
84
+
85
+ if (!isDuplicate) {
86
+ filteredDetections.push(detection);
87
+ } else if (duplicateIndex !== -1 && detection.score > filteredDetections[duplicateIndex].score) {
88
+ filteredDetections[duplicateIndex] = detection;
89
+ }
90
+ }
91
+
92
+ return filteredDetections;
93
+ }
94
+
95
+ function calculateIoU(detection1, detection2) {
96
+ const xOverlap = Math.max(0, Math.min(detection1.x2, detection2.x2) - Math.max(detection1.x1, detection2.x1));
97
+ const yOverlap = Math.max(0, Math.min(detection1.y2, detection2.y2) - Math.max(detection1.y1, detection2.y1));
98
+ const overlapArea = xOverlap * yOverlap;
99
+
100
+ const area1 = (detection1.x2 - detection1.x1) * (detection1.y2 - detection1.y1);
101
+ const area2 = (detection2.x2 - detection2.x1) * (detection2.y2 - detection2.y1);
102
+ const unionArea = area1 + area2 - overlapArea;
103
+
104
+ return overlapArea / unionArea;
105
+ }
106
+
107
+ const filteredResults = removeDuplicates(results, iouThreshold);
108
+
109
+ // Display results
110
+ for (const { x1, x2, y1, y2, score, keypoints } of filteredResults) {
111
+ console.log(`Found person at [${x1}, ${y1}, ${x2}, ${y2}] with score ${score.toFixed(3)}`)
112
+ for (let i = 0; i < keypoints.length; i += 3) {
113
+ const label = model.config.id2label[Math.floor(i / 3)];
114
+ const [x, y, point_score] = keypoints.slice(i, i + 3);
115
+ if (point_score < pointThreshold) continue;
116
+ console.log(` - ${label}: (${x.toFixed(2)}, ${y.toFixed(2)}) with score ${point_score.toFixed(3)}`);
117
+ }
118
+ }
119
+ ```
120
+
121
+ <details>
122
+
123
+ <summary>See example output</summary>
124
+
125
+ ```
126
+ Found person at [539.2378807067871, 41.92433733940124, 642.9805946350098, 334.98332471847533] with score 0.727
127
+ - nose: (445.67, 84.43) with score 0.976
128
+ - left_eye: (451.88, 76.89) with score 0.983
129
+ - right_eye: (440.39, 76.33) with score 0.888
130
+ - left_ear: (463.89, 81.68) with score 0.837
131
+ - left_shoulder: (478.95, 123.91) with score 0.993
132
+ - right_shoulder: (419.52, 123.44) with score 0.694
133
+ - left_elbow: (501.07, 180.46) with score 0.979
134
+ - left_wrist: (504.60, 238.34) with score 0.950
135
+ - left_hip: (469.53, 220.77) with score 0.985
136
+ - right_hip: (431.21, 222.54) with score 0.875
137
+ - left_knee: (473.45, 302.16) with score 0.972
138
+ - right_knee: (432.61, 302.91) with score 0.759
139
+ - left_ankle: (467.74, 380.37) with score 0.874
140
+ - right_ankle: (438.06, 381.94) with score 0.516
141
+ Found person at [0.59722900390625, 59.435689163208, 157.59026527404785, 370.3985949516296] with score 0.927
142
+ - nose: (56.99, 100.53) with score 0.959
143
+ - left_eye: (63.46, 94.19) with score 0.930
144
+ - right_eye: (51.11, 96.48) with score 0.846
145
+ - left_ear: (73.43, 97.84) with score 0.798
146
+ - right_ear: (46.36, 99.41) with score 0.484
147
+ - left_shoulder: (84.93, 134.17) with score 0.988
148
+ - right_shoulder: (41.60, 133.96) with score 0.976
149
+ - left_elbow: (96.33, 189.89) with score 0.959
150
+ - right_elbow: (24.60, 192.73) with score 0.879
151
+ - left_wrist: (104.79, 258.62) with score 0.928
152
+ - right_wrist: (7.89, 238.55) with score 0.830
153
+ - left_hip: (83.23, 234.45) with score 0.993
154
+ - right_hip: (53.89, 235.50) with score 0.991
155
+ - left_knee: (87.80, 326.73) with score 0.988
156
+ - right_knee: (49.44, 327.89) with score 0.982
157
+ - left_ankle: (100.93, 416.88) with score 0.925
158
+ - right_ankle: (44.52, 421.24) with score 0.912
159
+ Found person at [112.88127899169922, 13.998864459991454, 504.09095764160156, 533.4011061668397] with score 0.943
160
+ - nose: (122.64, 98.36) with score 0.366
161
+ - left_ear: (132.43, 77.58) with score 0.794
162
+ - left_shoulder: (196.67, 124.78) with score 0.999
163
+ - right_shoulder: (176.97, 142.00) with score 0.998
164
+ - left_elbow: (256.79, 196.00) with score 0.998
165
+ - right_elbow: (182.85, 279.47) with score 0.994
166
+ - left_wrist: (305.44, 270.10) with score 0.982
167
+ - right_wrist: (129.72, 281.09) with score 0.963
168
+ - left_hip: (275.59, 290.38) with score 1.000
169
+ - right_hip: (263.91, 310.60) with score 1.000
170
+ - left_knee: (237.89, 445.88) with score 0.998
171
+ - right_knee: (249.66, 477.34) with score 0.998
172
+ - left_ankle: (349.25, 438.70) with score 0.940
173
+ - right_ankle: (338.20, 586.62) with score 0.935
174
+ Found person at [424.730339050293, 67.2046113729477, 639.5703506469727, 493.03533136844635] with score 0.944
175
+ - nose: (416.55, 141.74) with score 0.991
176
+ - left_eye: (428.51, 130.99) with score 0.962
177
+ - right_eye: (408.83, 130.86) with score 0.938
178
+ - left_ear: (441.95, 133.48) with score 0.832
179
+ - right_ear: (399.56, 133.27) with score 0.652
180
+ - left_shoulder: (440.79, 193.75) with score 0.999
181
+ - right_shoulder: (372.38, 208.42) with score 0.998
182
+ - left_elbow: (453.56, 290.07) with score 0.995
183
+ - right_elbow: (350.56, 262.83) with score 0.992
184
+ - left_wrist: (482.36, 363.64) with score 0.995
185
+ - right_wrist: (398.84, 267.30) with score 0.993
186
+ - left_hip: (435.96, 362.27) with score 0.999
187
+ - right_hip: (388.40, 383.41) with score 0.999
188
+ - left_knee: (460.50, 425.60) with score 0.994
189
+ - right_knee: (403.19, 516.76) with score 0.992
190
+ - left_ankle: (459.31, 558.19) with score 0.893
191
+ - right_ankle: (426.29, 552.55) with score 0.868
192
+ ```
193
+ </details>