I was trying to implement MoveNet tflite pose estimation model on Flutter via https://pub.dev/packages/tflite package. MoveNet lightning model accepts 192x192 image size and uses float32 tensor of shape. I loaded the lightning model and camera streaming like this :
class _AppCameraState extends State<AppCamera> {
late CameraController controller;
bool isDetecting = false;
@override
void initState() {
super.initState();
if (widget.cameras!.length < 1) {
print('No camera is found');
} else {
controller = new CameraController(
widget.cameras![0],
ResolutionPreset.high,
);
controller.initialize().then((_) {
if (!mounted) {
return;
}
setState(() {});
detectPose();
});
}
}
detectPose() {
controller.startImageStream((CameraImage img) {
if (!isDetecting) {
isDetecting = true;
Tflite.runModelOnBinary(binary: imageToByteListFloat32(convertCameraImage(img)!, 192, 127.5, 127.5)).then((recognition) {
widget.setRecognitions!(recognition, 192, 192);
isDetecting = false;
});
}
});
}
@override
void dispose() {
controller.dispose();
super.dispose();
}
Helper functions :
Uint8List imageToByteListFloat32(imgLib.Image image, int inputSize, double mean, double std) {
var convertedBytes = Float32List(1 * inputSize * inputSize * 3);
var buffer = Float32List.view(convertedBytes.buffer);
int pixelIndex = 0;
for (var i = 0; i < inputSize; i++) {
for (var j = 0; j < inputSize; j++) {
var pixel = image.getPixel(j, i);
buffer[pixelIndex++] = (imgLib.getRed(pixel) - mean) / std;
buffer[pixelIndex++] = (imgLib.getGreen(pixel) - mean) / std;
buffer[pixelIndex++] = (imgLib.getBlue(pixel) - mean) / std;
}
}
return convertedBytes.buffer.asUint8List();
}
imageLib.Image? convertCameraImage(CameraImage cameraImage) {
if (cameraImage.format.group == ImageFormatGroup.yuv420) {
return _convertYUV420ToImage(cameraImage);
} else if (cameraImage.format.group == ImageFormatGroup.bgra8888) {
return _convertBGRA8888ToImage(cameraImage);
}
}
imageLib.Image _convertBGRA8888ToImage(CameraImage cameraImage) {
imageLib.Image img = imageLib.Image.fromBytes(cameraImage.planes[0].width!, cameraImage.planes[0].height!, cameraImage.planes[0].bytes,
format: imageLib.Format.bgra);
return img;
}
imageLib.Image _convertYUV420ToImage(CameraImage cameraImage) {
final int width = cameraImage.width;
final int height = cameraImage.height;
final int uvRowStride = cameraImage.planes[1].bytesPerRow;
final int uvPixelStride = cameraImage.planes[1].bytesPerPixel!;
final image = imageLib.Image(width, height);
for (int w = 0; w < width; w++) {
for (int h = 0; h < height; h++) {
final int uvIndex = uvPixelStride * (w / 2).floor() + uvRowStride * (h / 2).floor();
final int index = h * width + w;
final y = cameraImage.planes[0].bytes[index];
final u = cameraImage.planes[1].bytes[uvIndex];
final v = cameraImage.planes[2].bytes[uvIndex];
image.data[index] = _yuv2rgb(y, u, v);
}
}
return image;
}
int _yuv2rgb(int y, int u, int v) {
// Convert yuv pixel to rgb
int r = (y + v * 1436 / 1024 - 179).round();
int g = (y - u * 46549 / 131072 + 44 - v * 93604 / 131072 + 91).round();
int b = (y + u * 1814 / 1024 - 227).round();
// Clipping RGB values to be inside boundaries [ 0 , 255 ]
r = r.clamp(0, 255);
g = g.clamp(0, 255);
b = b.clamp(0, 255);
return 0xff000000 | ((b << 16) & 0xff0000) | ((g << 8) & 0xff00) | (r & 0xff);
}
And output like this below :
Which part should be edit? , thank you.
