下面是使用 opencv-camera，实时处理区域内手势关键点检测 android 推理 demo。
首先是整合 opcv-camera 进去：
为了方便直接将整个 opencv-android-sdk 全部导入：
然后在原来的项目模块app中添加 opencv的 java 相关依赖，主要添加红色两行：
app/build.grandle
dependencies {
implementation fileTree(dir: 'libs', include: ['*.jar'])
implementation 'androidx.appcompat:appcompat:1.4.1'
implementation 'com.google.android.material:material:1.5.0'
implementation 'androidx.constraintlayout:constraintlayout:2.1.3'
testImplementation 'junit:junit:4.13.2'
androidTestImplementation 'androidx.test.ext:junit:1.1.3'
androidTestImplementation 'androidx.test.espresso:espresso-core:3.4.0'
implementation project(':opencvsdk')
}最后在项目中要使用opencv的地方加载jni库，可以添加到 MainActivity 中：
System.loadLibrary("opencv_java4"); 或者 OpenCVLoader.initDebug();
要使用 opencv-camera，MainActivity 继承 CameraActivity，然后在回调函数中获取每一帧进行处理，比如下面对每一帧添加识别区域边框：
// 获取每一帧回调数据
private CameraBridgeViewBase.CvCameraViewListener2 cameraViewListener2 = new CameraBridgeViewBase.CvCameraViewListener2() {
@Override
public void onCameraViewStarted(int width, int height) {
System.out.println("开始预览 width="+width+",height="+height);
// 根据模型输入和预览界面中心点计算识别区域
int detection_x1 = (win_w - OnnxUtil.w)/2;
int detection_x2 = (win_w - OnnxUtil.w)/2 + OnnxUtil.w;
int detection_y1 = (win_h - OnnxUtil.h)/2;
int detection_y2 = (win_h - OnnxUtil.h)/2 + OnnxUtil.h;;
// 缓存识别区域两个点
detection_p1 = new Point(detection_x1,detection_y1);
detection_p2 = new Point(detection_x2,detection_y2);
detection_box_color = new Scalar(255, 0, 0);
detection_box_tickness = 1;
}
@Override
public void onCameraViewStopped() {}
@Override
public Mat onCameraFrame(CameraBridgeViewBase.CvCameraViewFrame frame) {
// 获取 cv::Mat
Mat mat = frame.rgba();
// 标注识别区域
Imgproc.rectangle(mat, detection_p1, detection_p2,detection_box_color,detection_box_tickness);
Imgproc.putText(mat,"area",detection_p1,Imgproc.FONT_HERSHEY_SIMPLEX,0.3,detection_box_color);
// 推理并标注
OnnxUtil.inference(mat,detection_p1,detection_p2);
return mat;
}
};在界面中开启预览：
// 开启预览
private BaseLoaderCallback baseLoaderCallback = new BaseLoaderCallback(this) {
@Override
public void onManagerConnected(int status) {
switch (status) {
case LoaderCallbackInterface.SUCCESS: {
if (camera2View != null) {
// 设置前置还是后置摄像头 0后置 1前置
camera2View.setCameraIndex(cameraId);
// 注册每一帧回调
camera2View.setCvCameraViewListener(cameraViewListener2);
// 显示/关闭 帧率 disableFpsMeter/enableFpsMeter
// 要修改字体和颜色直接修改 FpsMeter 类即可
camera2View.enableFpsMeter();
// 设置视图宽高和模型一致减少resize操作,模型输入一般尺寸不大,这样相机渲染fps会更高
camera2View.setMaxFrameSize(win_w,win_h);
// 开启
camera2View.enableView();
}
}
break;
default:
super.onManagerConnected(status);
break;
}
}
};下面是全部推理 MainActivity 代码：
package com.example.camera_opencv;
import android.content.pm.ActivityInfo;
import android.os.Bundle;
import android.view.WindowManager;
import com.example.camera_opencv.databinding.ActivityMainBinding;
import org.opencv.android.*;
import org.opencv.core.Mat;
import org.opencv.core.Point;
import org.opencv.core.Scalar;
import org.opencv.imgproc.Imgproc;
import java.util.Arrays;
import java.util.List;
public class MainActivity extends CameraActivity{
// 动态库
static {
// 我们自己的jni
System.loadLibrary("camera_opencv");
// 新加的 opencv 的jni
System.loadLibrary("opencv_java4");
}
private ActivityMainBinding binding;
// 预览界面
private JavaCamera2View camera2View;
// 相机编号 0后置 1前置
private int cameraId = 1;
// 设置预览界面宽高,在次宽高基础上限制识别区域
private int win_w = 320;
private int win_h = 240;
// 识别区域两个点
private Point detection_p1;
private Point detection_p2;
private Scalar detection_box_color;
private int detection_box_tickness;
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
binding = ActivityMainBinding.inflate(getLayoutInflater());
setContentView(binding.getRoot());
// 加载模型
OnnxUtil.loadModule(getAssets());
// 强制横屏
setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_LANDSCAPE);
// 隐藏上方状态栏
getWindow().setFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN, WindowManager.LayoutParams.FLAG_FULLSCREEN);
// 预览界面
camera2View = findViewById(R.id.camera_view);
}
@Override
protected List<? extends CameraBridgeViewBase> getCameraViewList() {
return Arrays.asList(camera2View);
}
@Override
public void onPause() {
super.onPause();
if (camera2View != null) {
// 关闭预览
camera2View.disableView();
}
}
@Override
public void onResume() {
super.onResume();
if (OpenCVLoader.initDebug()) {
baseLoaderCallback.onManagerConnected(LoaderCallbackInterface.SUCCESS);
} else {
OpenCVLoader.initAsync(OpenCVLoader.OPENCV_VERSION, this, baseLoaderCallback);
}
}
// 获取每一帧回调数据
private CameraBridgeViewBase.CvCameraViewListener2 cameraViewListener2 = new CameraBridgeViewBase.CvCameraViewListener2() {
@Override
public void onCameraViewStarted(int width, int height) {
System.out.println("开始预览 width="+width+",height="+height);
// 根据模型输入和预览界面中心点计算识别区域
int detection_x1 = (win_w - OnnxUtil.w)/2;
int detection_x2 = (win_w - OnnxUtil.w)/2 + OnnxUtil.w;
int detection_y1 = (win_h - OnnxUtil.h)/2;
int detection_y2 = (win_h - OnnxUtil.h)/2 + OnnxUtil.h;;
// 缓存识别区域两个点
detection_p1 = new Point(detection_x1,detection_y1);
detection_p2 = new Point(detection_x2,detection_y2);
detection_box_color = new Scalar(255, 0, 0);
detection_box_tickness = 1;
}
@Override
public void onCameraViewStopped() {}
@Override
public Mat onCameraFrame(CameraBridgeViewBase.CvCameraViewFrame frame) {
// 获取 cv::Mat
Mat mat = frame.rgba();
// 标注识别区域
Imgproc.rectangle(mat, detection_p1, detection_p2,detection_box_color,detection_box_tickness);
Imgproc.putText(mat,"area",detection_p1,Imgproc.FONT_HERSHEY_SIMPLEX,0.3,detection_box_color);
// 推理并标注
OnnxUtil.inference(mat,detection_p1,detection_p2);
return mat;
}
};
// 开启预览
private BaseLoaderCallback baseLoaderCallback = new BaseLoaderCallback(this) {
@Override
public void onManagerConnected(int status) {
switch (status) {
case LoaderCallbackInterface.SUCCESS: {
if (camera2View != null) {
// 设置前置还是后置摄像头 0后置 1前置
camera2View.setCameraIndex(cameraId);
// 注册每一帧回调
camera2View.setCvCameraViewListener(cameraViewListener2);
// 显示/关闭 帧率 disableFpsMeter/enableFpsMeter
// 要修改字体和颜色直接修改 FpsMeter 类即可
camera2View.enableFpsMeter();
// 设置视图宽高和模型一致减少resize操作,模型输入一般尺寸不大,这样相机渲染fps会更高
camera2View.setMaxFrameSize(win_w,win_h);
// 开启
camera2View.enableView();
}
}
break;
default:
super.onManagerConnected(status);
break;
}
}
};
}onnx 模型加载和推理代码：
使用的微软onnx推理框架：
implementation 'com.microsoft.onnxruntime:onnxruntime-android:latest.release'
implementation 'com.microsoft.onnxruntime:onnxruntime-extensions-android:latest.release'
package com.example.camera_opencv;
import ai.onnxruntime.*;
import android.content.res.AssetManager;
import org.opencv.core.*;
import org.opencv.imgproc.Imgproc;
import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.nio.FloatBuffer;
import java.util.*;
import java.util.concurrent.atomic.AtomicBoolean;
public class OnnxUtil {
// onnxruntime 环境
public static OrtEnvironment env;
public static OrtSession session;
// 模型输入
public static int w = 0;
public static int h = 0;
public static int c = 3;
// 标注颜色
public static Scalar green = new Scalar(0, 255, 0);
public static Scalar red = new Scalar(255, 0, 0);
public static int radius = 1;
public static int tickness = 2;
// 模型加载
public static void loadModule(AssetManager assetManager){
w = 224;
h = 224;
c = 3;
try {
InputStream inputStream = assetManager.open("hand_landmark_sparse_Nx3x224x224.onnx");
ByteArrayOutputStream buffer = new ByteArrayOutputStream();
int nRead;
byte[] data = new byte[1024];
while ((nRead = inputStream.read(data, 0, data.length)) != -1) {
buffer.write(data, 0, nRead);
}
buffer.flush();
byte[] module = buffer.toByteArray();
System.out.println("开始加载模型");
env = OrtEnvironment.getEnvironment();
session = env.createSession(module, new OrtSession.SessionOptions());
session.getInputInfo().entrySet().stream().forEach(n -> {
String inputName = n.getKey();
NodeInfo inputInfo = n.getValue();
long[] shape = ((TensorInfo) inputInfo.getInfo()).getShape();
String javaType = ((TensorInfo) inputInfo.getInfo()).type.toString();
System.out.println("模型输入: "+inputName + " -> " + Arrays.toString(shape) + " -> " + javaType);
});
session.getOutputInfo().entrySet().stream().forEach(n -> {
String outputName = n.getKey();
NodeInfo outputInfo = n.getValue();
long[] shape = ((TensorInfo) outputInfo.getInfo()).getShape();
String javaType = ((TensorInfo) outputInfo.getInfo()).type.toString();
System.out.println("模型输出: "+outputName + " -> " + Arrays.toString(shape) + " -> " + javaType);
});
} catch (Exception e) {
e.printStackTrace();
}
}
// 模型推理,输入原始图片和识别区域两个点
public static void inference(Mat mat,Point detection_p1,Point detection_p2){
int px = Double.valueOf(detection_p1.x).intValue();
int py = Double.valueOf(detection_p1.y).intValue();
// 提取rgb(chw存储)并做归一化,也就是 rgb rgb rgb rgb 顺序
float[] chw = new float[c*h*w];
// 像素点索引
int pi = 0;
for(int j=0 ; j<h ; j++){
for(int i=0 ; i<w ; i++){
// 第j行,第i列,根据识别区域p1得到xy坐标的偏移,直接加就行
double[] rgb = mat.get(j+py,i+px);
// 缓存到 chw 中,mat 是 rgba 数据对应的下标 2103
chw[pi] = (float)(rgb[2]/255);//r
chw[pi + w * h * 1 ] = (float)(rgb[1]/255);//G
chw[pi + w * h * 2 ] = (float)(rgb[0]/255);//b
pi ++;
}
}
// 创建张量并进行推理
try {
// ---------模型输入-----------
// input -> [-1, 3, 224, 224] -> FLOAT
// ---------模型输出-----------
// xyz_x21 -> [-1, 63] -> FLOAT
// hand_score -> [-1, 1] -> FLOAT
// lefthand_0_or_righthand_1 -> [-1, 1] -> FLOAT
OnnxTensor tensor = OnnxTensor.createTensor(env, FloatBuffer.wrap(chw), new long[]{1,c,h,w});
OrtSession.Result out = session.run(Collections.singletonMap("input", tensor));
// 21点
// xyz_x21 -> [-1, 63] -> FLOAT 21点,三位坐标
float[][] xyz_x21 = ((float[][])(out.get(0)).getValue());
// hand_score -> [-1, 1] -> FLOAT 置信度,手的置信度
float[][] hand_score = ((float[][])(out.get(1)).getValue());
// lefthand_0_or_righthand_1 -> [-1, 1] -> FLOAT 左手0右手1
float[][] lefthand_0_or_righthand_1 = ((float[][])(out.get(2)).getValue());
// 手的个数
int count = xyz_x21.length;
// 遍历每个手
for(int i=0;i<count;i++){
// 手的xyz信息
float[] xyz = xyz_x21[i];
// 手的置信度
float score = hand_score[i][0];
// 左右
float lr = lefthand_0_or_righthand_1[i][0];
// 置信度
if(score >= 0.5){
// 共21个点
float point1_x = xyz[0]+px;
float point1_y = xyz[1]+py;
float point2_x = xyz[3]+px;
float point2_y = xyz[4]+py;
float point3_x = xyz[6]+px;
float point3_y = xyz[7]+py;
float point4_x = xyz[9]+px;
float point4_y = xyz[10]+py;
float point5_x = xyz[12]+px;
float point5_y = xyz[13]+py;
float point6_x = xyz[15]+px;
float point6_y = xyz[16]+py;
float point7_x = xyz[18]+px;
float point7_y = xyz[19]+py;
float point8_x = xyz[21]+px;
float point8_y = xyz[22]+py;
float point9_x = xyz[24]+px;
float point9_y = xyz[25]+py;
float point10_x = xyz[27]+px;
float point10_y = xyz[28]+py;
float point11_x = xyz[30]+px;
float point11_y = xyz[31]+py;
float point12_x = xyz[33]+px;
float point12_y = xyz[34]+py;
float point13_x = xyz[36]+px;
float point13_y = xyz[37]+py;
float point14_x = xyz[39]+px;
float point14_y = xyz[40]+py;
float point15_x = xyz[42]+px;
float point15_y = xyz[43]+py;
float point16_x = xyz[45]+px;
float point16_y = xyz[46]+py;
float point17_x = xyz[48]+px;
float point17_y = xyz[49]+py;
float point18_x = xyz[51]+px;
float point18_y = xyz[52]+py;
float point19_x = xyz[54]+px;
float point19_y = xyz[55]+py;
float point20_x = xyz[57]+px;
float point20_y = xyz[58]+py;
float point21_x = xyz[60]+px;
float point21_y = xyz[61]+py;
// 标注点
Imgproc.circle(mat,new Point(point1_x,point1_y),radius,red,tickness);
Imgproc.circle(mat,new Point(point2_x,point2_y),radius,red,tickness);
Imgproc.circle(mat,new Point(point3_x,point3_y),radius,red,tickness);
Imgproc.circle(mat,new Point(point4_x,point4_y),radius,red,tickness);
Imgproc.circle(mat,new Point(point5_x,point5_y),radius,red,tickness);
Imgproc.circle(mat,new Point(point6_x,point6_y),radius,red,tickness);
Imgproc.circle(mat,new Point(point7_x,point7_y),radius,red,tickness);
Imgproc.circle(mat,new Point(point8_x,point8_y),radius,red,tickness);
Imgproc.circle(mat,new Point(point9_x,point9_y),radius,red,tickness);
Imgproc.circle(mat,new Point(point10_x,point10_y),radius,red,tickness);
Imgproc.circle(mat,new Point(point11_x,point11_y),radius,red,tickness);
Imgproc.circle(mat,new Point(point12_x,point12_y),radius,red,tickness);
Imgproc.circle(mat,new Point(point13_x,point13_y),radius,red,tickness);
Imgproc.circle(mat,new Point(point14_x,point14_y),radius,red,tickness);
Imgproc.circle(mat,new Point(point15_x,point15_y),radius,red,tickness);
Imgproc.circle(mat,new Point(point16_x,point16_y),radius,red,tickness);
Imgproc.circle(mat,new Point(point17_x,point17_y),radius,red,tickness);
Imgproc.circle(mat,new Point(point18_x,point18_y),radius,red,tickness);
Imgproc.circle(mat,new Point(point19_x,point19_y),radius,red,tickness);
Imgproc.circle(mat,new Point(point20_x,point20_y),radius,red,tickness);
Imgproc.circle(mat,new Point(point21_x,point21_y),radius,red,tickness);
// 标注连线 手指 1-2 2-3 3-4 4-5
Imgproc.line(mat,new Point(point1_x,point1_y),new Point(point2_x,point2_y),green);
Imgproc.line(mat,new Point(point2_x,point2_y),new Point(point3_x,point3_y),green);
Imgproc.line(mat,new Point(point3_x,point3_y),new Point(point4_x,point4_y),green);
Imgproc.line(mat,new Point(point4_x,point4_y),new Point(point5_x,point5_y),green);
// 标注连线 手指 6-7 7-8 8-9
Imgproc.line(mat,new Point(point6_x,point6_y),new Point(point7_x,point7_y),green);
Imgproc.line(mat,new Point(point7_x,point7_y),new Point(point8_x,point8_y),green);
Imgproc.line(mat,new Point(point8_x,point8_y),new Point(point9_x,point9_y),green);
// 标注连线 手指 10-11 11-12 12-13
Imgproc.line(mat,new Point(point10_x,point10_y),new Point(point11_x,point11_y),green);
Imgproc.line(mat,new Point(point11_x,point11_y),new Point(point12_x,point12_y),green);
Imgproc.line(mat,new Point(point12_x,point12_y),new Point(point13_x,point13_y),green);
// 标注连线 手指 14-15 15-16 16-17
Imgproc.line(mat,new Point(point14_x,point14_y),new Point(point15_x,point15_y),green);
Imgproc.line(mat,new Point(point15_x,point15_y),new Point(point16_x,point16_y),green);
Imgproc.line(mat,new Point(point16_x,point16_y),new Point(point17_x,point17_y),green);
// 标注连线 手指 18-19 19-20 20-21
Imgproc.line(mat,new Point(point18_x,point18_y),new Point(point19_x,point19_y),green);
Imgproc.line(mat,new Point(point19_x,point19_y),new Point(point20_x,point20_y),green);
Imgproc.line(mat,new Point(point20_x,point20_y),new Point(point21_x,point21_y),green);
// 标注连线 手掌 3-6 6-10 10-14 14-18 1-6 1-10 1-14 1-18
Imgproc.line(mat,new Point(point3_x,point3_y),new Point(point6_x,point6_y),green);
Imgproc.line(mat,new Point(point6_x,point6_y),new Point(point10_x,point10_y),green);
Imgproc.line(mat,new Point(point10_x,point10_y),new Point(point14_x,point14_y),green);
Imgproc.line(mat,new Point(point14_x,point14_y),new Point(point18_x,point18_y),green);
Imgproc.line(mat,new Point(point1_x,point1_y),new Point(point6_x,point6_y),green);
Imgproc.line(mat,new Point(point1_x,point1_y),new Point(point10_x,point10_y),green);
Imgproc.line(mat,new Point(point1_x,point1_y),new Point(point14_x,point14_y),green);
Imgproc.line(mat,new Point(point1_x,point1_y),new Point(point18_x,point18_y),green);
// 标注左右
}
}
// 关闭tensor
tensor.close();
out.close();
}
catch (Exception e){
e.printStackTrace();
}
}
}
项目详细代码：文章来源：https://www.toymoban.com/news/detail-847089.html
https://github.com/TangYuFan/deeplearn-mobile文章来源地址https://www.toymoban.com/news/detail-847089.html
到了这里，关于android 使用 onnxruntime 部署 hand_3d_landmark 手势关键点检测的文章就介绍完了。如果您还想了解更多内容，请在右上角搜索TOY模板网以前的文章或继续浏览下面的相关文章，希望大家以后多多支持TOY模板网！
