本示例看完后，可基本掌握以下内容
1、利用opencv+gstreamer拉流推流的基本步骤
2、可学习gstreamer，图像混合的一些插件用法
3、TX2 NX上的视频编解码与硬件加速，H264编码参数调整
4、linux下如何提高线程优先级
我需要实现的功能是在TX2 NX上，拉取摄像头数据，并在摄像头数据上与另一张图片混合
混合后推流到rtsp服务器。
由于混合的时候需要保留透明度，但是OpenCV不支持四通道的数据写入。硬件加速混合插件无法剔除透明度。当然可以用内存（不是显存，无法硬件加速）中剔除指定颜色的插件实现，但效率太低
所以只能利用VideoCapture先拉到摄像头数据，利用要混合的图片，手动计算叠加。
叠加后利用VideoWriter推流到rtsp 服务器
为提高推流效率，提高了推流线程优先级。
具体流程：
首先搭建rtsp服务端，可利用rtsp-simple-serve这个项目在局域网部署一套
安装gstreamer后
由于用到了rtspclientsink插件，所以需要安装rtsp插件
// sudo apt-get update
// sudo apt-get upgrade
// sudo apt install gstreamer1.0-rtsp
下面是利用VideoWriter推流的过程文章来源：https://www.toymoban.com/news/detail-486266.html
//cpp 实现
#include "rtsp_push_stream.h"
static int get_thread_policy(pthread_attr_t& attr) {
int policy;
int rs = pthread_attr_getschedpolicy(&attr, &policy);
assert(rs == 0);
switch (policy) {
case SCHED_FIFO:
cout << "policy = SCHED_FIFO" << endl;
break;
case SCHED_RR:
cout << "policy = SCHED_RR" << endl;
break;
case SCHED_OTHER:
cout << "policy = SCHED_OTHER" << endl;
break;
default:
cout << "policy = UNKNOWN" << endl;
break;
}
return policy;
}
static void show_thread_priority(pthread_attr_t& attr, int policy) {
int priority = sched_get_priority_max(policy);
assert(priority != -1);
cout << "max_priority = " << priority << endl;
priority = sched_get_priority_min(policy);
assert(priority != -1);
cout << "min_priority = " << priority << endl;
}
static int get_thread_priority(pthread_attr_t& attr) {
struct sched_param param;
int rs = pthread_attr_getschedparam(&attr, ¶m);
assert(rs == 0);
cout << "priority = " << param.__sched_priority << endl;
return param.__sched_priority;
}
static void set_thread_policy(pthread_attr_t& attr, int policy) {
int rs = pthread_attr_setschedpolicy(&attr, policy);
assert(rs == 0);
get_thread_policy(attr);
}
RtspPushStream::RtspPushStream() : active_(true) {}
RtspPushStream::~RtspPushStream() {}
void RtspPushStream::start() {
string appsrcpipeline =
"appsrc ! video/x-raw, format=BGR ! queue ! videoconvert ! video/x-raw,format=RGBA ! nvvidconv ! nvv4l2h264enc "
"! h264parse ! qtmux ! filesink location={filename} sync=false";
//使用 rtspclientsink 需要安装插件
// sudo apt-get update
// sudo apt-get upgrade
// sudo apt install gstreamer1.0-rtsp
std::string pipeline_useglvideomixer =
"appsrc "
"! video/x-raw, format=BGR "
"! videoconvert "
"! video/x-raw,format=(string)RGBA , width=(int)1024, height=(int)600"
"! queue2"
"! alpha method=blue "
"! glvideomixer name = compos sink_0::zorder=1 sink_0::alpha=0.85 sink_1::alpha=1 sink_1::zorder=0 "
"sink_1::width=1024 sink_1::height=600 "
"! nvvidconv"
"! video/x-raw(memory:NVMM), format=(string)I420, width=(int)1024, height=(int)600"
"! nvv4l2h264enc "
"! rtspclientsink location=rtsp://192.168.20.99:8554/my_pipeline"
" nvarguscamerasrc"
"! video/x-raw(memory:NVMM),format=(string)NV12, width=(int)1640, height=(int)1232, framerate=(fraction)25/1"
"! queue2"
"! nvvidconv left=0 right=1640 top=136 bottom=1096 "
"! compos. ";
// nvcompsositor 的两个输入必须是一样的图片格式和内存形式,需要将云图透明部分用白色(255,255,255)填充
std::string pipeline_nvcompsositor =
"appsrc "
"! video/x-raw, format=BGR "
"! videoconvert "
"! video/x-raw,format=(string)RGBA, width=(int)1024, height=(int)600"
"! nvvidconv "
"! queue2"
"! nvcompositor name = compos sink_0::zorder=1 sink_0::alpha=0.5 "
"sink_1::alpha=1 "
"sink_1::zorder=0 sink_1::width=1024 sink_1::height=600 "
"! nvvidconv "
"! nvv4l2h264enc "
"! rtspclientsink location=rtsp://192.168.20.99:8554/my_pipeline"
" nvarguscamerasrc "
"! video/x-raw(memory:NVMM), format=(string)NV12, width=(int)1640, height=(int)1232,framerate=(fraction)25/1 "
"! nvvidconv left=0 right=1640 top=136 bottom=1096 "
"! video/x-raw,format=(string)RGBA, width=(int)1024, height=(int)600 "
"! videobalance brightness=0.3 "
"! nvvidconv "
"! queue2"
"! compos. ";
video_writer_.open(pipeline_nvcompsositor, cv::CAP_GSTREAMER, 0, 25, cv::Size(1024, 600));
mat_ = cv::imread("test.jpg");
write_thread_ = make_shared<thread>(&RtspPushStream::run, this);
write_thread_->join();
}
void RtspPushStream::run() {
int id = 0;
while (active_) {
space_mat_ = cv::Mat(600, 1024, CV_8UC3, cv::Scalar(0, 0, 0));
video_writer_.write(space_mat_);
std::this_thread::sleep_for(std::chrono::milliseconds(3));
}
}
void RtspPushStream::end() {
active_ = false;
//LOG::info("RtspPushStream::end()!");
}
void RtspPushStream::start_capture() {
std::string pipeline_camear_capture =
" nvarguscamerasrc "
"! video/x-raw(memory:NVMM), format=(string)NV12, width=(int)1640, "
"height=(int)1232,framerate=(fraction)30/1 "
"! nvvidconv left=0 right=1640 top=136 bottom=1096 "
"! video/x-raw,format=(string)I420, width=(int)1024, height=(int)600 "
"! videoconvert "
"! video/x-raw,format=(string)BGR "
"! appsink";
video_capture_.open(pipeline_camear_capture, cv::CAP_GSTREAMER);
if (!video_capture_.isOpened()) {
//LOG::error("Failed to open VideoCapture");
return;
}
mat_ = cv::imread("test.jpg");
std::string pipeline_video_writer =
"appsrc "
"! video/x-raw, format=BGR "
"! videoconvert "
"! nvvidconv "
"! nvv4l2h264enc iframeinterval=10 idrinterval=32 " //这里增加H264 i帧与idr帧次数，降低webrtc拉流时卡顿的频率和时间
"! rtspclientsink location=rtsp://192.168.20.99:8554/my_pipeline";
video_writer_.open(pipeline_video_writer, cv::CAP_GSTREAMER, 0, 20, cv::Size(1024, 600));
cap_thread_ = make_shared<thread>(&RtspPushStream::run_capture, this);
cap_thread_->detach();
}
void RtspPushStream::run_capture() {
pthread_attr_t attr;
struct sched_param sched;
int rs;
rs = pthread_attr_init(&attr);
assert(rs == 0);
int policy = get_thread_policy(attr);
cout << "Show current configuration of priority" << endl;
show_thread_priority(attr, policy);
cout << "Show priority of current thread" << endl;
int priority = get_thread_priority(attr);
set_thread_policy(attr, SCHED_FIFO); // SCHED_RR
cout << "Restore current policy" << endl;
// set_thread_policy(attr, policy);
struct sched_param param_;
param_.sched_priority = sched_get_priority_max(SCHED_FIFO); // SCHED_RR
//这里设置线程优先级
cout << "param_.sched_priority = " << param_.sched_priority << endl;
pthread_setschedparam(pthread_self(), SCHED_FIFO, ¶m_);
cout << "pthread_setschedparam success" << endl;
rs = pthread_attr_destroy(&attr);
assert(rs == 0);
cout << "start capture!!!!!!!!!" << endl;
int id = 0;
cv::Mat src;
while (active_) {
//读取摄像头内容，也就是从摄像头拉流
video_capture_ >> src;
//在这里自定义融合算法就行了
//blender
// 两张图的透明度融合算法，取rgb像素进行融合
// data1[add_col] = data2[add_col] * 0.4 + data1[add_col] * 0.6;
// data1[add_col + 1] = data2[add_col + 1] * 0.4 + data1[add_col + 1] * 0.6;
// data1[add_col + 2] = data2[add_col + 2] * 0.4 + data1[add_col + 2] * 0.6;
//融合后，可以利用opencv给src写入当前时间用来观察
auto cur_time = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
std::stringstream ss;
ss << std::put_time(std::localtime(&cur_time), "%Y-%m-%d %H:%M:%S");
std::string str_time = ss.str();
cv::putText(src, str_time, cv::Point(830, 20), cv::HersheyFonts::FONT_ITALIC, 0.5, cv::Scalar(0, 0, 0), 1,
4);
video_writer_.write(src);
src.release();
}
//LOG::info("RtspPushStream::run_capture() end!");
}
//头文件
#ifndef RTSHPUSHSTREAM_H
#define RTSHPUSHSTREAM_H
#include <chrono>
#include <iostream>
#include <memory>
#include <mutex>
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/videoio/videoio.hpp>
#include <string>
#include <thread>
#include <vector>
using namespace std;
class RtspPushStream {
public:
RtspPushStream();
~RtspPushStream();
void start();
void write_image(cv::Mat image);
void run();
void start_capture();
void run_capture();
void end();
private:
mutex mutex_;
cv::VideoWriter video_writer_;
list<cv::Mat> img_mats_;
bool active_;
cv::Mat mat_;
cv::Mat space_mat_;
shared_ptr<thread> cap_thread_;
shared_ptr<thread> write_thread_;
cv::VideoCapture video_capture_;
std::vector<cv::Mat> push_frames_;
std::mutex push_frames_lock_;
};
#endif
推流后用相关视频软件拉流就能看到效果，如PotPlayer 64 bit，VLC等等文章来源地址https://www.toymoban.com/news/detail-486266.html
到了这里，关于使用Gstreamer+OpenCV实现两路图像数据混合拉流推流的文章就介绍完了。如果您还想了解更多内容，请在右上角搜索TOY模板网以前的文章或继续浏览下面的相关文章，希望大家以后多多支持TOY模板网！
