ffmpeg 硬件解码
ffmpeg硬件解码可以使用最新的vulkan来做,基本上来说,不挑操作系统是比较重要的,如果直接使用cuda也是非常好的选择。
AVPixelFormat sourcepf = AV_PIX_FMT_NV12;// AV_PIX_FMT_NV12;// AV_PIX_FMT_YUV420P;
AVPixelFormat destpf = AV_PIX_FMT_YUV420P; //AV_PIX_FMT_BGR24
AVBufferRef* hw_device_ctx = NULL;
下面的class 是我封装了一些功能,只是示例,读者可以自行修改,读的是rtsp 流,后面准备推流到gb28181上。
class c_AVDecoder:public TThreadRunable
{
AVPixelFormat sourcepf = AV_PIX_FMT_NV12;// AV_PIX_FMT_NV12;// AV_PIX_FMT_YUV420P;
AVPixelFormat destpf = AV_PIX_FMT_YUV420P; //AV_PIX_FMT_BGR24
AVBufferRef* hw_device_ctx = NULL;
SDLDraw v_draw;
int hw_decoder_init(AVCodecContext* ctx, const enum AVHWDeviceType type)
{
int err = 0;
if ((err = av_hwdevice_ctx_create(&hw_device_ctx, type,
NULL, NULL, 0)) < 0) {
fprintf(stderr, "Failed to create specified HW device.\n");
return err;
}
ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx);
return err;
}
//SDLDraw g_draw;
struct SwsContext* img_convert_ctx = NULL;
AVFormatContext* input_ctx = NULL;
int video_stream, ret;
AVStream* video = NULL;
AVCodecContext* decoder_ctx = NULL;
AVCodec* decoder = NULL;
AVPacket packet;
enum AVHWDeviceType type;
//转换成yuv420 或者rgb
int decode_write(AVCodecContext* avctx, AVPacket* packet)
{
AVFrame* frame = NULL, * sw_frame = NULL;
AVFrame* tmp_frame = NULL;
//AVFrame* pFrameDst = NULL;
unsigned char* out_buffer = NULL;
int ret = 0;
ret = avcodec_send_packet(avctx, packet);
if (ret < 0) {
fprintf(stderr, "Error during decoding\n");
return ret;
}
if (img_convert_ctx == NULL)
{
img_convert_ctx = sws_getContext(avctx->width, avctx->height, sourcepf,
WIDTH, HEIGHT, destpf, SWS_FAST_BILINEAR, NULL, NULL, NULL);
}
static int i = 0;
while (1) {
if (!(frame = av_frame_alloc()) || !(sw_frame = av_frame_alloc())) {
fprintf(stderr, "Can not alloc frame\n");
ret = AVERROR(ENOMEM);
goto fail;
}
//avctx->get_buffer2
ret = avcodec_receive_frame(avctx, frame);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
av_frame_free(&frame);
av_frame_free(&sw_frame);
return 0;
}
else if (ret < 0) {
fprintf(stderr, "Error while decoding\n");
goto fail;
}
if (frame->format == hw_pix_fmt) {
/* retrieve data from GPU to CPU */
sw_frame->format = sourcepf; // AV_PIX_FMT_NV12;// AV_PIX_FMT_YUV420P;// AV_PIX_FMT_NV12;// AV_PIX_FMT_BGR24;// AV_PIX_FMT_YUV420P;
if ((ret = av_hwframe_transfer_data(sw_frame, frame, 0)) < 0) {
fprintf(stderr, "Error transferring the data to system memory\n");
goto fail;
}
//av_frame_copy_props(sw_frame, frame);
tmp_frame = sw_frame;
}
else
tmp_frame = frame;
#if 0
if (out_buffer == NULL)
{
out_buffer = (unsigned char*)av_malloc(av_image_get_buffer_size(destpf,
WIDTH,
HEIGHT,
1));
//av_image_alloc()
av_image_fill_arrays(pFrameDst.data, pFrameDst.linesize, out_buffer,
destpf, WIDTH, HEIGHT, 1);
}
#endif
AVFrame* pFrameDst = av_frame_alloc();
av_image_alloc(pFrameDst->data, pFrameDst->linesize, WIDTH, HEIGHT, destpf, 1);
#if 1
sws_scale(img_convert_ctx, tmp_frame->data, tmp_frame->linesize,
0, avctx->height, pFrameDst->data, pFrameDst->linesize);
#endif
//cout << "into " << i++ << endl;
#if 1
if (!v_draw.push(pFrameDst))
{
av_freep(&pFrameDst->data[0]);
av_frame_free(&pFrameDst);
}
#endif
#if 0
size = av_image_get_buffer_size((AVPixelFormat)tmp_frame->format, tmp_frame->width,
tmp_frame->height, 1);
buffer = (uint8_t*)av_malloc(size);
if (!buffer) {
fprintf(stderr, "Can not alloc buffer\n");
ret = AVERROR(ENOMEM);
goto fail;
}
ret = av_image_copy_to_buffer(buffer, size,
(const uint8_t* const*)tmp_frame->data,
(const int*)tmp_frame->linesize, (AVPixelFormat)tmp_frame->format,
tmp_frame->width, tmp_frame->height, 1);
if (ret < 0) {
fprintf(stderr, "Can not copy image to buffer\n");
goto fail;
}
if ((ret = (int)fwrite(buffer, 1, size, output_file)) < 0) {
fprintf(stderr, "Failed to dump raw data.\n");
goto fail;
}
#endif
fail:
av_frame_free(&frame);
av_frame_free(&sw_frame);
//av_freep(&buffer);
if (ret < 0)
return ret;
}
}
int func_init(const char* url)
{
const char* stype = "dxva2";
type = av_hwdevice_find_type_by_name(stype);
if (type == AV_HWDEVICE_TYPE_NONE) {
fprintf(stderr, "Device type %s is not supported.\n", stype);
fprintf(stderr, "Available device types:");
while ((type = av_hwdevice_iterate_types(type)) != AV_HWDEVICE_TYPE_NONE)
fprintf(stderr, " %s", av_hwdevice_get_type_name(type));
fprintf(stderr, "\n");
return -1;
}
/* open the input file */
//const char * filename = "h:/video/a.mp4";
AVDictionary* opts = NULL;
av_dict_set(&opts, "rtsp_transport", "tcp", 0);
av_dict_set(&opts, "buffer_size", "1048576", 0);
av_dict_set(&opts, "fpsprobesize", "5", 0);
av_dict_set(&opts, "analyzeduration", "5000000", 0);
if (avformat_open_input(&input_ctx, url, NULL, &opts) != 0) {
fprintf(stderr, "Cannot open input file '%s'\n", url);
return -1;
}
if (avformat_find_stream_info(input_ctx, NULL) < 0) {
fprintf(stderr, "Cannot find input stream information.\n");
return -1;
}
/* find the video stream information */
ret = av_find_best_stream(input_ctx, AVMEDIA_TYPE_VIDEO, -1, -1, &decoder, 0);
if (ret < 0) {
fprintf(stderr, "Cannot find a video stream in the input file\n");
return -1;
}
video_stream = ret;
AVStream* stream = input_ctx->streams[video_stream];
float frame_rate = stream->avg_frame_rate.num / stream->avg_frame_rate.den;//每秒多少帧
std::cout << "frame_rate is:" << frame_rate << std::endl;
//优化直接定死
/*for (int i = 0;; i++) {
const AVCodecHWConfig* config = avcodec_get_hw_config(decoder, i);
if (!config) {
fprintf(stderr, "Decoder %s does not support device type %s.\n",
decoder->name, av_hwdevice_get_type_name(type));
return -1;
}
if (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX &&
config->device_type == type) {
hw_pix_fmt = config->pix_fmt;
break;
}
}*/
if (!(decoder_ctx = avcodec_alloc_context3(decoder)))
return AVERROR(ENOMEM);
video = input_ctx->streams[video_stream];
if (avcodec_parameters_to_context(decoder_ctx, video->codecpar) < 0)
return -1;
decoder_ctx->get_format = get_hw_format;
if (hw_decoder_init(decoder_ctx, type) < 0)
return -1;
if ((ret = avcodec_open2(decoder_ctx, decoder, NULL)) < 0) {
fprintf(stderr, "Failed to open codec for stream #%u\n", video_stream);
return -1;
}
/* open the file to dump raw data */
//output_file = fopen(argv[3], "w+");
v_draw.init(1280, 720, (int)frame_rate);
v_draw.Start();
/* actual decoding and dump the raw data */
while (ret >= 0) {
if ((ret = av_read_frame(input_ctx, &packet)) < 0)
break;
if (video_stream == packet.stream_index)
{
//这里需要更加精确的计算
ret = decode_write(decoder_ctx, &packet);
//SDL_Delay(1000 / frame_rate);
}
av_packet_unref(&packet);
if (IsStop())
break;
}
/* flush the decoder */
packet.data = NULL;
packet.size = 0;
ret = decode_write(decoder_ctx, &packet);
av_packet_unref(&packet);
avcodec_free_context(&decoder_ctx);
avformat_close_input(&input_ctx);
av_buffer_unref(&hw_device_ctx);
return 0;
}
public:
void Run()
{
while (1)
{
if(!IsStop())
func_init(rtspurl);
}
}
};
使用sdl来绘制画面
首先声明,使用sdl来渲染画面并不是有多好,如果可以,可以使用新的绘制方式,比如直接使用opengl,直接使用vulkan来绘制是更好的,sdl封装了opengl,d3d等绘制方式,但是也失去了灵活性,当然,你从头到尾读遍了源码,另当别论,如果要使用均值处理函数来做消除百叶窗效果,摩尔纹,还是直接使用opengl 渲染更为简单,使用glsl语言就行了,甚至要使用非线性函数来增亮图形,最好也是直接使用opengl来绘制。
#pragma once
#include<chrono>
#define __STDC_CONSTANT_MACROS
#define SDL_MAIN_HANDLED
extern "C"
{
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libavutil/pixdesc.h>
#include <libavutil/hwcontext.h>
#include <libavutil/opt.h>
#include <libavutil/avassert.h>
#include <libavutil/imgutils.h>
#include <libswscale/swscale.h>
#include "SDL2\SDL.h"
};
#include "c_ringbuffer.h"
#include "TThreadRunable.h"
#include "TYUVMerge.h"
#define SFM_REFRESH_EVENT (SDL_USEREVENT + 1)
#define SFM_BREAK_EVENT (SDL_USEREVENT + 2)
#define WIDTH 640
#define HEIGHT 360
typedef struct
{
int g_fps = 40;
int thread_exit = 0;
int thread_pause = 0;
}s_param;
static int sfp_refresh_thread(void *opaque)
{
s_param * param = (s_param*)opaque;
param->thread_exit = 0;
param->thread_pause = 0;
while (!param->thread_exit)
{
if (!param->thread_pause)
{
SDL_Event event;
event.type = SFM_REFRESH_EVENT;
SDL_PushEvent(&event);
}
SDL_Delay(1000 / param->g_fps);
}
param->thread_exit = 0;
param->thread_pause = 0;
//Break
SDL_Event event;
event.type = SFM_BREAK_EVENT;
SDL_PushEvent(&event);
return 0;
}
class TicToc
{
public:
TicToc()
{
tic();
}
void tic()
{
start = std::chrono::system_clock::now();
}
double toc()
{
end = std::chrono::system_clock::now();
std::chrono::duration<double> elapsed_seconds = end - start;
return elapsed_seconds.count() * 1000;
}
private:
std::chrono::time_point<std::chrono::system_clock> start, end;
};
class SDLDraw :public TThreadRunable//实际上这里可以编码发送出去
{
int m_w = 0, m_h = 0;
SDL_Window *screen = NULL;
SDL_Renderer *sdlRenderer = NULL;
SDL_Texture *sdlTexture = NULL;
SDL_Rect sdlRect;
SDL_Thread *video_tid;
SDL_Event event;
//struct SwsContext *img_convert_ctx = NULL;
bool m_window_init = false;
lfringqueue<AVFrame, 20> v_frames;
lfringqueue<AVFrame, 20> v_frames_1;
lfringqueue<AVFrame, 20> v_frames_2;
s_param v_param;
//画布
AVFrame * v_canvas_frame = NULL;
public:
void init(int w, int h,int fps)
{
if (w != m_w || h != m_h)
{
m_w = w;
m_h = h;
if (v_canvas_frame != NULL)
{
func_uninit();
}
}
v_param.g_fps = fps;
//这是背景画布
if (v_canvas_frame == NULL)
{
v_canvas_frame = av_frame_alloc();
av_image_alloc(v_canvas_frame->data, v_canvas_frame->linesize, 1280, 720, AV_PIX_FMT_YUV420P, 1);
}
}
void func_uninit()
{
av_freep(&v_canvas_frame->data[0]);
av_frame_free(&v_canvas_frame);
}
bool push(AVFrame * frame)
{
//尝试三次插入
return v_frames.enqueue(frame,3);
}
protected:
int draw_init(/*HWND hWnd,*/ )
{
//m_w = 1280;
//m_h = 720;
if (m_window_init == false)
{
SDL_Init(SDL_INIT_VIDEO);
SDL_SetHint(SDL_HINT_RENDER_SCALE_QUALITY, "1");
screen = SDL_CreateWindow("FF", SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED,
1920, 1000, SDL_WINDOW_SHOWN/* SDL_WINDOW_OPENGL | SDL_WINDOW_RESIZABLE*/);
//screen = SDL_CreateWindowFrom((void *)(hWnd));
for (int i = 0; i < SDL_GetNumRenderDrivers(); ++i)
{
SDL_RendererInfo rendererInfo = {};
SDL_GetRenderDriverInfo(i, &rendererInfo);
cout << i << " " << rendererInfo.name << endl;
//if (rendererInfo.name != std::string("direct3d11"))
//{
// continue;
//}
}
if (screen == NULL)
{
//printf("Window could not be created! SDL_Error: %s\n", SDL_GetError());
return -1;
}
sdlRenderer = SDL_CreateRenderer(screen,0, SDL_RENDERER_ACCELERATED | SDL_RENDERER_PRESENTVSYNC);
//sdlRenderer = SDL_CreateRenderer(screen, -1, SDL_RENDERER_ACCELERATED);
//SDL_SetHint(SDL_HINT_RENDER_DRIVER, "opengl");
m_window_init = true;
SDL_Thread *video_tid = SDL_CreateThread(sfp_refresh_thread, NULL, &v_param);
}
return 0;
}
void draw(uint8_t *data[], int linesize[])
{
if (sdlTexture != NULL)
{
SDL_DestroyTexture(sdlTexture);
sdlTexture = NULL;
}
//m_w = w;
//m_h = h;
if (sdlTexture == NULL)
{
sdlTexture = SDL_CreateTexture(sdlRenderer, SDL_PIXELFORMAT_IYUV,
SDL_TEXTUREACCESS_STREAMING, m_w, m_h);
sdlRect.x = 0;
sdlRect.y = 0;
sdlRect.w = m_w;
sdlRect.h = m_h;// nh;
}
SDL_UpdateYUVTexture(sdlTexture, &sdlRect,
data[0], linesize[0],
data[1], linesize[1],
data[2], linesize[2]);
//SDL_RenderClear(sdlRenderer);
SDL_RenderCopy(sdlRenderer, sdlTexture, NULL, NULL);
SDL_RenderPresent(sdlRenderer);
//video_tid = SDL_CreateThread(sfp_refresh_thread, NULL, NULL);
}
public:
void Run()
{
draw_init();
AVFrame * frame = NULL;
//while (!IsStop())
//{
// //解码播放 直接播放
// if (v_frames.dequeue(&frame))
// {
// //播放
// draw(frame->data, frame->linesize,WIDTH,HEIGHT);
// av_frame_free(&frame);
// SDL_Delay(10);
// }
// else
// {
// SDL_Delay(10);
// }
//}
#if 1
/*SDL_Thread *video_tid = SDL_CreateThread(sfp_refresh_thread, NULL, &v_param);*/
int tick = 0;
double x = 0.0f;
for (;;)
{
if (IsStop())
{
v_param.thread_exit = 1;
//break;
}
SDL_WaitEvent(&event);
if (event.type == SFM_REFRESH_EVENT)
{
if (v_frames.dequeue(&frame))
{
tick++;
TicToc tt;
//MergeYUV(v_canvas_frame->data[0], 1280, 720,
// frame->data[0], WIDTH, HEIGHT, 1, 10, 10);
MergeYUV_S(v_canvas_frame->data[0], 1280, 720,
frame->data[0], WIDTH, HEIGHT, 10, 10);
x += tt.toc();
//播放
draw(v_canvas_frame->data, v_canvas_frame->linesize);
av_freep(&frame->data[0]);
av_frame_free(&frame);
if (tick == 10)
{
tick = 0;
cout << x << endl;
x = 0.0f;
}
}
}
else if (event.type == SDL_KEYDOWN)
{
if (event.key.keysym.sym == SDLK_SPACE)
v_param.thread_pause = !v_param.thread_pause;
}
else if (event.type == SDL_QUIT)
{
v_param.thread_exit = 1;
}
else if (event.type == SFM_BREAK_EVENT)
{
break;
}
}
#endif
}
SDLDraw()
{
}
~SDLDraw()
{
func_uninit();
}
};
//SDL_FillRect(gScreenSurface, NULL, SDL_MapRGB(gScreenSurface->format, 0xFF, 0x00, 0x00));
ffmpeg 深度学习处理filter
dnn_processing从2018年开始就已经是FFmpeg中的一个视频filter,支持所有基于深度学习模型的图像处理算法,即输入和输出都是AVFrame,而处理过程使用的是深度学习模型。为什么要开发这样一个filter,因为作为FFmpeg DNN模块的maintainer,dnn_processing就是一个很好的使用者入手功能,读ffmpeg的代码就可以知道,其实自己写这些功能就行了,至于支持视频分析功能的filter,先不进行编码,主要考虑如何支持异步建立流水线,如何启用batch size,从而最大化的用好系统的并行计算能力,本来ffmpeg硬件解码后,如何直接在gpu中进行swscale,其实是不支持的,这一部分要自己写代码来支持,这是另外一回话,我们先使用dnn_processing模块再说,
这个模块可以完成针对灰度图的sobel算子的调用,其输入输出的格式是grayf32。除了这个dnn_processing还可以完成sr(超分辨率)和derain(去除雨点)filter的功能,下面使用ffmpeg命令演示对yuv和rgb格式的支持
./ffmpeg -i night.jpg -vf scale=iw*2:ih*2,format=yuv420p,dnn_processing=dnn_backend=tensorflow:model=./srcnn.pb:input=x:output=y srcnn.jpg
./ffmpeg -i small.jpg -vf format=yuv420p,scale=iw*2:ih*2,dnn_processing=dnn_backend=native:model=./srcnn.model:input=x:output=y -q:v 2 small.jpgsrcnn.jpg
./ffmpeg -i small.jpg -vf format=yuv420p,dnn_processing=dnn_backend=native:model=./espcn.model:input=x:output=y small_b.jpg
./ffmpeg -i rain.jpg -vf format=rgb24,dnn_processing=dnn_backend=native:model=./can.model:input=x:output=y derain.jpg
下图可以看出放大两倍以后还是可以的,不是很模糊,其实是因为我们的训练模型很浅,还没有好好做训练,即使如此,我仔细查看过线性差值比这个图像要差。
文章来源:https://www.toymoban.com/news/detail-627678.html
连贯执行
后面第二篇就要进行硬件解码,到提升质量filter,到输出编码了,请等待第二篇文章来源地址https://www.toymoban.com/news/detail-627678.html
到了这里,关于FFmpeg中硬解码后深度学习模型的图像处理dnn_processing(一)的文章就介绍完了。如果您还想了解更多内容,请在右上角搜索TOY模板网以前的文章或继续浏览下面的相关文章,希望大家以后多多支持TOY模板网!