webrtc 视频编解码实时音视频视觉检测 FFmpeg 代码实现流媒体推流RTSP

桃扇骨 2024-02-23 08:01 30阅读 0赞

实时录屏并把视频推流到RTSP服务器，具体流程是抓取屏幕内容(bitmap)，并把bitmap转化为YUV，接着把YUV编码成H264，再把H264码流推到RTSP服务器；把采集到的PCM编码为AAC，再把AAC推流至RTSP服务器。

RTSP服务器使用的是HappyTime的免费试用版本。

1. bitmap转YUV

我抓到的bitmap是BGRA格式的，所以使用的图像格式是AV\_PIX\_FMT\_BGRA，cropImage是含有rgba图像的数组

bool init_RGB_to_YUV(){
    
    //BGRA 转 YUV
    
    swrCtxBGRA2YUV = sws_getContext(
    
    cap_w, cap_h, AV_PIX_FMT_BGRA,
    
    cap_w, cap_h, AV_PIX_FMT_YUV420P,
    
    SWS_BICUBIC,
    
    NULL, NULL, NULL
    
    );
    
    //创建BGRA帧
    
    frame_bgra = av_frame_alloc();
    
    frame_bgra->format = AV_PIX_FMT_BGRA;
    
    frame_bgra->width = cap_w;
    
    frame_bgra->height = cap_h;
    
    if (av_frame_get_buffer(frame_bgra, 32) < 0) {
    
    printf("Failed: av_frame_get_buffer\n");
    
    return false;
    
    }
    
    frame_bgra->data[0] = cropImage;
    
    //YUV帧
    
    frame_yuv = av_frame_alloc();
    
    frame_yuv->width = cap_w;
    
    frame_yuv->height = cap_h;
    
    frame_yuv->format = AV_PIX_FMT_YUV420P;
    
    //
    
    uint8_t *picture_buf = (uint8_t *)av_malloc(cap_w * cap_h * 1.5);
    
    if (av_image_fill_arrays(frame_yuv->data, frame_yuv->linesize, picture_buf, AV_PIX_FMT_YUV420P, cap_w, cap_h, 1) < 0){
    
    printf("Failed: av_image_fill_arrays\n");
    
    return false;
    
    }
    
    return true;
    
    }

调用：

//BGRA 转 YUV
    
    if (sws_scale(swrCtxBGRA2YUV,
    
    frame_bgra->data, frame_bgra->linesize,
    
    0, cap_h,
    
    frame_yuv->data, frame_yuv->linesize) < 0)
    
    {
    
    printf("失败：BGRA 转 YUV\n");
    
    return;
    
    }
    
    frame_yuv->pts = av_gettime();

由于我是实时抓取的屏幕，frame\_yuv->pts设为当前的时间戳，以保证能正常播放。

2. H264编码

bool init\_YUV\_to\_H264()\{

//寻找编码器

codec_h264 = avcodec_find_encoder(AV_CODEC_ID_H264);
    
    if (!codec_h264){
    
    printf("Fail: avcodec_find_encoder\n");
    
    return false;
    
    }
    
    //编码器上下文
    
    codec_ctx_h264 = avcodec_alloc_context3(codec_h264);
    
    if (!codec_ctx_h264){
    
    printf("Fail: avcodec_alloc_context3\n");
    
    return false;
    
    }
    
    codec_ctx_h264->pix_fmt = AV_PIX_FMT_YUV420P;
    
    codec_ctx_h264->codec_type = AVMEDIA_TYPE_VIDEO;
    
    codec_ctx_h264->width = cap_w;
    
    codec_ctx_h264->height = cap_h;
    
    codec_ctx_h264->channels = 3;
    
    codec_ctx_h264->time_base = { 1, 25 };
    
    codec_ctx_h264->gop_size = 5; //图像组两个关键帧（I帧）的距离
    
    codec_ctx_h264->max_b_frames = 0;
    
    //codec_ctx_h264->qcompress = 0.6;
    
    //codec_ctx_h264->bit_rate = 90000;
    
    codec_ctx_h264->flags |= AV_CODEC_FLAG_GLOBAL_HEADER; //添加PPS、SPS
    
    av_opt_set(codec_ctx_h264->priv_data, "preset", "ultrafast", 0); //快速编码，但会损失质量
    
    //av_opt_set(codec_ctx_h264->priv_data, "tune", "zerolatency", 0); //适用于快速编码和低延迟流式传输,但是会出现绿屏
    
    //av_opt_set(codec_ctx_h264->priv_data, "x264opts", "crf=26:vbv-maxrate=728:vbv-bufsize=3640:keyint=25", 0);
    
    //打开编码器
    
    if (avcodec_open2(codec_ctx_h264, codec_h264, NULL) < 0){
    
    printf("Fail: avcodec_open2\n");
    
    return false;
    
    }
    
    //用于接收编码好的H264
    
    pkt_h264 = av_packet_alloc();
    
    return true;
    
    }

调用：

ret = avcodec_send_frame(codec_ctx_h264, frame_yuv);
    
    if (ret < 0){
    
    printf("send frame fail\n");
    
    return;
    
    }
    
    while (ret >= 0) {
    
    ret = avcodec_receive_packet(codec_ctx_h264, pkt_h264);
    
    if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF){
    
    break;
    
    }
    
    if (ret < 0){
    
    printf("Error during encoding\n");
    
    break;
    
    }
    
    pkt_h264->stream_index = videoindex;
    
    //printf("pkt_h264 timestamp = %d\n", pkt_h264->pts);
    
    if (av_interleaved_write_frame(fmt_ctx, pkt_h264) < 0) {
    
    printf("Error muxing packet\n");
    
    }
    
    av_packet_unref(pkt_h264);
    
    }

3. AAC编码

bool init_PCM_to_AAC(){
    
    codec_aac = avcodec_find_encoder(AV_CODEC_ID_AAC);
    
    if (!codec_aac) {
    
    printf("avcodec_find_encoder fail\n");
    
    return false;
    
    }
    
    codec_ctx_aac = avcodec_alloc_context3(codec_aac);
    
    if (!codec_ctx_aac) {
    
    printf("avcodec_find_encoder fail\n");
    
    return false;
    
    }
    
    codec_ctx_aac->sample_fmt = AV_SAMPLE_FMT_FLT;
    
    codec_ctx_aac->codec_type = AVMEDIA_TYPE_AUDIO;
    
    codec_ctx_aac->channels = channels;
    
    codec_ctx_aac->channel_layout = av_get_default_channel_layout(channels);
    
    codec_ctx_aac->sample_rate = sample_rete;
    
    if (avcodec_open2(codec_ctx_aac, codec_aac, NULL) < 0) {
    
    printf("open codec fail\n");
    
    return false;
    
    }
    
    swrCtxS162FLT = swr_alloc_set_opts(NULL,
    
    codec_ctx_aac->channel_layout, codec_ctx_aac->sample_fmt, codec_ctx_aac->sample_rate,
    
    codec_ctx_aac->channel_layout, AV_SAMPLE_FMT_S16, codec_ctx_aac->sample_rate,
    
    0, 0);
    
    if (!swrCtxS162FLT)
    
    {
    
    printf("swr_alloc_set_opts error\n");
    
    return false;
    
    }
    
    if (swr_init(swrCtxS162FLT) < 0) {
    
    printf("open resample fail\n");
    
    return false;
    
    }
    
    frame_pcm = av_frame_alloc();
    
    frame_pcm->nb_samples = nbSamples_; //一帧音频存放的样本数量
    
    frame_pcm->format = codec_ctx_aac->sample_fmt;
    
    frame_pcm->channels = codec_ctx_aac->channels;
    
    frame_pcm->channel_layout = codec_ctx_aac->channel_layout;
    
    if (av_frame_get_buffer(frame_pcm, 0) < 0) {
    
    printf("av_frame_get_buffer error\n");
    
    return false;
    
    }
    
    pkt_aac = av_packet_alloc();
    
    return true;
    
    }

调用： 其中pcm\_buff是包含pcm数据的数组

const uint8_t *pcm[1];
    
    pcm[0] = pcm_buff;
    
    int len = swr_convert(swrCtxS162FLT,
    
    frame_pcm->data, frame_pcm->nb_samples,
    
    pcm, nbSamples_);
    
    if (len <= 0) {
    
    printf("---Encodec:PCM->AAC--- swr_convert fail \n");
    
    return;
    
    }
    
    frame_pcm->pts = av_gettime();
    
    //printf("channels = %d\n", frame_pcm->channels);
    
    //printf("framePCM->linesize = %6d %6d\n", frame_pcm->linesize[0], frame_pcm->linesize[1]);

//AAC编码

int ret = avcodec_send_frame(codec_ctx_aac, frame_pcm);
    
    if (ret < 0){
    
    printf("send frame fail\n");
    
    return;
    
    }
    
    ret = avcodec_receive_packet(codec_ctx_aac, pkt_aac);
    
    if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF){
    
    return;
    
    }
    
    if (ret < 0){
    
    printf("Error during encoding\n");
    
    return;
    
    }
    
    pkt_aac->stream_index = audioindex;
    
    //printf("pkt_aac timestamp = %d\n", pkt_aac->pts);
    
    if (av_interleaved_write_frame(fmt_ctx, pkt_aac) < 0) {
    
    printf("Error muxing packet\n");
    
    }
    
    av_packet_unref(pkt_aac);

4. 推流器

使用udp传输时传到1400多帧就断开链接了，原因不明，所以改用使用tcp协议传输

bool init_rtsp_pusher(){
    
    //RTSP
    
    if (avformat_alloc_output_context2(&fmt_ctx, NULL, "RTSP", RTSP_URL.c_str()) < 0){
    
    printf("Fail: avformat_alloc_output_context2\n");
    
    return false;
    
    }

//使用tcp协议传输

av_opt_set(fmt_ctx->priv_data, "rtsp_transport", "tcp", 0);

//检查所有流是否都有数据，如果没有数据会等待max\_interleave\_delta微秒

fmt_ctx->max_interleave_delta = 1000000;

//输出视频流

AVStream *video_s = avformat_new_stream(fmt_ctx, codec_h264);
    
    if (!video_s){
    
    printf("Fail: avformat_new_stream\n");
    
    return false;
    
    }
    
    video_s->time_base = { 1, 25 };
    
    videoindex = video_s->id = fmt_ctx->nb_streams - 1; //加入到fmt_ctx流

//复制AVCodecContext的设置

if (avcodec_copy_context(video_s->codec, codec_ctx_h264) < 0) {
    
    printf("Fail: avcodec_copy_context\n");
    
    return false;
    
    }
    
    video_s->codec->codec_tag = 0;
    
    if (fmt_ctx->oformat->flags & AVFMT_GLOBALHEADER)
    
    video_s->codec->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
    
    avcodec_parameters_from_context(video_s->codecpar, codec_ctx_h264);

//输出音频流

AVStream *audio_s = avformat_new_stream(fmt_ctx, codec_ctx_aac->codec);
    
    if (!audio_s){
    
    printf("Fail: avformat_new_stream\n");
    
    return false;
    
    }
    
    audio_s->time_base = { 1, 25 };
    
    audioindex = audio_s->id = fmt_ctx->nb_streams - 1;

//复制AVCodecContext的设置

if (avcodec_copy_context(audio_s->codec, codec_ctx_aac) < 0) {
    
    printf("Fail: avcodec_copy_context\n");
    
    return false;
    
    }
    
    audio_s->codec->codec_tag = 0;
    
    if (fmt_ctx->oformat->flags & AVFMT_GLOBALHEADER)
    
    audio_s->codec->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
    
    avcodec_parameters_from_context(audio_s->codecpar, codec_ctx_aac);
    
    //printf("fmt_ctx nb_streams = %d\n", fmt_ctx->nb_streams);
    
    av_dump_format(fmt_ctx, 0, fmt_ctx->filename, 1);
    
    if (!(fmt_ctx->oformat->flags & AVFMT_NOFILE)) { //???

//打开输出URL（Open output URL）

if (avio_open(&fmt_ctx->pb, fmt_ctx->filename, AVIO_FLAG_WRITE) < 0) {
    
    printf("Fail: avio_open('%s')\n", fmt_ctx->filename);
    
    return false;
    
    }
    
    }
    
    return true;
    
    }