大多数RTSP的播放都面向直播场景，除了H.264，还需要支持H.265，目前市面上的RTSP H.265摄像头越来越多，支持H.265的RTSP播放器迫在眉睫，此外，单纯的播放H.265还不够，还需要可以能把H.265的数据能录制下来。
青犀团队接触到好多开发者，希望能在播放的同时，获取到YUV或RGB数据，进行人脸匹配等算法分析，所以音视频回调可选。
EasyPlayer-RTSP实现支持H265编码格式
EasyPlayer-RTSP-Win已全面支持H265的RTSP流的解码播放，这里就支持H265过程做简要介绍；
一、libEasyRTSPClient库已支持H265视频源的RTSP流的拉取和解析
二、H265头解析
H265和H264类似，不过其NAL type格式更多样化，除了SPS，PPS之外，还增加了VPS，下面就针对H265帧nal 头做简单分析；
首先，看X265源码中的H265nal头格式定义：
//H265 NAL type
//this enum have been defined in x265.h
typedef enum tagH265NalUnitType
{
NAL_UNIT_CODED_SLICE_TRAIL_N = 0, // 0
NAL_UNIT_CODED_SLICE_TRAIL_R, // 1
NAL_UNIT_CODED_SLICE_TSA_N, // 2
NAL_UNIT_CODED_SLICE_TLA, // 3 // Current name in the spec: TSA_R
NAL_UNIT_CODED_SLICE_STSA_N, // 4
NAL_UNIT_CODED_SLICE_STSA_R, // 5
NAL_UNIT_CODED_SLICE_RADL_N, // 6
NAL_UNIT_CODED_SLICE_DLP, // 7 // Current name in the spec: RADL_R
NAL_UNIT_CODED_SLICE_RASL_N, // 8
NAL_UNIT_CODED_SLICE_TFD, // 9 // Current name in the spec: RASL_R
NAL_UNIT_RESERVED_10,
NAL_UNIT_RESERVED_11,
NAL_UNIT_RESERVED_12,
NAL_UNIT_RESERVED_13,
NAL_UNIT_RESERVED_14,
NAL_UNIT_RESERVED_15,
NAL_UNIT_CODED_SLICE_BLA, // 16 // Current name in the spec: BLA_W_LP
NAL_UNIT_CODED_SLICE_BLANT, // 17 // Current name in the spec: BLA_W_DLP
NAL_UNIT_CODED_SLICE_BLA_N_LP, // 18
NAL_UNIT_CODED_SLICE_IDR, // 19 // Current name in the spec: IDR_W_DLP
NAL_UNIT_CODED_SLICE_IDR_N_LP, // 20
NAL_UNIT_CODED_SLICE_CRA, // 21
NAL_UNIT_RESERVED_22,
NAL_UNIT_RESERVED_23,
NAL_UNIT_RESERVED_24,
NAL_UNIT_RESERVED_25,
NAL_UNIT_RESERVED_26,
NAL_UNIT_RESERVED_27,
NAL_UNIT_RESERVED_28,
NAL_UNIT_RESERVED_29,
NAL_UNIT_RESERVED_30,
NAL_UNIT_RESERVED_31,
NAL_UNIT_VPS, // 32
NAL_UNIT_SPS, // 33
NAL_UNIT_PPS, // 34
NAL_UNIT_ACCESS_UNIT_DELIMITER, // 35
NAL_UNIT_EOS, // 36
NAL_UNIT_EOB, // 37
NAL_UNIT_FILLER_DATA, // 38
NAL_UNIT_SEI, // 39 Prefix SEI
NAL_UNIT_SEI_SUFFIX, // 40 Suffix SEI
NAL_UNIT_RESERVED_41,
NAL_UNIT_RESERVED_42,
NAL_UNIT_RESERVED_43,
NAL_UNIT_RESERVED_44,
NAL_UNIT_RESERVED_45,
NAL_UNIT_RESERVED_46,
NAL_UNIT_RESERVED_47,
NAL_UNIT_UNSPECIFIED_48,
NAL_UNIT_UNSPECIFIED_49,
NAL_UNIT_UNSPECIFIED_50,
NAL_UNIT_UNSPECIFIED_51,
NAL_UNIT_UNSPECIFIED_52,
NAL_UNIT_UNSPECIFIED_53,
NAL_UNIT_UNSPECIFIED_54,
NAL_UNIT_UNSPECIFIED_55,
NAL_UNIT_UNSPECIFIED_56,
NAL_UNIT_UNSPECIFIED_57,
NAL_UNIT_UNSPECIFIED_58,
NAL_UNIT_UNSPECIFIED_59,
NAL_UNIT_UNSPECIFIED_60,
NAL_UNIT_UNSPECIFIED_61,
NAL_UNIT_UNSPECIFIED_62,
NAL_UNIT_UNSPECIFIED_63,
NAL_UNIT_INVALID,
}H265NalUnitType;
#endif
我们可以看到其中VPS, SPS和PPS的定义：
NAL_UNIT_VPS, // 32
NAL_UNIT_SPS, // 33
NAL_UNIT_PPS, // 34
同样，我们也很容易知道P帧NAL type定义是0-9， I帧定义是16-21；可见H265的NAL type定义比H264要多样化，判断也不限制于一种类型；
同时，测试发现，实际H265帧数据中的VPS=0x40 , SPS=0x42, PPS=0x44, 通过换算，我们不难得出：NALtype*2 = 实际的流中的NaLType;
具体解析过程如下：
//输入的pbuf必须包含start code(00 00 00 01)
int GetH265VPSandSPSandPPS(char *pbuf, int bufsize, char *_vps, int *_vpslen, char *_sps, int *_spslen, char *_pps, int *_ppslen)
{
char vps[512]={0}, sps[512] = {0}, pps[128] = {0};
int vpslen=0, spslen=0, ppslen=0, i=0, iStartPos=0, ret=-1;
int iFoundVPS=0, iFoundSPS=0, iFoundPPS=0, iFoundSEI=0;
if (NULL == pbuf || bufsize<4) return -1;
for (i=0; i<bufsize; i++)
{
if ( (unsigned char)pbuf[i] == 0x00 && (unsigned char)pbuf[i+1] == 0x00 &&
(unsigned char)pbuf[i+2] == 0x00 && (unsigned char)pbuf[i+3] == 0x01 )
{
printf("0x%X\n", (unsigned char)pbuf[i+4]);
switch ((unsigned char)pbuf[i+4])
{
case 0x40: //VPS
{
iFoundVPS = 1;
iStartPos = i+4;
}
break;
case 0x42: //SPS
{
if (iFoundVPS == 0x01 && i>4)
{
vpslen = i-iStartPos;
if (vpslen>256) return -1; //vps长度超出范围
memset(vps, 0x00, sizeof(vps));
memcpy(vps, pbuf+iStartPos, vpslen);
}
iStartPos = i+4;
iFoundSPS = 1;
}
break;
case 0x44: //PPS
{
if (iFoundSPS == 0x01 && i>4)
{
spslen = i-iStartPos;
if (spslen>256) return -1;
memset(sps, 0x0, sizeof(sps));
memcpy(sps, pbuf+iStartPos, spslen);
}
iStartPos = i+4;
iFoundPPS = 1;
}
break;
case 0x4E: //Prefix SEI
case 0x50: //Suffix SEI
case 0x20: //I frame 16
case 0x22: //I frame 17
case 0x24: //I frame 18
case 0x26: //I frame 19
case 0x28: //I frame 20
case 0x2A: //I frame 21(acturally we should find naltype 16-21)
{
if (iFoundPPS == 0x01 && i>4)
{
ppslen = i-iStartPos;
if (ppslen>256) return -1;
memset(pps, 0x0, sizeof(pps));
memcpy(pps, pbuf+iStartPos, ppslen);
}
iStartPos = i+4;
iFoundSEI = 1;
}
break;
default:
break;
}
}
if (iFoundSEI == 0x01) break;
}
if (iFoundVPS == 0x01)
{
if (vpslen < 1)
{
if (bufsize < sizeof(vps))
{
vpslen = bufsize-4;
memset(vps, 0x00, sizeof(vps));
memcpy(vps, pbuf+4, vpslen);
}
}
if (vpslen > 0)
{
if (NULL != _vps) memcpy(_vps, vps, vpslen);
if (NULL != _vpslen) *_vpslen = vpslen;
}
ret = 0;
}
if (iFoundSPS == 0x01)
{
if (spslen < 1)
{
if (bufsize < sizeof(sps))
{
spslen = bufsize-4;
memset(sps, 0x00, sizeof(sps));
memcpy(sps, pbuf+4, spslen);
}
}
if (spslen > 0)
{
if (NULL != _sps) memcpy(_sps, sps, spslen);
if (NULL != _spslen) *_spslen = spslen;
}
ret = 0;
}
if (iFoundPPS == 0x01)
{
if (ppslen < 1)
{
if (bufsize < sizeof(pps))
{
ppslen = bufsize-4;
memset(pps, 0x00, sizeof(pps));
memcpy(pps, pbuf+4, ppslen); //pps
}
}
if (ppslen > 0)
{
if (NULL != _pps) memcpy(_pps, pps, ppslen);
if (NULL != _ppslen) *_ppslen = ppslen;
}
ret = 0;
}
return ret;
}
三、 解码器需支持H265
解码器直接使用最新的FFMPEG库即支持H265解码，且软解效率还可以，大家如果不知道怎么用，可以去看看ffplay的源码，这里不做过多赘述；这里就EasyPlayer调用遇到的问题做简单说明：
- 旧版的ffmpeg以及live555等对H265的定义是对“H265”子串做的字串格式组合，而新版的FFMPEG使用的自定义的顺序定义的枚举类型，所以在使用过程中可能出现对应不上的情况，比如，在libEasyRTSPClient库中对H265的定义为：#define EASY_SDK_VIDEO_CODEC_H265 0x48323635 /* 1211250229 */
而FFMPEG中定义H265（HEVC）格式为174
EasyPlayer中进行格式统一代码如下：
//H265 codecID改成FFMPEG新版的
int nCodec = (_frameinfo->codec == EASY_SDK_VIDEO_CODEC_H265) ? 174 : _frameinfo->codec;
- EasyPlayer中之前对关键帧帧解码失败的处理是将以该I帧为关键帧为依托的所有P帧丢弃，当然这从某种程度上是可以避免花屏的，但是测试解码H265时发现，H265的第一个I帧会经常解码失败，经调试发现其实是FFNPEG的解码函数返回没有解码完成的结果被程序判断为解码失败，而这个时候应该不做任何处理等下一次返回的时候就能获取到正确的返回结果了，EasyPlayer处理如下：
nRet = FFD_DecodeVideo3(pDecoderObj->ffDecoder, pbuf, frameinfo.length, pThread->yuvFrame[pThread->decodeYuvIdx].pYuvBuf, frameinfo.width, frameinfo.height, lTimestamp, lTimestamp);
if (0 != nRet)
{
if(nRet == -4)//-4表示为当前帧尚未解码完成，不作为错误判断
{
_TRACE("视频帧解码尚未完成[%d]... framesize:%d %02X %02X %02X %02X %02X %02X %02X %02X %02X %02X\n", nRet, frameinfo.length,
(unsigned char)pbuf[0], (unsigned char)pbuf[1], (unsigned char)pbuf[2], (unsigned char)pbuf[3], (unsigned char)pbuf[4],
(unsigned char)pbuf[5], (unsigned char)pbuf[6], (unsigned char)pbuf[7], (unsigned char)pbuf[8], (unsigned char)pbuf[9]);
}
else
{
_TRACE("视频帧解解码失败[%d]... framesize:%d %02X %02X %02X %02X %02X %02X %02X %02X %02X %02X\n", nRet, frameinfo.length,
(unsigned char)pbuf[0], (unsigned char)pbuf[1], (unsigned char)pbuf[2], (unsigned char)pbuf[3], (unsigned char)pbuf[4],
(unsigned char)pbuf[5], (unsigned char)pbuf[6], (unsigned char)pbuf[7], (unsigned char)pbuf[8], (unsigned char)pbuf[9]);
if (frameinfo.type == EASY_SDK_VIDEO_FRAME_I) //关键帧
{
_TRACE("[ch%d]当前关键帧解码失败...\n", pThread->channelId);
#ifdef _DEBUG
FILE *f = fopen("keyframe.txt", "wb");
if (NULL != f)
{
fwrite(pbuf, 1, frameinfo.length, f);
fclose(f);
}
#endif
}
else
{
#ifdef _DEBUG
FILE *f = fopen("pframe.txt", "wb");
if (NULL != f)
{
fwrite(pbuf, 1, frameinfo.length, f);
fclose(f);
}
#endif
}
pThread->findKeyframe = 0x01;
}
经测试，经过如上代码 修改就能有效的避免的解码H265视频的时候开头帧总是会局部卡帧或者花屏的情况出现。
四、H265格式视频写MP4
这里接着之前EasyPlayer系列的写MP4篇讲，将H265封装MP4；
-
解析H265的头，或者VPS，SPS和PPS
从H265帧中取出NAL头在上文已经作过讲解这里就不做过多赘述； -
写入VPS, SPS和PPS 等关键解码信息
bool EasyMP4Writer::WriteH265VPSSPSandPPS(unsigned char*vps, int vpslen, unsigned char*sps, int spslen,
unsigned char*pps, int ppslen, int width, int height)
{
if (m_nCreateFileFlag&ZOUTFILE_FLAG_VIDEO)
{
m_videtrackid = gf_isom_new_track(p_file, 0, GF_ISOM_MEDIA_VISUAL, 1000);
gf_isom_set_track_enabled(p_file, m_videtrackid, 1);
}
else
{
return false;
}
p_videosample = gf_isom_sample_new();
p_videosample->data = (char*)malloc(1024 * 1024);
p_hevc_config = gf_odf_hevc_cfg_new();
p_hevc_config->nal_unit_size = 32 / 8;
//gf_isom_avc_config_new(p_file, m_videtrackid, p_config, NULL, NULL, &i_videodescidx);
//gf_isom_set_visual_info(p_file, m_videtrackid, i_videodescidx, width, height);
//初始化配置
gf_isom_hevc_config_new(p_file, m_videtrackid, p_hevc_config, NULL, NULL, &i_videodescidx);
gf_isom_set_nalu_extract_mode(p_file, m_videtrackid, GF_ISOM_NALU_EXTRACT_INSPECT);
gf_isom_set_cts_packing(p_file, m_videtrackid, GF_TRUE);
HEVCState hevc = { 0 };
m_slotsps = { 0 };
m_slotpps = { 0 };
m_slotvps = { 0 };
m_spss = { 0 };
m_ppss = { 0 };
m_vpss = { 0 };
p_hevc_config->configurationVersion = 1;
//Config vps
int idx = gf_media_hevc_read_vps((char*)vps, vpslen, &hevc);
hevc.vps[idx].crc = gf_crc_32((char*)vps, vpslen);
p_hevc_config->avgFrameRate = hevc.vps[idx].rates[0].avg_pic_rate;
p_hevc_config->constantFrameRate = hevc.vps[idx].rates[0].constand_pic_rate_idc;
p_hevc_config->numTemporalLayers = hevc.vps[idx].max_sub_layers;
p_hevc_config->temporalIdNested = hevc.vps[idx].temporal_id_nesting;
m_vpss.nalus = gf_list_new();
gf_list_add(p_hevc_config->param_array, &m_vpss);
m_vpss.array_completeness = 1;
m_vpss.type = GF_HEVC_NALU_VID_PARAM;// naltype = VPS
m_slotvps.id = idx;
m_slotvps.size = vpslen;
m_slotvps.data = (char*)malloc(vpslen);
memcpy(m_slotvps.data, vps, vpslen);
gf_list_add(m_vpss.nalus, &m_slotvps);
//Config sps
idx = gf_media_hevc_read_sps((char*)sps, spslen, &hevc);
hevc.sps[idx].crc = gf_crc_32((char*)sps, spslen);
p_hevc_config->profile_space = hevc.sps[idx].ptl.profile_space;
p_hevc_config->tier_flag = hevc.sps[idx].ptl.tier_flag;
p_hevc_config->profile_idc = hevc.sps[idx].ptl.profile_idc;
m_spss.nalus = gf_list_new();
gf_list_add(p_hevc_config->param_array, &m_spss);
m_spss.array_completeness = 1;
m_spss.type = GF_HEVC_NALU_SEQ_PARAM;// naltype = SPS
m_slotsps.id = idx;
m_slotsps.size = spslen;
m_slotsps.data = (char*)malloc(spslen);
memcpy(m_slotsps.data, sps, spslen);
gf_list_add(m_spss.nalus, &m_slotsps);
int act_width = hevc.sps[idx].width;
int act_height = hevc.sps[idx].height;
//Config pps
idx = gf_media_hevc_read_pps((char*)pps, ppslen, &hevc);
hevc.pps[idx].crc = gf_crc_32((char*)pps, ppslen);
m_ppss.nalus = gf_list_new();
gf_list_add(p_hevc_config->param_array, &m_ppss);
m_ppss.array_completeness = 1;
m_ppss.type = GF_HEVC_NALU_PIC_PARAM;// naltype = PPS
m_slotpps.id = idx;
m_slotpps.size = ppslen;
m_slotpps.data = (char*)malloc(ppslen);
memcpy(m_slotpps.data, pps, ppslen);
gf_list_add(m_ppss.nalus, &m_slotpps);
gf_isom_set_visual_info(p_file, m_videtrackid, i_videodescidx, act_width, act_height);
gf_isom_hevc_config_update(p_file, m_videtrackid, 1, p_hevc_config);
//销毁申请的内存资源
gf_list_del(m_vpss.nalus);
gf_list_del(m_spss.nalus);
gf_list_del(m_ppss.nalus);
free(m_slotvps.data);
free(m_slotsps.data);
free(m_slotpps.data);
p_hevc_config->param_array = NULL;
return true;
}
结合写MP4篇我们不难看出，MP4BOX对H265专门封装了个结构函数gf_isom_hevc_config_new（）用以对H265参数的设置，设置方法和H264相似，不过对H265处理更加细致，MP4BOX将VPS,SPS,PPS的各个参数拆分出来进行赋值，通过gf_isom_hevc_config_update写入解码参数信息。