RTP Payload Format for Transport of MPEG-4 Elementary Streams over http

1．SDP

（1）Http Request

GET /getSdpForUrl?HttpUrl=nphMpeg4/g726-640x480 HTTP/1.0/r/n

Host: 58.63.71.90:8011/r/n

Accept: */*/r/n

Accept-Language: */r/n

Accept-Encoding: */r/n

Authorization: Basic NjU0MzIxOjEyMzQ1NgA=/r/n

User-Agent: Streaming Sdk 1.0/r/n

/r/n

（2）Http Response

HTTP/1.0 200 OK/r/n

Content-Length: 372/r/n

Content-type: application/sdp/r/n

/r/n

v=0/r/n

o=- 1 1 IN IP4 0.0.0.0/r/n

s=NWC-Live/r/n

t=0 0/r/n // unbounded and permanent session

m=audio 0 RTP/AVP 99/r/n // submedia1

b=AS:32.00/r/n // Application Specification:32kbps

a=rtpmap:99 G726-32/8000/1/r/n // G726-32，8kHz

a=control:rtpOverHttp?Url=nphMpeg4/g726-nil/r/n // session-level attribute to distinguish submedia(audio channel)

m=video 0 RTP/AVP 96/r/n // submedia2

b=AS:3000.0/r/n // Application Specification:3000kbps（原始码率）

a=rtpmap:96 MP4V-ES/90000/r/n // MP4V-ES（MPEG-4），90kHz

a=control:rtpOverHttp?Url=nphMpeg4/nil-640x480/r/n // session-level attribute to distinguish submedia(vedio channel)

a=fmtp:96 profile-level-id=1;cpresent=0;config=000001b001000001b5090000010000000120008c4007a8a021e0a31f/r/n

/r/n

从以上两个submedia的a=control会话属性知，后面的A-G726、V-MPEG4采用rtp over http的打包方式传输。

 

2．G726-32

（1）Http Request

GET /rtpOverHttp?Url=nphMpeg4/g726-nil HTTP/1.0/r/n

Host: 58.63.71.90:8011/r/n

Accept: */*/r/n

Accept-Language: */r/n

Accept-Encoding: */r/n

Authorization: Basic NjU0MzIxOjEyMzQ1NgA=/r/n

User-Agent: Streaming Sdk 1.0/r/n

/r/n

（2）Http Response

HTTP/1.0 200 OK/r/n

Content-type: audio/x-pcc-nwc-rtp/r/n

/r/n

后面为G726-32音频流。

char peer1_1[] = {

0x24, 0x00, 0x03, 0xf4, 0x80, 0xe3, 0x23, 0xa7, 0x00, 0x00, 0x8d, 0x65, 0x54, 0x03, 0xd6, 0x44,

0xd1, 0xae, 0xcf, 0xe2, 0xe4, 0x3d, 0x61, 0xe5, 0x32, 0x35, 0x2e, 0x2e, 0xed, 0xda, 0xfb, 0xae,

……

};

char peer1_2[] = {

0x24, 0x00, 0x03, 0xf4, 0x80, 0xe3, 0x23, 0xa8, 0x00, 0x00, 0x95, 0x35, 0x54, 0x03, 0xd6, 0x44,

0xad, 0x3a, 0xfe, 0x3d, 0x53, 0x12, 0xf1, 0x4e, 0xd3, 0xca, 0x39, 0xf2, 0xcf, 0xf3, 0xf2, 0x46,

……

};

char peer1_3[] = {

0x24, 0x00, 0x03, 0xf4, 0x80, 0xe3, 0x23, 0xa9, 0x00, 0x00, 0x9d, 0x05, 0x54, 0x03, 0xd6, 0x44,

0x12, 0x5e, 0x34, 0xa2, 0x2a, 0xdc, 0x3d, 0x1c, 0x2e, 0xed, 0xd3, 0xfe, 0xe1, 0xbc, 0xe4, 0xe1,

……

};

3．MP4V-ES

（1）Http Request

GET /rtpOverHttp?Url=nphMpeg4/nil-640x480 HTTP/1.0/r/n

Host: 58.63.71.90:8011/r/n

Accept: */*/r/n

Accept-Language: */r/n

Accept-Encoding: */r/n

Authorization: Basic NjU0MzIxOjEyMzQ1NgA=/r/n

User-Agent: Streaming Sdk 1.0/r/n

/r/n

（2）Http Response

HTTP/1.0 200 OK/r/n

Content-type: video/x-pcc-nwc-rtp/r/n

/r/n

后面为MP4V-ES视频流

char peer1_1[] = {

0x24, 0x00, 0x44, 0xcb, 0x80, 0xe0, 0x37, 0xfa, 0x00, 0x00, 0x63, 0x43, 0x54, 0x03, 0xd6, 0x44

0x00, 0x00, 0x01, 0xb0, 0x01, 0x00, 0x00, 0x01, 0xb5, 0x09, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,

0x01, 0x20, 0x00, 0x84, 0x40, 0x07, 0xa8, 0xa0, 0x21, 0xe0, 0xa3, 0x1f, 0x00, 0x00, 0x01, 0xb6,

0x3f, 0xff, 0xb3, 0xc2, 0x62, 0x06, 0x1e, 0x5b, 0xe0, 0x30, 0xcd, 0x4c, 0xfb, 0xff, 0xf7, 0x64,

……

};

char peer1_2[] = {

0x24, 0x00, 0x03, 0x39, 0x80, 0xe0, 0x37, 0xfb, 0x00, 0x00, 0xb7, 0xa3, 0x54, 0x03, 0xd6, 0x44,

0x00, 0x00, 0x01, 0xb6, 0x5d, 0xf0, 0xe3, 0x72, 0x27, 0x30, 0xf7, 0x96, 0xe6, 0x4e, 0x5b, 0xcd,

……

};

char peer1_3[] = {

0x24, 0x00, 0x02, 0xf9, 0x80, 0xe0, 0x37, 0xfc, 0x00, 0x01, 0x0c, 0x03, 0x54, 0x03, 0xd6, 0x44,

0x00, 0x00, 0x01, 0xb6, 0x69, 0x30, 0x71, 0xbc, 0x3f, 0xdc, 0xae, 0x67, 0xbc, 0xf7, 0x93, 0x91,

……

};

char peer1_4[] = {

0x24, 0x00, 0x04, 0x6c, 0x80, 0xe0, 0x37, 0xfd, 0x00, 0x01, 0x60, 0x63, 0x54, 0x03, 0xd6, 0x44,

0x00, 0x00, 0x01, 0xb6, 0x55, 0xf0, 0x92, 0xb6, 0x7d, 0xe7, 0x67, 0x46, 0x84, 0x5a, 0x47, 0x5b,

……

};

char peer1_5[] = {

0x24, 0x00, 0x06, 0x00, 0x80, 0xe0, 0x37, 0xfe, 0x00, 0x01, 0xb4, 0xc3, 0x54, 0x03, 0xd6, 0x44,

0x00, 0x00, 0x01, 0xb6, 0x59, 0x60, 0x93, 0x79, 0xdb, 0xc6, 0x0f, 0x35, 0xa4, 0x2b, 0x5f, 0x13,

……

};

以上peer1_1à peer1_5为一个GOP，包括1个I-Frame和4个P-Frame。

char peer1_6[] = {

0x24, 0x00, 0x3c, 0xca, 0x80, 0xe0, 0x37, 0xff, 0x00, 0x02, 0x5d, 0x83, 0x54, 0x03, 0xd6, 0x44

0x00, 0x00, 0x01, 0xb6, 0x28, 0xf0, 0xa8, 0x83, 0xfe, 0x5b, 0xe1, 0x71, 0x9d, 0x33, 0xee, 0x9d,

……

};

char peer1_7[] = {

0x24, 0x00, 0x03, 0x8e, 0x80, 0xe0, 0x38, 0x00, 0x00, 0x02, 0xb1, 0xe3, 0x54, 0x03, 0xd6, 0x44,

0x00, 0x00, 0x01, 0xb6, 0x55, 0x70, 0xf3, 0x72, 0xef, 0x2d, 0xe5, 0xbc, 0xdc, 0x8b, 0x79, 0x5c,

……

};

char peer1_8[] = {

0x24, 0x00, 0x03, 0xf4, 0x80, 0xe0, 0x38, 0x01, 0x00, 0x03, 0x06, 0x43, 0x54, 0x03, 0xd6, 0x44,

0x00, 0x00, 0x01, 0xb6, 0x58, 0xe0, 0xd3, 0xef, 0x3b, 0x18, 0xc5, 0xe7, 0xff, 0xe2, 0xce, 0x77,

……

};

char peer1_9[] = {

0x24, 0x00, 0x06, 0x26, 0x80, 0xe0, 0x38, 0x02, 0x00, 0x03, 0x5a, 0xa3, 0x54, 0x03, 0xd6, 0x44,

0x00, 0x00, 0x01, 0xb6, 0x5c, 0x70, 0x82, 0x2f, 0x35, 0x6a, 0x61, 0x28, 0x8b, 0x22, 0x54, 0xa6,

……

};

char peer1_10[] = {

0x24, 0x00, 0x07, 0x2c, 0x80, 0xe0, 0x38, 0x03, 0x00, 0x03, 0xaf, 0x03, 0x54, 0x03, 0xd6, 0x44,

0x00, 0x00, 0x01, 0xb6, 0x68, 0x70, 0x51, 0xb9, 0x9b, 0x90, 0x6b, 0xcd, 0x69, 0x12, 0x45, 0xc8,

……

};

以上peer1_6à peer1_10为一个GOP，包括1个I-Frame和4个P-Frame。

char peer1_11[] = {

0x24, 0x00, 0x3c, 0x90, 0x80, 0xe0, 0x38, 0x04, 0x00, 0x04, 0x57, 0xc3, 0x54, 0x03, 0xd6, 0x44,

0x00, 0x00, 0x01, 0xb6, 0x18, 0x61, 0x51, 0x07, 0xdc, 0xbf, 0xc2, 0xe3, 0x35, 0x33, 0xfd, 0xef,

……

};

……

rtp over http打包方式说明

头16个字节为rtp over http格式头。

第1个字节为’$’(0x24)，为RtpOverHttp包头标志；第2个字节在interleave传输方式中为通道号，Panasonic BL-C111摄像头AV输出使用一个端口，此处不用置0；第3个字节和第4个字节合成16进制网络字节顺序（big-endian）的包长度（紫色示意）。

接下来的12个字节为RTP的payload format header：2个字节的头 + 2个字节的序号（绿色示意） + 4个字节的时间戳 + 4个字节的同步源。

从第17个字节开始即为纯AV码流数据。

MP4V-ES视频码流分析

关于MPEG-4的概念基础，参考《MPEG-4视频压缩基础》。

关于MPEG-4压缩标准及编解码算法，参考ISO/IEC 14496-1(MPEG-4 Systems)，ISO/IEC 14496-2(MPEG4 Visual)，ISO/IEC 14496-3(MPEG4 Audio)。

请求后的第一个数据包peer1_1[]中含有MP4V-ES的视频对象起始序列码。

peer1_1[16]~peer1_1[20]的5个字节{0x00, 0x00, 0x01, 0xb0, 0x01}为Visual Object Sequence，其中profile_and_level_indication = 0x01 (Simple Profile/Level 1。0x000001b0为visual_object_sequence_start_code。

peer1_1[21]~peer1_1[29]的9个字节{0x00, 0x00, 0x01, 0xb5, 0x09, 0x00, 0x00, 0x01, 0x00}为Visual Object。0x000001b5为visual_object_start_code，0x00000100为video_object_start_code。

peer1_1[30]~peer1_1[43]的14个字节{0x00, 0x00, 0x01, 0x20, 0x00, 0x84, 0x40, 0x07, 0xa8, 0xa0, 0x21, 0xe0, 0xa3, 0x1f}为Video Object Layer，里面包含有video_object_layer_width = 640，video_object_layer_height = 480等信息。0x00000120为video_object_layer_start_code。

VOL是VO的时间或空间的伸屈性描述，目标的伸屈性即是通过VOL来实现的。VO包含一个或多个VOL分辨层，VOL包括一系列VO在时间上的采样VOP。VOP序列是VO在不同分辨层的时间采样，MPEG-4的视频编码就是基于VOP进行的，它用形状、运动和色彩三组参数描述VOP。

对于解码器而言，码流可以不包含VOS和VO信息，但一般至少包含VOL信息。对于摄像头，其编码参数一般全局配置置，一经配置，其采样频率一般固定。故其后的VOP都是针对一个固定的VOL上的采样。

除去前面28个字节的VOS+VO+VOL信息，后面的即为由VOP（I-Frame、P-Frame、B-Frame）等组成的GOP（Group Of Pictures）了。关于起始码start_code参考ISO/IEC 14496-2(MPEG4 Visual)-6.2 Visual bitstream syntax-6.2.1 Start codes一节的Table6-3—Start code values。关于MPEG4的头部结构及解析，可参考xvidcore中libxvidcore/bitstream/bitstream.c中的BitstreamReadHeaders。

从peer1_1[44]开始，0x00, 0x00, 0x01, 0xb6为vop_start_code。视频对象平面（VOP：Visual Object Plane），也就是常说的“帧”。

vop_start_code（0x00, 0x00, 0x01, 0xb6）之后的两位(bit)，00表示I-frame，01表示P-frame，10表示B-frame。

peer1_1中，{0x00, 0x00, 0x01, 0xb6}之后的0x3f(00110000)的前两位为00，故接下来为I-Frame。

peer1_2中，{0x00, 0x00, 0x01, 0xb6}之后的0x5d(01011101)的前两位为01，故接下来为P-Frame。

// /xvidcore-1.2.2/xvidcore/src/bitstream/bitstream.h

/* vop coding types  */

/* intra, prediction, backward, sprite, not_coded */

#define I_VOP          0

#define P_VOP          1

#define B_VOP         2

#define S_VOP          3

#define N_VOP         4

通过分析以上抓包可知，网络流传输时，一般只有I帧和P帧，少有B帧。

以上peer1_1à peer1_5为一个GOP，包括1个I-Frame和4个P-Frame。

以上peer1_6à peer1_10为一个GOP，包括1个I-Frame和4个P-Frame。

关键帧I-Frame，保留了一个场景的所有信息，压缩比为1:7。

未来单向预测帧P-frame，只储存与之前一个已解压画面的差值，压缩比为1:20。

双向预测帧B-frame，除了参考之前解压过了的画面外，亦会参考后面一帧中的画面信息，压缩比为1:50。

一个I-Frame后接若干P-Frame和B-Frame 构成一个视频对象平面组GOV(GOP)。

当画面场景呈静态少有变化时，I帧比较大，P帧比较小且GOP持续较长。当画面场景变化较大时，一般将重新构造新的I帧、形成新的GOP。

以下为获取松下BB-HCM581的MP4V-ES码流，并将其存储至文件的代码。保存的panasonic.mpeg4文件可通过SMPlayer等播放器播放。

// WsaInit.h
#include <winsock.h>
#pragma comment(lib, "wsock32.lib")
// #include <WINSOCK2.H>
// #pragma comment(lib, "WS2_32.LIB") // ws2_32.dll

// class WSAInitializer is a part of the Socket class (on win32)
// as a static instance - so whenever an application uses a Socket,
// winsock is initialized
class WSAInitializer // Winsock Initializer
{
public:
    WSAInitializer()
    {
        if (WSAStartup(0x101, &m_wsadata))
        {
            exit(-1);
        }
    }

    ~WSAInitializer()
    {
        WSACleanup();
    }

private:
    WSADATA m_wsadata;
}WsaInit;

// /Panasonic/Mpeg4Proc/mpeg4.cpp
#include <stdio.h>
#include <string.h>
#include "WsaInit.h"

#define MP4V_ES 96 // rtp payload type

// host
char ip[] = "219.117.194.183";
unsigned short port = 60151;
char http_req[] = "GET /rtpOverHttp?Url=nphMpeg4/nil-640x480 HTTP/1.1/r/n/r/n";

int ReadHttpResponse(SOCKET s, char *buf);
int ReadBuffer(SOCKET s, char *buf, int len);

int main()
{
    // Create a socket stands for the client process
    SOCKET clientSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
    if(clientSocket == INVALID_SOCKET) // Socket creation failure
    {
        printf("Error #%d in socket()!/n", WSAGetLastError());
        return 1;
    }

    // Fill the sockaddr_in structure
    sockaddr_in servAddr;
    servAddr.sin_family = AF_INET;
    servAddr.sin_port = htons(port);
    servAddr.sin_addr.S_un.S_addr = inet_addr(ip);

    // Establishes a connection to the clientSocket.
    if(connect(clientSocket, (sockaddr*)&servAddr, sizeof(servAddr)) == SOCKET_ERROR)
    {
        printf("Error #%d in connect()!/n", WSAGetLastError());
        return 1;
    }

    // Send http request
    send(clientSocket, http_req, strlen(http_req), 0);

    // Receive some echo data from the server
    char header[128];

    // http response
    ReadHttpResponse(clientSocket, header);
    printf(header);

    // data stream
    bool bRightHeader;
    unsigned char firstByte;
    unsigned char TrackId; // channel track id
    unsigned short Mpeg4FrameLen; // RTP packet len
    unsigned char RtpHeader[12];
    unsigned char Mpeg4Frame[36*1024]; //
    int nRecv = 0;
    FILE *fpPanasonicMpeg4 = fopen("panasonic.mpeg4", "w+b");

    for (int i=0; i<200; i++)
    {
        bRightHeader = false;
        Mpeg4FrameLen = 0;

        do {
            // (1) read and analyze the rtp over http header (16 byte)
            if (recv(clientSocket, (char*)&firstByte, 1, 0)!=1 || firstByte!='$') // packet header flag
                break;

            if (recv(clientSocket, (char*)&TrackId, 1, 0) != 1)
                break;

            if (recv(clientSocket, (char*)&Mpeg4FrameLen, 2, 0) != 2)
                break;

            nRecv = recv(clientSocket, (char*)RtpHeader, 12, 0);
            if (nRecv>0 && ((RtpHeader[1]&0x7F)==MP4V_ES)) // check the payload type
            {
                Mpeg4FrameLen = htons(Mpeg4FrameLen)-12;
                bRightHeader = true;
            }
        } while (0);

        // (2) read the following mpeg4 stream
        if (bRightHeader)
        {
            nRecv = ReadBuffer(clientSocket, (char*)Mpeg4Frame, Mpeg4FrameLen);

            if (nRecv > 0)
            {
                printf("Mpeg4FrameLen = %d, nRecv = %d/n", Mpeg4FrameLen, nRecv);
                fwrite(Mpeg4Frame, 1, nRecv, fpPanasonicMpeg4);
            }
        }
    }

    fclose(fpPanasonicMpeg4);

    // closes client socket
    if (closesocket(clientSocket) == SOCKET_ERROR)
    {
        printf("Error #%d in closesocket()!/n", WSAGetLastError());
        return 1;
    }

    return 0;
}

int ReadHttpResponse(SOCKET s, char *buf)
{
    bool done = false;
    int nRecv = 0;
    char c;
    int i = 0;
    int chars = 0; // count chars in a line

    // read response hader, until see "/n/n" or "/r/n/r/n"
    while(!done)
    {
        nRecv = recv(s, &c, 1, 0);
        if (nRecv <= 0)
        {
            break;
        }

        if (c == '/n')
        {
            if (chars == 0)
            {
                done = true; // empty line
            }
            else
            {
                chars = 0;
            }
        }
        else if (c != '/r')
        {
            chars++;
        }

        buf[i++] = c;
    }

    buf[i] = 0; // fill a null-terminated flag for the C string

    return i;
}

int ReadBuffer(SOCKET s, char *buf, int len)
{
    int nBytesToRead = len; // bytes to be read
    int nBytesRead = 0;     // bytes read already
    int nCurBytesRead = 0;  // bytes read everytime, return by recv()

    while ((nCurBytesRead = recv(s, buf+nBytesRead, nBytesToRead, 0)) > 0)
    {
        nBytesRead += nCurBytesRead;

        if (nBytesRead >= len)
        {
            break;
        }

        nBytesToRead -= nCurBytesRead;
    }

    return nBytesRead;
}

说明：

本文网络抓包工具为Wireshark(Ethereal)，码流分析工具为Elecard StreamEye Studio中的Elecard Stream Analyzer。

RtpOverHttp参考：

《rtsp/rtp over http》

SDP参考：

《RFC 2327 - SDP Session Description Protocol》

《SDP Format for RTSP Streams》

G726参考：

《G.726 Vocoder》

《Recommendation G.726 - Corr.1 (AAP14-05/05)》

MPEG4参考：

《MPEG1和MPEG2码流结构分析》

《MPEG4码流分析》

《MPEG4码流简单分析》

《视频文件解码与起始码(startcode) 研究》

RFC参考：

《RFC 3550 - RTP A Transport Protocol for Real-Time Applications》

《RFC 3016 - RTP Payload Format for MPEG-4 Audio-Visual Streams》

《RFC 3640 - RTP Payload Format for Transport of MPEG-4 Elementary Streams》

《RFC 5691 - RTP Payload Format for Elementary Streams with MPEG Surround Multi-Channel Audio》