Where can there be an error in a self-written RTP server?

Question

C++ / C#

Hello! Can you tell me how to solve such an annoying problem? I am writing a small RTP server for streaming H.264 video.
An unknown problem appeared: after some time (20-30 seconds), the stream stops playing correctly (the picture crumbles). I suspect that somehow I am packing H.264 frames in RTP wrong.
Frames are generated by the Raspberry Pi hardware encoder, which is handled through OpenMAX. The data source for the encoder is the camera. Well, it's not really that important.
The received frame is passed to a small underlying function (send_data_to_rtp), which packs it into RTP. It takes arguments: data - pointer to the image in memory, len - image size, framerate - number of frames per second. Left a sufficient number of comments to the code.

// заголовки
  #define BUF_SIZE		1500
  #define RTP_PAYLOAD_MAX_SIZE	1400

  /* RTP ЗАГОЛОВОК. 12 байт */
  typedef struct{
    /* первый байт */
    uint8_t csrc_len: 	4;	
    uint8_t extension:	1;	
    uint8_t padding:	1;	
    uint8_t version:	2;	
    /* второй байт */
    uint8_t payload_type:  	7;	
    uint8_t marker:		1;	
    /* третий-четвертый байты */
    uint16_t seq_no;		
    /* пятый-восьмой байты */
    uint32_t timestamp;		
    /* девятый-двенадцатый байт */
    uint32_t ssrc;			
  }__attribute__ ((packed)) rtp_header;

  typedef struct {
    uint8_t type:		5;	
    uint8_t nri: 		2;	/*  */
    uint8_t f:		1;	/*  */
  }__attribute__ ((packed)) nalu_header;


  typedef struct {
    uint8_t type: 5;
    uint8_t nri: 2;
    uint8_t f: 1;
  } __attribute__ ((packed)) fu_indicator;


  typedef struct {
    uint8_t type: 5;
    uint8_t r: 1;
    uint8_t e: 1;
    uint8_t s: 1;
  } __attribute__ ((packed)) fu_header;

.......
// Формируем пакет
static void send_data_to_rtp(uint8_t *data, int len, int framerate)
{
  static uint8_t sendbuf[BUF_SIZE];
  static uint32_t ts_current = 0;
  static uint16_t seq_num = 0;
  static uint16_t pack_num, last_pack_size, current_pack;

  uint8_t *nalu_playload;
  /* заголовок RTP */
  rtp_header *rtp_hdr;
  /* Заголовок NALU */
  nalu_header *nalu_hdr;
  /* Заголовк NALU фрагментированых данных */
  fu_indicator *fu_ind;
  /* заголовк NALU фрагментированых данных. Идентифицирует фрагмент */
  fu_header *fu_hdr;

  ts_current += (90000 / framerate);

  memset(sendbuf, 0, sizeof(sendbuf));

  rtp_hdr = (rtp_header*)&sendbuf[0];
  
  rtp_hdr->version = 2;
  rtp_hdr->marker = 0;
  rtp_hdr->csrc_len = 0;
  rtp_hdr->extension = 0;
  rtp_hdr->padding = 0;
  rtp_hdr->ssrc = htonl(SSRC_NUM);
  rtp_hdr->payload_type = TYPE_H264;
  rtp_hdr->timestamp = htonl(ts_current);

  if (len <= RTP_PAYLOAD_MAX_SIZE) {
    rtp_hdr->marker = 1;
    rtp_hdr->seq_no = htons(++seq_num);

    nalu_hdr = (nalu_header*)&sendbuf[12];
    /* тип фрагмента */
    nalu_hdr->type = data[0] & 0x1f;
    nalu_hdr->f = data[0] & 0x80;
    nalu_hdr->nri = data[0] & 0x60 >> 5;

    nalu_playload = (uint8_t*)&sendbuf[13];
    /* Копирую буфер без ЗАГОЛОВКА(т.е. первого байта, являющийся заголовком NALU)! */
    memcpy(nalu_playload, data + 1, len-1);
    
    send_data_client(sendbuf, len + 13);
  } else {
    /* пакет не помещается в MTU, значит будем фрагментировать. 
    * Начало s = 1, e = r = 0
    * Середина s = 0, e = 0  r = 0
    * Конец s = 0, e = 1 r = 0
    */
    /* определяю требуемое колличество пакетов */
    pack_num = (len % RTP_PAYLOAD_MAX_SIZE) ? (len / RTP_PAYLOAD_MAX_SIZE + 1) : (len / RTP_PAYLOAD_MAX_SIZE);
    /* размер данных в последнем пакете */
    last_pack_size = (len % RTP_PAYLOAD_MAX_SIZE) ? (len % RTP_PAYLOAD_MAX_SIZE) : (RTP_PAYLOAD_MAX_SIZE);

    current_pack = 0;

    /* описываю заголовки, которые используются на каждой иттерации */
    fu_ind = (fu_indicator *)&sendbuf[12];
    fu_ind->f = data[0] & 0x80;
    fu_ind->nri = (data[0] & 0x60) >> 5;
    /* говорим, что пакет у нас фрагментирован */
    fu_ind->type = 28;
    fu_hdr = (fu_header *)&sendbuf[13];
    fu_hdr->type = data[0] & 0x1f;
    
    while (current_pack < pack_num) {
      rtp_hdr->seq_no = htons(++seq_num);
      /* первый пакет */
      if(current_pack == 0) {
        /* начало фрагментированого блока */
        fu_hdr->s = 1, fu_hdr->e = 0, fu_hdr->r = 0;
        rtp_hdr->marker = 0;
        
        nalu_playload = (uint8_t*)&sendbuf[14];
        memset(nalu_playload, 0, RTP_PAYLOAD_MAX_SIZE);
        /* Копирую буфер без ЗАГОЛОВКА(т.е. первого байта, являющийся заголовком NALU)! */
        memcpy(nalu_playload, data + 1, RTP_PAYLOAD_MAX_SIZE);
        send_data_client(sendbuf, RTP_PAYLOAD_MAX_SIZE + 14);
      /* середина */
      } else if(current_pack < pack_num - 1){
        fu_hdr->s = 0, fu_hdr->e = 0, fu_hdr->r = 0;
        rtp_hdr->marker = 0;

        nalu_playload = (uint8_t*)&sendbuf[14];
        memset(nalu_playload, 0, RTP_PAYLOAD_MAX_SIZE);

        memcpy(nalu_playload, data + (current_pack * RTP_PAYLOAD_MAX_SIZE) + 1, RTP_PAYLOAD_MAX_SIZE);
        send_data_client(sendbuf, RTP_PAYLOAD_MAX_SIZE + 14);
      /* последний пакет */
      } else {
        rtp_hdr->marker = 1;
        nalu_playload = (uint8_t*)&sendbuf[14];
        fu_hdr->s = 0, fu_hdr->e = 1, fu_hdr->r = 0;
        memset(nalu_playload, 0, RTP_PAYLOAD_MAX_SIZE);

        memcpy(nalu_playload, data + (current_pack * RTP_PAYLOAD_MAX_SIZE) + 1, last_pack_size - 1);
        send_data_client(sendbuf, last_pack_size - 1 + 14);
      }
      current_pack += 1;
    }
  }
}




// Тут пишу в сокет.
static void send_data_client(uint8_t *send_buf, size_t len_sendbuf)
{
  sendto(socket_fd, send_buf, len_sendbuf, 0, (struct sockaddr *)&addr,sizeof(addr));
}

One of the frames looks like this: 0000 0001 2588 804f ec82 b9bf e0b7 1789 .....
Before calling the send_data_to_rtp function, I cut off 00 00 00 01. After the RTP header (12 bytes) and two more bytes (headers for fragmentation), the sequence is ready The package looks like this 88 804f ec82 b9bf e0b7 1789 .....
The frame data in the correct order “spreads” over the packages (I looked in the wireshark) nothing extra is copied or lost. So the question is: Am I formatting the packet headers correctly?
When playing, errors appear, like (played the stream in mplayer):

V:  22.3   0/  0 10%  1%  0.0% 0 0 
[h264 @ 0xb6497640]error while decoding MB 1 14, bytestream (-5)
[h264 @ 0xb6497640]concealing 68 DC, 68 AC, 68 MV errors
V:  22.3   0/  0 10%  1%  0.0% 0 0 
[h264 @ 0xb6497640]error while decoding MB 16 14, bytestream (-9)
[h264 @ 0xb6497640]concealing 53 DC, 53 AC, 53 MV errors
V:  22.6   0/  0 10%  1%  0.0% 0 0 
[h264 @ 0xb6497640]error while decoding MB 17 3, bytestream (-3)
[h264 @ 0xb6497640]concealing 272 DC, 272 AC, 272 MV errors
V:  22.7   0/  0 10%  1%  0.0% 0 0

Answer 1

In the absence of RTCP. Without it, almost all clients / servers break the connection after 30 seconds. At least in all the cases that I encountered it was so. If memory serves, 30 seconds is the maximum RTCP sending interval.
Try to set the logging level higher or reproduce with ffplay -v debug rtp://xxx