Why and how to fix this error with video output to gtk widget using ffmpeg?

Question

linux

I've been working on this for several days now. At first I did it on gstreamer, but in this library the video starts to be processed only when the stream is assigned to a widget, for example gtk_drawing_area. but I want to stream video over the network, and since I could not find a solution in gstreamer, I decided to try ffmpeg. In general, I wrote such a code. and I'll show you what happened.

/* dating_chat-window.c
 *
 * Copyright 2021 cf
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "dating-chat-window.h"
#include <libavdevice/avdevice.h>
#include <libavcodec/avcodec.h>
#include <libswscale/swscale.h>
#include <libavutil/avutil.h>
#include <libavutil/imgutils.h>
#include <libavutil/pixfmt.h>

struct _DatingChatWindow
{
  GtkApplicationWindow  parent_instance;

  AVFormatContext *ctx;
  AVCodecContext *codec_ctx;
  struct SwsContext *sws;
  int video_stream;
  AVFrame *frame;
  AVFrame *frame_rgb;
  uint8_t *buffer;
  int num;
  /* Template widgets */
  GtkWidget *area;

};

G_DEFINE_TYPE (DatingChatWindow, dating_chat_window, GTK_TYPE_APPLICATION_WINDOW)

static void
dating_chat_window_class_init (DatingChatWindowClass *klass)
{
  GtkWidgetClass *widget_class = GTK_WIDGET_CLASS (klass);
  (void) widget_class;
}

gboolean
drawing_render_cb (GtkWidget *widget, cairo_t *cr, gpointer data) {
  DatingChatWindow *self = (DatingChatWindow *) data;

  AVPacket packet;
  int frame_finished;

  av_read_frame (self->ctx, &packet);

  if (packet.stream_index == self->video_stream) {
    static int a = 1;
    avcodec_send_packet (self->codec_ctx, &packet);
    avcodec_receive_frame (self->codec_ctx, self->frame);
    int width, height;
    gtk_widget_get_size_request (self->area, &width, &height);

#if 0
    av_image_fill_arrays (
          self->frame_rgb->data,
          self->frame_rgb->linesize,
          self->buffer,
          AV_PIX_FMT_RGB24,
          self->codec_ctx->width,
          self->codec_ctx->height,
          1
        );
#endif
#if 1
    uint8_t *rgb[1] = { self->buffer };
    int rgb_stride[4] = { self->codec_ctx->width * 3, 0, 0, 0 };
    sws_scale (
        self->sws,
        (const uint8_t * const *) self->frame->data,
        self->frame->linesize,
        0,
        height,
        rgb,
        rgb_stride
        );

#endif
#if 0
    int rt = av_image_copy_to_buffer (self->buffer,
        self->num,
        (const uint8_t * const *) self->frame_rgb->data,
        self->frame_rgb->linesize,
      		AV_PIX_FMT_YUYV422,
        //AV_PIX_FMT_RGB24,
        width,
        height,
        1
        );

#endif
    int stride = cairo_format_stride_for_width (CAIRO_FORMAT_RGB24, self->codec_ctx->width);

    cairo_surface_t *surface = cairo_image_surface_create_for_data (
        self->buffer,
        CAIRO_FORMAT_RGB24,
        self->codec_ctx->width,
        self->codec_ctx->height,
        stride);

    cairo_set_source_surface (cr, surface, 0, 0);
    cairo_paint (cr);
    cairo_surface_finish (surface);

  }

  av_packet_unref (&packet);

  return FALSE;
}

gboolean
timeout_frame (gpointer data) {
  DatingChatWindow *self = (DatingChatWindow *) data;

  gtk_widget_queue_draw (self->area);

  return G_SOURCE_CONTINUE;
}

static void
dating_chat_window_init (DatingChatWindow *self)
{


  self->area = gtk_drawing_area_new ( );

  

  avdevice_register_all ();
  AVInputFormat *input_format = av_find_input_format ("v4l2");
  AVFormatContext *ctx = NULL;
  AVCodec *codec_in = NULL;
  AVCodecContext *codec_ctx = NULL;

  AVDictionary *options = NULL;
  av_dict_set (&options, "video_size", "640x480", 0);
  av_dict_set (&options, "framerate", "30", 0);
  //av_dict_set (&options, "pixel_format", "mjpeg", 0);
  av_dict_set (&options, "pixel_format", "yuyv422", 0);


  int ret = avformat_open_input (&ctx, "/dev/video0", input_format, &options);

  avformat_find_stream_info (ctx, NULL);

  int video_stream_id = -1;

  for (int i = 0; i < ctx->nb_streams; i++) {
    if (ctx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
      printf ("found: %d\n", i);
      video_stream_id = i;
      break;
    }
  }
  self->video_stream = video_stream_id;

  printf ("codec id: %d\n", ctx->streams[video_stream_id]->codecpar->codec_id);
  codec_in = avcodec_find_decoder (ctx->streams[video_stream_id]->codecpar->codec_id);
  if (codec_in == NULL) {
    fprintf (stderr, "decoder not found\n");
    exit (-1);
  }

  codec_ctx = avcodec_alloc_context3 (codec_in);
  self->codec_ctx = codec_ctx;
  printf ("codec ctx: %p\n", codec_ctx);
  codec_ctx->width = 640;
  codec_ctx->height = 480;
  codec_ctx->sample_rate = 1000 / 33;
  codec_ctx->pix_fmt = AV_PIX_FMT_YUYV422;

  ret = avcodec_open2 ((AVCodecContext *) codec_ctx, codec_in, &options);
  char *buf = calloc (512, 1);
  av_strerror (ret, buf, 512);
  printf ("avcodec open2: %d %s\n", ret, buf);
  free (buf);

  AVFrame *frame = NULL;
  AVFrame *frame_rgb = NULL;

  frame = av_frame_alloc ( );
  frame_rgb = av_frame_alloc ( );

  self->frame = frame;
  self->frame_rgb = frame_rgb;

  uint8_t *buffer = NULL;

  int numbytes;

  printf ("codec_ctx->width: %d\n", codec_ctx->width);
  printf ("codec_ctx->height: %d\n", codec_ctx->height);

  numbytes = av_image_get_buffer_size (AV_PIX_FMT_RGB24, codec_ctx->width, codec_ctx->height, 1);
  self->num = numbytes;
  self->buffer = (uint8_t *) av_malloc (numbytes);



  self->ctx = ctx;
#if 1
  self->sws = sws_getContext (codec_ctx->width, 
      codec_ctx->height,
      codec_ctx->pix_fmt,
      codec_ctx->width,
      codec_ctx->height,
      AV_PIX_FMT_RGB24,
      SWS_BILINEAR,
      NULL,
      NULL,
      NULL
      );
#endif

  g_signal_connect (self->area, "draw", G_CALLBACK (drawing_render_cb), self);
  gtk_widget_set_size_request (self->area, 640, 480);
  gtk_widget_set_visible (self->area, TRUE);

  gtk_container_add (GTK_CONTAINER (self), self->area);

  //g_idle_add (timeout_frame, self);
  g_timeout_add ( 1000 / codec_ctx->sample_rate, timeout_frame, self);
}

I seem to be doing everything right, but it does not work as it should. the conversion process is done using sws_ctx. but he does it in a weird way.

Answer 1

everything I could do. Now we need to work on memory leaks, if any.
here is the code.

/* dating_chat-window.c
 *
 * Copyright 2021 cf
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "dating-chat-window.h"
#include <libavdevice/avdevice.h>
#include <libavcodec/avcodec.h>
#include <libswscale/swscale.h>
#include <libavutil/avutil.h>
#include <libavutil/imgutils.h>
#include <libavutil/pixfmt.h>

struct _DatingChatWindow
{
  GtkApplicationWindow  parent_instance;

  AVFormatContext *ctx;
  AVCodecContext *codec_ctx;
  struct SwsContext *sws;
  int video_stream;
  AVFrame *frame;
  AVFrame *frame_rgb;
  uint8_t *buffer;
  uint8_t *b;
  int num;
  /* Template widgets */
  GtkWidget *area;

};

G_DEFINE_TYPE (DatingChatWindow, dating_chat_window, GTK_TYPE_APPLICATION_WINDOW)

static void
dating_chat_window_class_init (DatingChatWindowClass *klass)
{
  GtkWidgetClass *widget_class = GTK_WIDGET_CLASS (klass);
  (void) widget_class;
}

gboolean
drawing_render_cb (GtkWidget *widget, cairo_t *cr, gpointer data) {
  DatingChatWindow *self = (DatingChatWindow *) data;

  AVPacket packet;
  int frame_finished;

  av_read_frame (self->ctx, &packet);

  if (packet.stream_index == self->video_stream) {
    static int a = 1;
    avcodec_send_packet (self->codec_ctx, &packet);
    avcodec_receive_frame (self->codec_ctx, self->frame);
    int width, height;
    gtk_widget_get_size_request (self->area, &width, &height);

    uint8_t *rgb[4] = { self->buffer, 0, 0, 0 };
    int rgb_stride[4] = { self->codec_ctx->width * 3, 0, 0, 0 };
    memset ( self->buffer, 255, self->num);

    self->sws = sws_getCachedContext (
        self->sws,
        width,
        height,
        AV_PIX_FMT_YUYV422,
        width,
        height,
        AV_PIX_FMT_RGB24,
        SWS_BICUBIC,
        0,
        0,
        0
        );

    sws_scale (
        self->sws,
        (const uint8_t * const *) self->frame->data,
        self->frame->linesize,
        0,
        height,
        rgb,
        rgb_stride
        );

    int ind = 0;
    for (int i = 0; i < self->num;) {
      self->b[ind + 0] = self->buffer[i + 2];
      self->b[ind + 1] = self->buffer[i + 1];
      self->b[ind + 2] = self->buffer[i + 0];
      self->b[ind + 3] = 255;
      ind += 4;
      i += 3;
    }

    int stride = cairo_format_stride_for_width (CAIRO_FORMAT_RGB24, width);

    cairo_surface_t *surface = cairo_image_surface_create_for_data (
        self->b,
        CAIRO_FORMAT_RGB24,
        self->codec_ctx->width,
        self->codec_ctx->height,
        stride);

    cairo_set_source_surface (cr, surface, 0, 0);
    cairo_paint (cr);
    cairo_surface_finish (surface);

  }

  av_packet_unref (&packet);

  return FALSE;
}

gboolean
timeout_frame (gpointer data) {
  DatingChatWindow *self = (DatingChatWindow *) data;

  gtk_widget_queue_draw (self->area);

  return G_SOURCE_CONTINUE;
}

static void
dating_chat_window_init (DatingChatWindow *self)
{


  self->area = gtk_drawing_area_new ( );

  

  avdevice_register_all ();
  AVInputFormat *input_format = av_find_input_format ("v4l2");
  AVFormatContext *ctx = NULL;
  AVCodec *codec_in = NULL;
  AVCodecContext *codec_ctx = NULL;

  AVDictionary *options = NULL;
  av_dict_set (&options, "video_size", "640x480", 0);
  av_dict_set (&options, "framerate", "30", 0);
  //av_dict_set (&options, "pixel_format", "mjpeg", 0);
  av_dict_set (&options, "pixel_format", "yuyv422", 0);


  int ret = avformat_open_input (&ctx, "/dev/video0", input_format, &options);

  avformat_find_stream_info (ctx, NULL);

  int video_stream_id = -1;

  for (int i = 0; i < ctx->nb_streams; i++) {
    if (ctx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
      printf ("found: %d\n", i);
      video_stream_id = i;
      break;
    }
  }
  self->video_stream = video_stream_id;

  printf ("codec id: %d\n", ctx->streams[video_stream_id]->codecpar->codec_id);
  codec_in = avcodec_find_decoder (ctx->streams[video_stream_id]->codecpar->codec_id);
  if (codec_in == NULL) {
    fprintf (stderr, "decoder not found\n");
    exit (-1);
  }

  codec_ctx = avcodec_alloc_context3 (codec_in);
  self->codec_ctx = codec_ctx;
  printf ("codec ctx: %p\n", codec_ctx);
  codec_ctx->width = 640;
  codec_ctx->height = 480;
  codec_ctx->sample_rate = 1000 / 33;
  codec_ctx->pix_fmt = AV_PIX_FMT_YUYV422;

  ret = avcodec_open2 ((AVCodecContext *) codec_ctx, codec_in, &options);
  char *buf = calloc (512, 1);
  av_strerror (ret, buf, 512);
  printf ("avcodec open2: %d %s\n", ret, buf);
  free (buf);

  AVFrame *frame = NULL;
  AVFrame *frame_rgb = NULL;

  frame = av_frame_alloc ( );
  frame_rgb = av_frame_alloc ( );

  self->frame = frame;
  self->frame_rgb = frame_rgb;

  uint8_t *buffer = NULL;

  int numbytes;

  printf ("codec_ctx->width: %d\n", codec_ctx->width);
  printf ("codec_ctx->height: %d\n", codec_ctx->height);

  numbytes = av_image_get_buffer_size (AV_PIX_FMT_RGB24, codec_ctx->width, codec_ctx->height, 1);
  g_print ("numbytes: %d\n", numbytes);
  self->num = numbytes;
  self->buffer = (uint8_t *) av_malloc (numbytes);
  self->b = (uint8_t *) av_malloc (numbytes);



  self->ctx = ctx;
#if 1
  self->sws = sws_getContext (codec_ctx->width, 
      codec_ctx->height,
      AV_PIX_FMT_YUYV422,
      codec_ctx->width,
      codec_ctx->height,
      AV_PIX_FMT_RGB24,
      SWS_BICUBIC,
      NULL,
      NULL,
      NULL
      );
#endif

  g_signal_connect (self->area, "draw", G_CALLBACK (drawing_render_cb), self);
  gtk_widget_set_size_request (self->area, 640, 480);
  gtk_widget_set_visible (self->area, TRUE);

  gtk_container_add (GTK_CONTAINER (self), self->area);

  //g_idle_add (timeout_frame, self);
  g_timeout_add ( codec_ctx->sample_rate, timeout_frame, self);
}