TWCCMan evaluates recovery pct for FEC

subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtprepairmeta.c

@@ -0,0 +1,159 @@
+/* GStreamer
+ * Copyright (C) <2024> Mikhail Baranov <[email protected]>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "gstrtprepairmeta.h"
+#include <string.h>
+
+static gboolean gst_rtp_repair_meta_init(GstRTPRepairMeta * meta, 
+    G_GNUC_UNUSED gpointer params, G_GNUC_UNUSED GstBuffer * buffer);
+static void gst_rtp_repair_meta_free(GstRTPRepairMeta *meta,
+    G_GNUC_UNUSED GstBuffer *buffer);
+
+
+GType gst_rtp_repair_meta_api_get_type(void)
+{
+  static GType type = 0;
+  static const gchar *tags[] = {NULL};
+
+  if (g_once_init_enter(&type)) {
+    GType _type = gst_meta_api_type_register("GstRTPRepairMetaAPI", tags);
+    g_once_init_leave(&type, _type);
+  }
+
+  return type;
+}
+
+const GstMetaInfo *gst_rtp_repair_meta_get_info(void)
+{
+  static const GstMetaInfo *meta_info = NULL;
+
+  if (g_once_init_enter(&meta_info)) {
+    const GstMetaInfo *mi = gst_meta_register( GST_RTP_REPAIR_META_API_TYPE,
+        "GstRTPRepairMeta",
+        sizeof(GstRTPRepairMeta),
+        (GstMetaInitFunction) gst_rtp_repair_meta_init,
+        (GstMetaFreeFunction) gst_rtp_repair_meta_free,
+        NULL );
+    g_once_init_leave(&meta_info, mi);
+  }
+
+  return meta_info;
+}
+
+GstRTPRepairMeta *gst_buffer_get_rtp_repair_meta(GstBuffer *buffer)
+{
+  return (GstRTPRepairMeta *)gst_buffer_get_meta(buffer,
+    gst_rtp_repair_meta_api_get_type());
+}
+
+GstRTPRepairMeta *gst_buffer_add_rtp_repair_meta(GstBuffer *buffer,
+  const guint16 idx_red_packets, const guint16 num_red_packets,
+  const guint32 ssrc, const guint16 *seqnum, guint seqnum_count)
+{
+  GstRTPRepairMeta *repair_meta = (GstRTPRepairMeta *) gst_buffer_add_meta (buffer,
+      GST_RTP_REPAIR_META_INFO, NULL);
+  if (repair_meta == NULL) {
+    return NULL;
+  }
+
+  repair_meta->idx_red_packets = idx_red_packets;
+  repair_meta->num_red_packets = num_red_packets;
+  repair_meta->ssrc = ssrc;
+  g_array_insert_vals (repair_meta->seqnums, 0, seqnum, seqnum_count);
+
+  return repair_meta;
+}
+
+gboolean gst_buffer_repairs_seqnum(GstBuffer *buffer, guint16 seqnum, guint32 ssrc)
+{
+  GstRTPRepairMeta *repair_meta = gst_buffer_get_rtp_repair_meta(buffer);
+  if (repair_meta) {
+    if (repair_meta->ssrc != ssrc) {
+      return FALSE;
+    }
+
+    for (guint i = 0; i < repair_meta->seqnums->len; i++) {
+      guint16 stored_seqnum = g_array_index(repair_meta->seqnums, guint16, i);
+      if (stored_seqnum == seqnum) {
+        return TRUE;
+      }
+    }
+  }
+  return FALSE;
+}
+
+gint gst_buffer_get_repair_idx(GstBuffer *buffer)
+{
+  GstRTPRepairMeta *repair_meta = gst_buffer_get_rtp_repair_meta(buffer);
+  if (repair_meta) {
+    return repair_meta->idx_red_packets;
+  }
+  return -1;
+}
+
+gint gst_buffer_get_repair_num(GstBuffer *buffer)
+{
+  GstRTPRepairMeta *repair_meta = gst_buffer_get_rtp_repair_meta(buffer);
+  if (repair_meta) {
+    return repair_meta->num_red_packets;
+  }
+  return -1;
+}
+
+gboolean gst_buffer_get_repair_seqnums(GstBuffer *buffer, guint32 *ssrc,
+    GArray **seqnums)
+{
+  GstRTPRepairMeta *repair_meta = gst_buffer_get_rtp_repair_meta(buffer);
+  if (repair_meta && repair_meta->seqnums->len > 0) {
+    if (ssrc) {
+      *ssrc = repair_meta->ssrc;
+    }
+    if (seqnums) {
+      *seqnums = g_array_ref (repair_meta->seqnums);
+    }
+    return TRUE;
+  } else {
+    *ssrc = 0;
+    *seqnums = NULL;
+  }
+  return FALSE;
+}
+
+static gboolean
+gst_rtp_repair_meta_init(GstRTPRepairMeta * meta, G_GNUC_UNUSED gpointer params, 
+    G_GNUC_UNUSED GstBuffer * buffer)
+{
+  meta->idx_red_packets = 0;
+  meta->num_red_packets = 0;
+  meta->ssrc = 0;
+  meta->seqnums = g_array_new(FALSE, FALSE, sizeof(guint16));
+
+  return TRUE;
+}
+
+static void
+gst_rtp_repair_meta_free(GstRTPRepairMeta *meta,
+    G_GNUC_UNUSED GstBuffer *buffer)
+{
+  g_array_unref (meta->seqnums);
+}

subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtprepairmeta.h

@@ -0,0 +1,86 @@
+/* GStreamer
+ * Copyright (C) <2024> Mikhail Baranov <[email protected]>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __GST_RTP_REPAIR_META_H__
+#define __GST_RTP_REPAIR_META_H__
+
+#include <gst/gst.h>
+#include <glib.h>
+#include <gst/rtp/rtp-prelude.h>
+
+G_BEGIN_DECLS
+
+#define GST_RTP_REPAIR_META_API_TYPE  (gst_rtp_repair_meta_api_get_type())
+#define GST_RTP_REPAIR_META_INFO  (gst_rtp_repair_meta_get_info())
+typedef struct _GstRTPRepairMeta GstRTPRepairMeta;
+
+/**
+ * GstRTPRepairMeta:
+ * @meta: parent GstMeta structure
+ * @idx_red_packets: index of this redundant packet for the block
+ * @num_red_packets: number of redundant packets
+ * @ssrc: SSRC
+ * @seqnums: array of sequence numbers of data packets
+ *
+ * Meta describing the reduandant packet, e.g. FEC or RTX.
+ * It is used to tie together the original packet and the redundant packets.
+ */
+struct _GstRTPRepairMeta
+{
+  GstMeta meta;
+
+  guint16 idx_red_packets;
+  guint16 num_red_packets;
+  guint32 ssrc;
+  GArray *seqnums;
+};
+
+GST_RTP_API
+GType               gst_rtp_repair_meta_api_get_type     (void);
+
+GST_RTP_API
+GstRTPRepairMeta *gst_buffer_add_rtp_repair_meta(GstBuffer *buffer,
+      const guint16 idx_red_packets, const guint16 num_red_packets,
+      const guint32 ssrc, const guint16 *seqnum, guint seqnum_count);
+
+GST_RTP_API
+GstRTPRepairMeta *  gst_buffer_get_rtp_repair_meta       (GstBuffer * buffer);
+
+GST_RTP_API
+gboolean gst_buffer_repairs_seqnum(GstBuffer *buffer, guint16 seqnum, guint32 ssrc);
+
+GST_RTP_API
+gboolean gst_buffer_get_repair_seqnums(GstBuffer *buffer, guint32 * ssrc,
+    GArray ** seqnums);
+
+/* If this packet is a FEC/RTX packet, what is it sequential number a block */
+GST_RTP_API
+gint gst_buffer_get_repair_idx(GstBuffer *buffer);
+
+/* If this packet is a FEC/RTX packet, how many redundancy packets are in a block.
+ * -1 if not a repair packet */ 
+GST_RTP_API
+gint gst_buffer_get_repair_num(GstBuffer *buffer);
+
+GST_RTP_API
+const GstMetaInfo * gst_rtp_repair_meta_get_info         (void);
+
+G_END_DECLS
+
+#endif /* __GST_RTP_REPAIR_META_H__ */

subprojects/gst-plugins-base/gst-libs/gst/rtp/meson.build

@@ -4,6 +4,7 @@ rtp_sources = files([
   'gstrtppayloads.c',
   'gstrtphdrext.c',
   'gstrtpmeta.c',
+  'gstrtprepairmeta.c',
   'gstrtpbaseaudiopayload.c',
   'gstrtpbasepayload.c',
   'gstrtpbasedepayload.c'
@@ -18,6 +19,7 @@ rtp_headers = files([
   'gstrtpdefs.h',
   'gstrtphdrext.h',
   'gstrtpmeta.h',
+  'gstrtprepairmeta.h',
   'gstrtppayloads.h',
   'rtp-prelude.h',
   'rtp.h',

subprojects/gst-plugins-good/gst/rtp/gstrtpulpfecenc.c

@@ -86,6 +86,7 @@
 
 #include <gst/rtp/gstrtp-enumtypes.h>
 #include <gst/rtp/gstrtpbuffer.h>
+#include <gst/rtp/gstrtprepairmeta.h>
 #include <string.h>
 
 #include "gstrtpelements.h"
@@ -150,6 +151,7 @@ gst_rtp_ulpfec_enc_stream_ctx_start (GstRtpUlpFecEncStreamCtx * ctx,
   guint i;
 
   g_array_set_size (ctx->info_arr, packets->length);
+  g_array_set_size (ctx->block_seqnums, packets->length);
 
   for (i = 0; i < packets->length; ++i) {
     GstBuffer *buffer = it->data;
@@ -159,6 +161,8 @@ gst_rtp_ulpfec_enc_stream_ctx_start (GstRtpUlpFecEncStreamCtx * ctx,
       g_assert_not_reached ();
 
     GST_LOG_RTP_PACKET (ctx->parent, "rtp header (incoming)", &info->rtp);
+    g_array_index (ctx->block_seqnums, guint16, i) =
+        gst_rtp_buffer_get_seq (&info->rtp);
 
     it = g_list_previous (it);
   }
@@ -388,6 +392,12 @@ gst_rtp_ulpfec_enc_stream_ctx_push_fec_packets (GstRtpUlpFecEncStreamCtx * ctx,
         gst_rtp_buffer_unmap (&rtp);
       }
 
+      /*Add repair packet meta so that TWCC will be able to to tie it 
+        with lost packets */
+      gst_buffer_add_rtp_repair_meta (fec,
+        fec_packets_pushed, fec_packets_num, ctx->ssrc,
+        (guint16*)ctx->block_seqnums->data, ctx->block_seqnums->len);
+
       GST_LOG_OBJECT (ctx->parent, "ctx %p pushing generated fec buffer %"
           GST_PTR_FORMAT, ctx, fec);
       ret = gst_pad_push (ctx->srcpad, fec);
@@ -487,6 +497,7 @@ gst_rtp_ulpfec_enc_stream_ctx_new (guint ssrc,
   ctx->info_arr = g_array_new (FALSE, TRUE, sizeof (RtpUlpFecMapInfo));
   g_array_set_clear_func (ctx->info_arr,
       (GDestroyNotify) rtp_ulpfec_map_info_unmap);
+  ctx->block_seqnums = g_array_new(FALSE, FALSE, sizeof(guint16));
   ctx->parent = parent;
   ctx->scratch_buf = g_array_new (FALSE, TRUE, sizeof (guint8));
   gst_rtp_ulpfec_enc_stream_ctx_configure (ctx, pt,
@@ -508,6 +519,7 @@ gst_rtp_ulpfec_enc_stream_ctx_free (GstRtpUlpFecEncStreamCtx * ctx)
   g_assert (0 == ctx->info_arr->len);
   g_array_free (ctx->info_arr, TRUE);
   g_array_free (ctx->scratch_buf, TRUE);
+  g_array_free (ctx->block_seqnums, TRUE);
   g_free (ctx);
 }
 

subprojects/gst-plugins-good/gst/rtp/gstrtpulpfecenc.h

@@ -88,6 +88,7 @@ typedef struct {
 
   GArray *info_arr;
   GArray *scratch_buf;
+  GArray *block_seqnums;
 
   guint fec_packets;
   guint fec_packet_idx;

subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxsend.c

@@ -46,6 +46,7 @@
 
 #include "gstrtprtxsend.h"
 #include "rtpstats.h"
+#include <gst/rtp/gstrtprepairmeta.h>
 
 GST_DEBUG_CATEGORY_STATIC (gst_rtp_rtx_send_debug);
 #define GST_CAT_DEFAULT gst_rtp_rtx_send_debug
@@ -765,22 +766,31 @@ gst_rtp_rtx_buffer_new (GstRtpRtxSend * rtx, GstBuffer * buffer, guint8 padlen)
   SSRCRtxData *data;
   guint32 ssrc;
   guint16 seqnum;
+  guint32 orig_ssrc;
+  guint16 orig_seqnum;
   guint8 fmtp;
 
   gst_rtp_buffer_map (buffer, GST_MAP_READ, &rtp);
 
   /* get needed data from GstRtpRtxSend */
-  ssrc = gst_rtp_buffer_get_ssrc (&rtp);
-  data = gst_rtp_rtx_send_get_ssrc_data (rtx, ssrc);
+  orig_ssrc = gst_rtp_buffer_get_ssrc (&rtp);
+  data = gst_rtp_rtx_send_get_ssrc_data (rtx, orig_ssrc);
   ssrc = data->rtx_ssrc;
   seqnum = data->next_seqnum++;
   fmtp = GPOINTER_TO_UINT (g_hash_table_lookup (rtx->rtx_pt_map,
           GUINT_TO_POINTER (gst_rtp_buffer_get_payload_type (&rtp))));
+
+  orig_seqnum = gst_rtp_buffer_get_seq (&rtp);
 
   GST_DEBUG_OBJECT (rtx, "creating rtx buffer, orig seqnum: %u, "
-      "rtx seqnum: %u, rtx ssrc: %X", gst_rtp_buffer_get_seq (&rtp),
+      "rtx seqnum: %u, rtx ssrc: %X", orig_seqnum,
       seqnum, ssrc);
 
+  /*Add repair packet meta so that TWCC will be able to to tie it 
+    with a lost data packet */
+  gst_buffer_add_rtp_repair_meta (new_buffer,
+      0, 1, orig_ssrc, &orig_seqnum, 1);
+
   /* gst_rtp_buffer_map does not map the payload so do it now */
   gst_rtp_buffer_get_payload (&rtp);
 

subprojects/gst-plugins-good/gst/rtpmanager/meson.build

@@ -23,6 +23,7 @@ rtpmanager_sources = [
   'rtpstats.c',
   'rtptimerqueue.c',
   'rtptwcc.c',
+  'rtptwccstats.c',
   'gstrtpsession.c',
   'gstrtpfunnel.c',
   'gstrtpst2022-1-fecdec.c',