diff --git a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtprepairmeta.c b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtprepairmeta.c new file mode 100644 index 0000000000..22685c12d7 --- /dev/null +++ b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtprepairmeta.c @@ -0,0 +1,159 @@ +/* GStreamer + * Copyright (C) <2024> Mikhail Baranov + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstrtprepairmeta.h" +#include + +static gboolean gst_rtp_repair_meta_init(GstRTPRepairMeta * meta, + G_GNUC_UNUSED gpointer params, G_GNUC_UNUSED GstBuffer * buffer); +static void gst_rtp_repair_meta_free(GstRTPRepairMeta *meta, + G_GNUC_UNUSED GstBuffer *buffer); + + +GType gst_rtp_repair_meta_api_get_type(void) +{ + static GType type = 0; + static const gchar *tags[] = {NULL}; + + if (g_once_init_enter(&type)) { + GType _type = gst_meta_api_type_register("GstRTPRepairMetaAPI", tags); + g_once_init_leave(&type, _type); + } + + return type; +} + +const GstMetaInfo *gst_rtp_repair_meta_get_info(void) +{ + static const GstMetaInfo *meta_info = NULL; + + if (g_once_init_enter(&meta_info)) { + const GstMetaInfo *mi = gst_meta_register( GST_RTP_REPAIR_META_API_TYPE, + "GstRTPRepairMeta", + sizeof(GstRTPRepairMeta), + (GstMetaInitFunction) gst_rtp_repair_meta_init, + (GstMetaFreeFunction) gst_rtp_repair_meta_free, + NULL ); + g_once_init_leave(&meta_info, mi); + } + + return meta_info; +} + +GstRTPRepairMeta *gst_buffer_get_rtp_repair_meta(GstBuffer *buffer) +{ + return (GstRTPRepairMeta *)gst_buffer_get_meta(buffer, + gst_rtp_repair_meta_api_get_type()); +} + +GstRTPRepairMeta *gst_buffer_add_rtp_repair_meta(GstBuffer *buffer, + const guint16 idx_red_packets, const guint16 num_red_packets, + const guint32 ssrc, const guint16 *seqnum, guint seqnum_count) +{ + GstRTPRepairMeta *repair_meta = (GstRTPRepairMeta *) gst_buffer_add_meta (buffer, + GST_RTP_REPAIR_META_INFO, NULL); + if (repair_meta == NULL) { + return NULL; + } + + repair_meta->idx_red_packets = idx_red_packets; + repair_meta->num_red_packets = num_red_packets; + repair_meta->ssrc = ssrc; + g_array_insert_vals (repair_meta->seqnums, 0, seqnum, seqnum_count); + + return repair_meta; +} + +gboolean gst_buffer_repairs_seqnum(GstBuffer *buffer, guint16 seqnum, guint32 ssrc) +{ + GstRTPRepairMeta *repair_meta = gst_buffer_get_rtp_repair_meta(buffer); + if (repair_meta) { + if (repair_meta->ssrc != ssrc) { + return FALSE; + } + + for (guint i = 0; i < repair_meta->seqnums->len; i++) { + guint16 stored_seqnum = g_array_index(repair_meta->seqnums, guint16, i); + if (stored_seqnum == seqnum) { + return TRUE; + } + } + } + return FALSE; +} + +gint gst_buffer_get_repair_idx(GstBuffer *buffer) +{ + GstRTPRepairMeta *repair_meta = gst_buffer_get_rtp_repair_meta(buffer); + if (repair_meta) { + return repair_meta->idx_red_packets; + } + return -1; +} + +gint gst_buffer_get_repair_num(GstBuffer *buffer) +{ + GstRTPRepairMeta *repair_meta = gst_buffer_get_rtp_repair_meta(buffer); + if (repair_meta) { + return repair_meta->num_red_packets; + } + return -1; +} + +gboolean gst_buffer_get_repair_seqnums(GstBuffer *buffer, guint32 *ssrc, + GArray **seqnums) +{ + GstRTPRepairMeta *repair_meta = gst_buffer_get_rtp_repair_meta(buffer); + if (repair_meta && repair_meta->seqnums->len > 0) { + if (ssrc) { + *ssrc = repair_meta->ssrc; + } + if (seqnums) { + *seqnums = g_array_ref (repair_meta->seqnums); + } + return TRUE; + } else { + *ssrc = 0; + *seqnums = NULL; + } + return FALSE; +} + +static gboolean +gst_rtp_repair_meta_init(GstRTPRepairMeta * meta, G_GNUC_UNUSED gpointer params, + G_GNUC_UNUSED GstBuffer * buffer) +{ + meta->idx_red_packets = 0; + meta->num_red_packets = 0; + meta->ssrc = 0; + meta->seqnums = g_array_new(FALSE, FALSE, sizeof(guint16)); + + return TRUE; +} + +static void +gst_rtp_repair_meta_free(GstRTPRepairMeta *meta, + G_GNUC_UNUSED GstBuffer *buffer) +{ + g_array_unref (meta->seqnums); +} diff --git a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtprepairmeta.h b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtprepairmeta.h new file mode 100644 index 0000000000..dac88ff5d9 --- /dev/null +++ b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtprepairmeta.h @@ -0,0 +1,86 @@ +/* GStreamer + * Copyright (C) <2024> Mikhail Baranov + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_RTP_REPAIR_META_H__ +#define __GST_RTP_REPAIR_META_H__ + +#include +#include +#include + +G_BEGIN_DECLS + +#define GST_RTP_REPAIR_META_API_TYPE (gst_rtp_repair_meta_api_get_type()) +#define GST_RTP_REPAIR_META_INFO (gst_rtp_repair_meta_get_info()) +typedef struct _GstRTPRepairMeta GstRTPRepairMeta; + +/** + * GstRTPRepairMeta: + * @meta: parent GstMeta structure + * @idx_red_packets: index of this redundant packet for the block + * @num_red_packets: number of redundant packets + * @ssrc: SSRC + * @seqnums: array of sequence numbers of data packets + * + * Meta describing the reduandant packet, e.g. FEC or RTX. + * It is used to tie together the original packet and the redundant packets. + */ +struct _GstRTPRepairMeta +{ + GstMeta meta; + + guint16 idx_red_packets; + guint16 num_red_packets; + guint32 ssrc; + GArray *seqnums; +}; + +GST_RTP_API +GType gst_rtp_repair_meta_api_get_type (void); + +GST_RTP_API +GstRTPRepairMeta *gst_buffer_add_rtp_repair_meta(GstBuffer *buffer, + const guint16 idx_red_packets, const guint16 num_red_packets, + const guint32 ssrc, const guint16 *seqnum, guint seqnum_count); + +GST_RTP_API +GstRTPRepairMeta * gst_buffer_get_rtp_repair_meta (GstBuffer * buffer); + +GST_RTP_API +gboolean gst_buffer_repairs_seqnum(GstBuffer *buffer, guint16 seqnum, guint32 ssrc); + +GST_RTP_API +gboolean gst_buffer_get_repair_seqnums(GstBuffer *buffer, guint32 * ssrc, + GArray ** seqnums); + +/* If this packet is a FEC/RTX packet, what is it sequential number a block */ +GST_RTP_API +gint gst_buffer_get_repair_idx(GstBuffer *buffer); + +/* If this packet is a FEC/RTX packet, how many redundancy packets are in a block. + * -1 if not a repair packet */ +GST_RTP_API +gint gst_buffer_get_repair_num(GstBuffer *buffer); + +GST_RTP_API +const GstMetaInfo * gst_rtp_repair_meta_get_info (void); + +G_END_DECLS + +#endif /* __GST_RTP_REPAIR_META_H__ */ diff --git a/subprojects/gst-plugins-base/gst-libs/gst/rtp/meson.build b/subprojects/gst-plugins-base/gst-libs/gst/rtp/meson.build index 14bf1fdf1b..74b1a914e5 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/rtp/meson.build +++ b/subprojects/gst-plugins-base/gst-libs/gst/rtp/meson.build @@ -4,6 +4,7 @@ rtp_sources = files([ 'gstrtppayloads.c', 'gstrtphdrext.c', 'gstrtpmeta.c', + 'gstrtprepairmeta.c', 'gstrtpbaseaudiopayload.c', 'gstrtpbasepayload.c', 'gstrtpbasedepayload.c' @@ -18,6 +19,7 @@ rtp_headers = files([ 'gstrtpdefs.h', 'gstrtphdrext.h', 'gstrtpmeta.h', + 'gstrtprepairmeta.h', 'gstrtppayloads.h', 'rtp-prelude.h', 'rtp.h', diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtpulpfecenc.c b/subprojects/gst-plugins-good/gst/rtp/gstrtpulpfecenc.c index c95fe0d89a..1449176ffa 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtpulpfecenc.c +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtpulpfecenc.c @@ -86,6 +86,7 @@ #include #include +#include #include #include "gstrtpelements.h" @@ -150,6 +151,7 @@ gst_rtp_ulpfec_enc_stream_ctx_start (GstRtpUlpFecEncStreamCtx * ctx, guint i; g_array_set_size (ctx->info_arr, packets->length); + g_array_set_size (ctx->block_seqnums, packets->length); for (i = 0; i < packets->length; ++i) { GstBuffer *buffer = it->data; @@ -159,6 +161,8 @@ gst_rtp_ulpfec_enc_stream_ctx_start (GstRtpUlpFecEncStreamCtx * ctx, g_assert_not_reached (); GST_LOG_RTP_PACKET (ctx->parent, "rtp header (incoming)", &info->rtp); + g_array_index (ctx->block_seqnums, guint16, i) = + gst_rtp_buffer_get_seq (&info->rtp); it = g_list_previous (it); } @@ -388,6 +392,12 @@ gst_rtp_ulpfec_enc_stream_ctx_push_fec_packets (GstRtpUlpFecEncStreamCtx * ctx, gst_rtp_buffer_unmap (&rtp); } + /*Add repair packet meta so that TWCC will be able to to tie it + with lost packets */ + gst_buffer_add_rtp_repair_meta (fec, + fec_packets_pushed, fec_packets_num, ctx->ssrc, + (guint16*)ctx->block_seqnums->data, ctx->block_seqnums->len); + GST_LOG_OBJECT (ctx->parent, "ctx %p pushing generated fec buffer %" GST_PTR_FORMAT, ctx, fec); ret = gst_pad_push (ctx->srcpad, fec); @@ -487,6 +497,7 @@ gst_rtp_ulpfec_enc_stream_ctx_new (guint ssrc, ctx->info_arr = g_array_new (FALSE, TRUE, sizeof (RtpUlpFecMapInfo)); g_array_set_clear_func (ctx->info_arr, (GDestroyNotify) rtp_ulpfec_map_info_unmap); + ctx->block_seqnums = g_array_new(FALSE, FALSE, sizeof(guint16)); ctx->parent = parent; ctx->scratch_buf = g_array_new (FALSE, TRUE, sizeof (guint8)); gst_rtp_ulpfec_enc_stream_ctx_configure (ctx, pt, @@ -508,6 +519,7 @@ gst_rtp_ulpfec_enc_stream_ctx_free (GstRtpUlpFecEncStreamCtx * ctx) g_assert (0 == ctx->info_arr->len); g_array_free (ctx->info_arr, TRUE); g_array_free (ctx->scratch_buf, TRUE); + g_array_free (ctx->block_seqnums, TRUE); g_free (ctx); } diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtpulpfecenc.h b/subprojects/gst-plugins-good/gst/rtp/gstrtpulpfecenc.h index a92fc3d1ad..b8917495ea 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtpulpfecenc.h +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtpulpfecenc.h @@ -88,6 +88,7 @@ typedef struct { GArray *info_arr; GArray *scratch_buf; + GArray *block_seqnums; guint fec_packets; guint fec_packet_idx; diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxsend.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxsend.c index 377b31f328..142632b2bf 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxsend.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxsend.c @@ -46,6 +46,7 @@ #include "gstrtprtxsend.h" #include "rtpstats.h" +#include GST_DEBUG_CATEGORY_STATIC (gst_rtp_rtx_send_debug); #define GST_CAT_DEFAULT gst_rtp_rtx_send_debug @@ -765,22 +766,31 @@ gst_rtp_rtx_buffer_new (GstRtpRtxSend * rtx, GstBuffer * buffer, guint8 padlen) SSRCRtxData *data; guint32 ssrc; guint16 seqnum; + guint32 orig_ssrc; + guint16 orig_seqnum; guint8 fmtp; gst_rtp_buffer_map (buffer, GST_MAP_READ, &rtp); /* get needed data from GstRtpRtxSend */ - ssrc = gst_rtp_buffer_get_ssrc (&rtp); - data = gst_rtp_rtx_send_get_ssrc_data (rtx, ssrc); + orig_ssrc = gst_rtp_buffer_get_ssrc (&rtp); + data = gst_rtp_rtx_send_get_ssrc_data (rtx, orig_ssrc); ssrc = data->rtx_ssrc; seqnum = data->next_seqnum++; fmtp = GPOINTER_TO_UINT (g_hash_table_lookup (rtx->rtx_pt_map, GUINT_TO_POINTER (gst_rtp_buffer_get_payload_type (&rtp)))); + + orig_seqnum = gst_rtp_buffer_get_seq (&rtp); GST_DEBUG_OBJECT (rtx, "creating rtx buffer, orig seqnum: %u, " - "rtx seqnum: %u, rtx ssrc: %X", gst_rtp_buffer_get_seq (&rtp), + "rtx seqnum: %u, rtx ssrc: %X", orig_seqnum, seqnum, ssrc); + /*Add repair packet meta so that TWCC will be able to to tie it + with a lost data packet */ + gst_buffer_add_rtp_repair_meta (new_buffer, + 0, 1, orig_ssrc, &orig_seqnum, 1); + /* gst_rtp_buffer_map does not map the payload so do it now */ gst_rtp_buffer_get_payload (&rtp); diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/meson.build b/subprojects/gst-plugins-good/gst/rtpmanager/meson.build index 0a0e6a14a9..416c91d45e 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/meson.build +++ b/subprojects/gst-plugins-good/gst/rtpmanager/meson.build @@ -23,6 +23,7 @@ rtpmanager_sources = [ 'rtpstats.c', 'rtptimerqueue.c', 'rtptwcc.c', + 'rtptwccstats.c', 'gstrtpsession.c', 'gstrtpfunnel.c', 'gstrtpst2022-1-fecdec.c', diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c index c810a78c6d..034e534bef 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c @@ -120,7 +120,6 @@ enum PROP_TWCC_FEEDBACK_INTERVAL, PROP_UPDATE_NTP64_HEADER_EXT, PROP_TIMEOUT_INACTIVE_SOURCES, - PROP_RTX_SSRC_MAP, PROP_LAST, }; @@ -804,18 +803,6 @@ rtp_session_class_init (RTPSessionClass * klass) DEFAULT_TIMEOUT_INACTIVE_SOURCES, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); - /** - * RTPSession:rtx-ssrc-map: - * - * Mapping from SSRC to RTX ssrcs. - * - * Since: 1.24 - */ - properties[PROP_RTX_SSRC_MAP] = - g_param_spec_boxed ("rtx-ssrc-map", "RTX SSRC Map", - "Map of SSRCs to their retransmission SSRCs", - GST_TYPE_STRUCTURE, G_PARAM_WRITABLE | G_PARAM_STATIC_STRINGS); - g_object_class_install_properties (gobject_class, PROP_LAST, properties); klass->get_twcc_windowed_stats = @@ -915,7 +902,6 @@ rtp_session_init (RTPSession * sess) sess->is_doing_ptp = TRUE; sess->twcc = rtp_twcc_manager_new (sess->mtu); - sess->rtx_ssrc_to_ssrc = g_hash_table_new (NULL, NULL); sess->timedout_ssrcs = g_hash_table_new (NULL, NULL); } @@ -939,9 +925,6 @@ rtp_session_finalize (GObject * object) g_hash_table_destroy (sess->ssrcs[i]); g_object_unref (sess->twcc); - if (sess->rtx_ssrc_map) - gst_structure_free (sess->rtx_ssrc_map); - g_hash_table_destroy (sess->rtx_ssrc_to_ssrc); g_hash_table_destroy (sess->timedout_ssrcs); g_mutex_clear (&sess->lock); @@ -1145,16 +1128,6 @@ rtp_session_set_property (GObject * object, guint prop_id, case PROP_TIMEOUT_INACTIVE_SOURCES: sess->timeout_inactive_sources = g_value_get_boolean (value); break; - case PROP_RTX_SSRC_MAP: - RTP_SESSION_LOCK (sess); - if (sess->rtx_ssrc_map) - gst_structure_free (sess->rtx_ssrc_map); - sess->rtx_ssrc_map = g_value_dup_boxed (value); - g_hash_table_remove_all (sess->rtx_ssrc_to_ssrc); - gst_structure_foreach (sess->rtx_ssrc_map, - structure_to_hash_table_reverse, sess->rtx_ssrc_to_ssrc); - RTP_SESSION_UNLOCK (sess); - break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); @@ -2418,14 +2391,6 @@ update_packet (GstBuffer ** buffer, guint idx, RTPPacketInfo * pinfo) /* RTP header extensions */ pinfo->header_ext = gst_rtp_buffer_get_extension_bytes (&rtp, &pinfo->header_ext_bit_pattern); - - /* if RTX, store the original seqnum (OSN) and SSRC */ - if (GST_BUFFER_FLAG_IS_SET (*buffer, GST_RTP_BUFFER_FLAG_RETRANSMISSION)) { - guint8 *payload = gst_rtp_buffer_get_payload (&rtp); - if (payload) { - pinfo->rtx_osn = GST_READ_UINT16_BE (payload); - } - } } if (pinfo->ntp64_ext_id != 0 && pinfo->send && !pinfo->have_ntp64_ext) { @@ -2496,8 +2461,6 @@ update_packet_info (RTPSession * sess, RTPPacketInfo * pinfo, pinfo->marker = FALSE; pinfo->ntp64_ext_id = send ? sess->send_ntp64_ext_id : 0; pinfo->have_ntp64_ext = FALSE; - pinfo->rtx_osn = -1; - pinfo->rtx_ssrc = 0; if (is_list) { GstBufferList *list = GST_BUFFER_LIST_CAST (data); @@ -2511,11 +2474,6 @@ update_packet_info (RTPSession * sess, RTPPacketInfo * pinfo, pinfo->arrival_time = GST_BUFFER_DTS (buffer); } - if (pinfo->rtx_osn != -1) - pinfo->rtx_ssrc = - GPOINTER_TO_UINT (g_hash_table_lookup (sess->rtx_ssrc_to_ssrc, - GUINT_TO_POINTER (pinfo->ssrc))); - return res; } diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.h b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.h index b45c9c20ff..84512a1e40 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.h +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.h @@ -330,7 +330,6 @@ struct _RTPSession { /* Transport-wide cc-extension */ RTPTWCCManager *twcc; GstStructure *rtx_ssrc_map; - GHashTable *rtx_ssrc_to_ssrc; }; /** diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpstats.h b/subprojects/gst-plugins-good/gst/rtpmanager/rtpstats.h index d9e955858e..07bb11d21b 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpstats.h +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpstats.h @@ -114,8 +114,6 @@ typedef struct { guint16 header_ext_bit_pattern; guint8 ntp64_ext_id; gboolean have_ntp64_ext; - gint32 rtx_osn; - guint32 rtx_ssrc; } RTPPacketInfo; /** diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtptwcc.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtptwcc.c index 1719cf8b99..baf3886624 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtptwcc.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtptwcc.c @@ -20,10 +20,13 @@ #define GLIB_DISABLE_DEPRECATION_WARNINGS #include "rtptwcc.h" +#include "rtptwccstats.h" #include +#include #include #include #include +#include #include "gstrtputils.h" @@ -81,11 +84,29 @@ typedef struct GstClockTime socket_ts; GstClockTime remote_ts; guint16 seqnum; + guint16 orig_seqnum; + guint32 ssrc; guint8 pt; guint size; gboolean lost; - gint32 rtx_osn; - guint32 rtx_ssrc; + gint redundant_idx; /* if it's redudndant packet -- series number in a block, + -1 otherwise */ + gint redundant_num; /* if it'r a redundant packet -- how many packets are + in the block, -1 otherwise */ + guint32 protects_ssrc; /* for redundant packets: SSRC of the data stream */ + + /* For redundant packets: seqnums of the packets being protected + * by this packet. + * IMPORTANT: Once the packet is checked in before transmission, this array + * contains rtp seqnums. After receiving a feedback on the packet, the array + * is converted to TWCC seqnums. This is done to shift some work to the + * get_windowed_stats function, which should be less time-critical. + */ + GArray * protects_seqnums; + gboolean stats_processed; + + TWCCPktState state; + gint update_stats; } SentPacket; typedef struct @@ -95,508 +116,13 @@ typedef struct GstClockTime remote_ts; } ParsedPacket; -typedef struct -{ - GstClockTime org_ts; - GstClockTime local_ts; - GstClockTime remote_ts; - guint16 seqnum; - guint size; - guint8 pt; - - GstClockTimeDiff local_delta; - GstClockTimeDiff remote_delta; - GstClockTimeDiff delta_delta; - gboolean recovered; -} StatsPacket; - -static void -stats_packet_init_from_sent_packet (StatsPacket * pkt, SentPacket * sent) -{ - pkt->org_ts = sent->local_ts; - pkt->local_ts = sent->local_ts; - pkt->remote_ts = sent->remote_ts; - pkt->seqnum = sent->seqnum; - pkt->size = sent->size; - pkt->pt = sent->pt; - pkt->recovered = FALSE; - pkt->local_delta = GST_CLOCK_STIME_NONE; - pkt->remote_delta = GST_CLOCK_STIME_NONE; - pkt->delta_delta = GST_CLOCK_STIME_NONE; - - /* if we have a socket-timestamp, use that instead */ - if (GST_CLOCK_TIME_IS_VALID (sent->socket_ts)) { - pkt->local_ts = sent->socket_ts; - } -} - -typedef struct -{ - GArray *packets; - gint64 new_packets_idx; - - /* windowed stats */ - guint packets_sent; - guint packets_recv; - guint bitrate_sent; - guint bitrate_recv; - gdouble packet_loss_pct; - gdouble recovery_pct; - gint64 avg_delta_of_delta; - gdouble delta_of_delta_growth; -} TWCCStatsCtx; - -static void -_append_structure_to_value_array (GValueArray * array, GstStructure * s) -{ - GValue *val; - g_value_array_append (array, NULL); - val = g_value_array_get_nth (array, array->n_values - 1); - g_value_init (val, GST_TYPE_STRUCTURE); - g_value_take_boxed (val, s); -} - -static void -_structure_take_value_array (GstStructure * s, - const gchar * field_name, GValueArray * array) -{ - GValue value = G_VALUE_INIT; - g_value_init (&value, G_TYPE_VALUE_ARRAY); - g_value_take_boxed (&value, array); - gst_structure_take_value (s, field_name, &value); - g_value_unset (&value); -} - -static TWCCStatsCtx * -twcc_stats_ctx_new (void) -{ - TWCCStatsCtx *ctx = g_new0 (TWCCStatsCtx, 1); - - ctx->packets = g_array_new (FALSE, FALSE, sizeof (StatsPacket)); - - return ctx; -} - -static void -twcc_stats_ctx_free (TWCCStatsCtx * ctx) -{ - g_array_unref (ctx->packets); - g_free (ctx); -} - -static GstClockTime -twcc_stats_ctx_get_last_local_ts (TWCCStatsCtx * ctx) -{ - GstClockTime ret = GST_CLOCK_TIME_NONE; - StatsPacket *pkt = NULL; - if (ctx->packets->len > 0) - pkt = &g_array_index (ctx->packets, StatsPacket, ctx->packets->len - 1); - if (pkt) - ret = pkt->local_ts; - return ret; -} - -static gboolean -_get_stats_packets_window (GArray * array, - GstClockTimeDiff start_time, GstClockTimeDiff end_time, - guint * start_idx, guint * num_packets) -{ - gboolean ret = FALSE; - guint end_idx = 0; - guint i; - - if (array->len < 2) { - GST_DEBUG ("Not enough starts to do a window"); - return FALSE; - } - - for (i = 0; i < array->len; i++) { - StatsPacket *pkt = &g_array_index (array, StatsPacket, i); - if (GST_CLOCK_TIME_IS_VALID (pkt->local_ts)) { - GstClockTimeDiff offset = GST_CLOCK_DIFF (pkt->local_ts, start_time); - *start_idx = i; - /* positive number here means it is older than our start time */ - if (offset > 0) { - GST_LOG ("Packet #%u is too old: %" - GST_TIME_FORMAT, pkt->seqnum, GST_TIME_ARGS (pkt->local_ts)); - } else { - GST_LOG ("Setting first packet in our window to #%u: %" - GST_TIME_FORMAT, pkt->seqnum, GST_TIME_ARGS (pkt->local_ts)); - ret = TRUE; - break; - } - } - } - - /* jump out early if we could not find a start_idx */ - if (!ret) { - return FALSE; - } - - ret = FALSE; - for (i = 0; i < array->len - *start_idx - 1; i++) { - guint idx = array->len - 1 - i; - StatsPacket *pkt = &g_array_index (array, StatsPacket, idx); - if (GST_CLOCK_TIME_IS_VALID (pkt->local_ts)) { - GstClockTimeDiff offset = GST_CLOCK_DIFF (pkt->local_ts, end_time); - if (offset >= 0) { - GST_LOG ("Setting last packet in our window to #%u: %" - GST_TIME_FORMAT, pkt->seqnum, GST_TIME_ARGS (pkt->local_ts)); - end_idx = idx; - ret = TRUE; - break; - } else { - GST_LOG ("Packet #%u is too new: %" - GST_TIME_FORMAT, pkt->seqnum, GST_TIME_ARGS (pkt->local_ts)); - } - } - } - - /* jump out early if we could not find a window */ - if (!ret) { - return FALSE; - } - - *num_packets = end_idx - *start_idx + 1; - - return ret; -} - -static gboolean -twcc_stats_ctx_calculate_windowed_stats (TWCCStatsCtx * ctx, - GstClockTimeDiff start_time, GstClockTimeDiff end_time) -{ - GArray *packets = ctx->packets; - guint start_idx; - guint packets_sent = 0; - guint packets_recv = 0; - guint packets_recovered = 0; - guint packets_lost = 0; - - /* FIXME: property ? */ - guint max_stats_packets = 1000; - - guint i; - guint bits_sent = 0; - guint bits_recv = 0; - - GstClockTimeDiff delta_delta_sum = 0; - guint delta_delta_count = 0; - GstClockTimeDiff first_delta_delta_sum = 0; - guint first_delta_delta_count = 0; - GstClockTimeDiff last_delta_delta_sum = 0; - guint last_delta_delta_count = 0; - - StatsPacket *first_local_pkt = NULL; - StatsPacket *last_local_pkt = NULL; - StatsPacket *first_remote_pkt = NULL; - StatsPacket *last_remote_pkt = NULL; - - GstClockTimeDiff local_duration = 0; - GstClockTimeDiff remote_duration = 0; - - ctx->packet_loss_pct = 0.0; - ctx->avg_delta_of_delta = 0; - ctx->delta_of_delta_growth = 0.0; - ctx->bitrate_sent = 0; - ctx->bitrate_recv = 0; - ctx->recovery_pct = -1.0; - - gboolean ret = - _get_stats_packets_window (packets, start_time, end_time, &start_idx, - &packets_sent); - if (!ret || packets_sent < 2) { - GST_INFO ("Not enough packets to fill our window yet!"); - return FALSE; - } - - for (i = 0; i < packets_sent; i++) { - StatsPacket *pkt = &g_array_index (packets, StatsPacket, i + start_idx); - GST_LOG ("STATS WINDOW: %u/%u: pkt #%u, pt: %u, size: %u, arrived: %s, " - "local-ts: %" GST_TIME_FORMAT ", remote-ts %" GST_TIME_FORMAT, - i + 1, packets_sent, pkt->seqnum, pkt->pt, pkt->size * 8, - GST_CLOCK_TIME_IS_VALID (pkt->remote_ts) ? "YES" : "NO", - GST_TIME_ARGS (pkt->local_ts), GST_TIME_ARGS (pkt->remote_ts)); - - if (GST_CLOCK_TIME_IS_VALID (pkt->local_ts)) { - /* don't count the bits for the first packet in the window */ - if (first_local_pkt == NULL) { - first_local_pkt = pkt; - } else { - bits_sent += pkt->size * 8; - } - last_local_pkt = pkt; - } - - if (GST_CLOCK_TIME_IS_VALID (pkt->remote_ts)) { - /* don't count the bits for the first packet in the window */ - if (first_remote_pkt == NULL) { - first_remote_pkt = pkt; - } else { - bits_recv += pkt->size * 8; - } - last_remote_pkt = pkt; - packets_recv++; - } else { - GST_LOG ("Packet #%u is lost, recovered=%u", pkt->seqnum, pkt->recovered); - packets_lost++; - if (pkt->recovered) { - packets_recovered++; - } - } - - if (GST_CLOCK_STIME_IS_VALID (pkt->delta_delta)) { - delta_delta_sum += pkt->delta_delta; - delta_delta_count++; - if (i < packets_sent / 2) { - first_delta_delta_sum += pkt->delta_delta; - first_delta_delta_count++; - } else { - last_delta_delta_sum += pkt->delta_delta; - last_delta_delta_count++; - } - } - } - - ctx->packets_sent = packets_sent; - ctx->packets_recv = packets_recv; - - if (first_local_pkt && last_local_pkt) { - local_duration = - GST_CLOCK_DIFF (first_local_pkt->local_ts, last_local_pkt->local_ts); - } - if (first_remote_pkt && last_remote_pkt) { - remote_duration = - GST_CLOCK_DIFF (first_remote_pkt->remote_ts, - last_remote_pkt->remote_ts); - } - - if (packets_sent) - ctx->packet_loss_pct = (packets_lost * 100) / (gfloat) packets_sent; - - if (packets_lost) { - ctx->recovery_pct = (packets_recovered * 100) / (gfloat) packets_lost; - } - - if (delta_delta_count) { - ctx->avg_delta_of_delta = delta_delta_sum / delta_delta_count; - } - - if (first_delta_delta_count && last_delta_delta_count) { - GstClockTimeDiff first_avg = - first_delta_delta_sum / first_delta_delta_count; - GstClockTimeDiff last_avg = last_delta_delta_sum / last_delta_delta_count; - - /* filter out very small numbers */ - first_avg = MAX (first_avg, 100 * GST_USECOND); - last_avg = MAX (last_avg, 100 * GST_USECOND); - ctx->delta_of_delta_growth = (double) last_avg / (double) first_avg; - } - - if (local_duration > 0) { - ctx->bitrate_sent = - gst_util_uint64_scale (bits_sent, GST_SECOND, local_duration); - } - if (remote_duration > 0) { - ctx->bitrate_recv = - gst_util_uint64_scale (bits_recv, GST_SECOND, remote_duration); - } - - GST_INFO ("Got stats: bits_sent: %u, bits_recv: %u, packets_sent = %u, " - "packets_recv: %u, packetlost_pct = %lf, recovery_pct = %lf, " - "local_duration=%" GST_TIME_FORMAT ", remote_duration=%" GST_TIME_FORMAT - ", " "sent_bitrate = %u, " "recv_bitrate = %u, delta-delta-avg = %" - GST_STIME_FORMAT ", delta-delta-growth=%lf", bits_sent, bits_recv, - packets_sent, packets_recv, ctx->packet_loss_pct, ctx->recovery_pct, - GST_TIME_ARGS (local_duration), GST_TIME_ARGS (remote_duration), - ctx->bitrate_sent, ctx->bitrate_recv, - GST_STIME_ARGS (ctx->avg_delta_of_delta), ctx->delta_of_delta_growth); - - /* trim the stats array down to max packets */ - if (packets->len > max_stats_packets) { - g_array_remove_range (packets, 0, packets->len - max_stats_packets); - } - - return TRUE; -} - -static GstStructure * -twcc_stats_ctx_get_structure (TWCCStatsCtx * ctx) -{ - return gst_structure_new ("RTPTWCCStats", - "packets-sent", G_TYPE_UINT, ctx->packets_sent, - "packets-recv", G_TYPE_UINT, ctx->packets_recv, - "bitrate-sent", G_TYPE_UINT, ctx->bitrate_sent, - "bitrate-recv", G_TYPE_UINT, ctx->bitrate_recv, - "packet-loss-pct", G_TYPE_DOUBLE, ctx->packet_loss_pct, - "recovery-pct", G_TYPE_DOUBLE, ctx->recovery_pct, - "avg-delta-of-delta", G_TYPE_INT64, ctx->avg_delta_of_delta, - "delta-of-delta-growth", G_TYPE_DOUBLE, ctx->delta_of_delta_growth, NULL); -} - - -static void -_update_stats_for_packet_idx (GArray * array, gint packet_idx) -{ - StatsPacket *pkt; - StatsPacket *prev; - - /* we need at least 2 packets to do any stats */ - if (array->len < 2) { - GST_DEBUG ("Not yet 2 packets in stats"); - return; - } - - /* we can't do stats on the first packet */ - if (packet_idx == 0) { - GST_DEBUG ("First packet can't have stats calculated"); - return; - } - - /* packet_idx must be inside the array */ - if (array->len <= packet_idx) { - GST_DEBUG ("Packet idx %u is outside of array!", packet_idx); - return; - } - - pkt = &g_array_index (array, StatsPacket, packet_idx); - prev = &g_array_index (array, StatsPacket, packet_idx - 1); - - if (GST_CLOCK_TIME_IS_VALID (pkt->local_ts) && - GST_CLOCK_TIME_IS_VALID (prev->local_ts)) { - pkt->local_delta = GST_CLOCK_DIFF (prev->local_ts, pkt->local_ts); - GST_LOG ("Calculated local_delta for packet #%u to %" GST_STIME_FORMAT, - pkt->seqnum, GST_STIME_ARGS (pkt->local_delta)); - } - - if (GST_CLOCK_TIME_IS_VALID (pkt->remote_ts) && - GST_CLOCK_TIME_IS_VALID (prev->remote_ts)) { - pkt->remote_delta = GST_CLOCK_DIFF (prev->remote_ts, pkt->remote_ts); - GST_LOG ("Calculated remote_delta for packet #%u to %" GST_STIME_FORMAT, - pkt->seqnum, GST_STIME_ARGS (pkt->remote_delta)); - } - - if (GST_CLOCK_STIME_IS_VALID (pkt->local_delta) && - GST_CLOCK_STIME_IS_VALID (pkt->remote_delta)) { - pkt->delta_delta = pkt->remote_delta - pkt->local_delta; - GST_LOG ("Calculated delta-of-delta for packet #%u to %" GST_STIME_FORMAT, - pkt->seqnum, GST_STIME_ARGS (pkt->delta_delta)); - } -} - -static gint -_get_packet_idx_for_seqnum (GArray * array, guint16 seqnum) -{ - guint i; - - for (i = 0; i < array->len; i++) { - StatsPacket *pkt = &g_array_index (array, StatsPacket, i); - if (pkt->seqnum == seqnum) { - return (gint) i; - } - } - - return -1; -} - -/* assumes all seqnum are in order */ -static gint -_get_packet_idx_for_seqnum_fast (GArray * array, guint16 seqnum) -{ - StatsPacket *first; - guint16 idx; - - if (array->len == 0) - return -1; - - first = &g_array_index (array, StatsPacket, 0); - idx = gst_rtp_buffer_compare_seqnum (first->seqnum, seqnum); - - if (idx < array->len) { - StatsPacket *found = &g_array_index (array, StatsPacket, idx); - if (found->seqnum == seqnum) { - return (gint) idx; - } - } - - return -1; -} - -static gint -twcc_stats_ctx_get_packet_idx_for_seqnum (TWCCStatsCtx * ctx, guint16 seqnum) -{ - gint ret; - - /* assumes all packets seqnum are in order */ - ret = _get_packet_idx_for_seqnum_fast (ctx->packets, seqnum); - if (ret != -1) - return ret; - - return _get_packet_idx_for_seqnum (ctx->packets, seqnum); -} - -static StatsPacket * -twcc_stats_ctx_get_packet_for_seqnum (TWCCStatsCtx * ctx, guint16 seqnum) -{ - StatsPacket *ret = NULL; - gint idx = twcc_stats_ctx_get_packet_idx_for_seqnum (ctx, seqnum); - if (idx != -1) - ret = &g_array_index (ctx->packets, StatsPacket, idx); - - return ret; -} - -static void -twcc_stats_ctx_add_packet (TWCCStatsCtx * ctx, StatsPacket * pkt) -{ - StatsPacket *last = NULL; - gint pkt_idx; - - if (ctx->packets->len > 0) - last = &g_array_index (ctx->packets, StatsPacket, ctx->packets->len - 1); - - /* first a quick check to see if we are going forward in seqnum, - in which case we simply append */ - if (last == NULL || pkt->seqnum > last->seqnum) { - GST_LOG ("Appending #%u to stats packets", pkt->seqnum); - g_array_append_val (ctx->packets, *pkt); - /* and update the stats for this packet */ - _update_stats_for_packet_idx (ctx->packets, ctx->packets->len - 1); - return; - } - - /* here we are dealing with a reordered packet, we start by searching for it */ - pkt_idx = twcc_stats_ctx_get_packet_idx_for_seqnum (ctx, pkt->seqnum); - if (pkt_idx != -1) { - StatsPacket *existing = &g_array_index (ctx->packets, StatsPacket, pkt_idx); - /* only update if we don't already have information about this packet, and - this packet brings something new */ - if (!GST_CLOCK_TIME_IS_VALID (existing->remote_ts) && - GST_CLOCK_TIME_IS_VALID (pkt->remote_ts)) { - GST_DEBUG ("Updating stats packet #%u", pkt->seqnum); - g_array_index (ctx->packets, StatsPacket, pkt_idx) = *pkt; - - /* now update stats for this packet and the next one along */ - _update_stats_for_packet_idx (ctx->packets, pkt_idx); - _update_stats_for_packet_idx (ctx->packets, pkt_idx + 1); - } - return; - } -} - -/******************************************************/ - struct _RTPTWCCManager { GObject object; GMutex recv_lock; - GHashTable *ssrc_to_seqmap; GHashTable *pt_to_twcc_ext_id; - TWCCStatsCtx *stats_ctx; - GHashTable *stats_ctx_by_pt; - guint8 send_ext_id; guint8 recv_ext_id; guint16 send_seqnum; @@ -607,7 +133,14 @@ struct _RTPTWCCManager guint64 fb_pkt_count; - GArray *sent_packets; + /* Array of SentPackets struct */ + GstQueueArray *sent_packets; + /* Array of GArrays with pointers to SentPackets structs from sent_packets + to which twcc feedbacks were received and are waiting to be processed by + statistics thread. + */ + GstQueueArray *sent_packets_feedbacks; + GArray *parsed_packets; GQueue *rtcp_buffers; @@ -624,16 +157,22 @@ struct _RTPTWCCManager GstClockTime next_feedback_send_time; GstClockTime feedback_interval; - GstClockTimeDiff avg_rtt; GstClockTime last_report_time; RTPTWCCManagerCaps caps_cb; gpointer caps_ud; -}; + TWCCStatsManager *stats_manager; +}; -static void rtp_twcc_manager_tx_feedback (GstTxFeedback * parent, - guint64 buffer_id, GstClockTime ts); +static void +rtp_twcc_manager_tx_feedback (GstTxFeedback * parent, guint64 buffer_id, + GstClockTime ts) +{ + RTPTWCCManager *twcc = RTP_TWCC_MANAGER_CAST (parent); + const guint16 seqnum = (guint16) buffer_id; + rtp_twcc_stats_set_sock_ts (twcc->stats_manager, seqnum, ts); +} static void _tx_feedback_init (gpointer g_iface, G_GNUC_UNUSED gpointer iface_data) @@ -648,21 +187,15 @@ G_DEFINE_TYPE_WITH_CODE (RTPTWCCManager, rtp_twcc_manager, G_TYPE_OBJECT, static void rtp_twcc_manager_init (RTPTWCCManager * twcc) { - twcc->ssrc_to_seqmap = g_hash_table_new_full (NULL, NULL, NULL, - (GDestroyNotify) g_hash_table_destroy); twcc->pt_to_twcc_ext_id = g_hash_table_new (NULL, NULL); twcc->recv_packets = g_array_new (FALSE, FALSE, sizeof (RecvPacket)); - twcc->sent_packets = g_array_new (FALSE, FALSE, sizeof (SentPacket)); + twcc->parsed_packets = g_array_new (FALSE, FALSE, sizeof (ParsedPacket)); g_mutex_init (&twcc->recv_lock); twcc->rtcp_buffers = g_queue_new (); - twcc->stats_ctx = twcc_stats_ctx_new (); - twcc->stats_ctx_by_pt = g_hash_table_new_full (NULL, NULL, - NULL, (GDestroyNotify) twcc_stats_ctx_free); - twcc->recv_media_ssrc = -1; twcc->recv_sender_ssrc = -1; @@ -671,6 +204,8 @@ rtp_twcc_manager_init (RTPTWCCManager * twcc) twcc->feedback_interval = GST_CLOCK_TIME_NONE; twcc->next_feedback_send_time = GST_CLOCK_TIME_NONE; twcc->last_report_time = GST_CLOCK_TIME_NONE; + + twcc->stats_manager = rtp_twcc_stats_manager_new (G_OBJECT(twcc)); } static void @@ -678,17 +213,13 @@ rtp_twcc_manager_finalize (GObject * object) { RTPTWCCManager *twcc = RTP_TWCC_MANAGER_CAST (object); - g_hash_table_destroy (twcc->ssrc_to_seqmap); g_hash_table_destroy (twcc->pt_to_twcc_ext_id); g_array_unref (twcc->recv_packets); - g_array_unref (twcc->sent_packets); g_array_unref (twcc->parsed_packets); g_queue_free_full (twcc->rtcp_buffers, (GDestroyNotify) gst_buffer_unref); g_mutex_clear (&twcc->recv_lock); - - g_hash_table_destroy (twcc->stats_ctx_by_pt); - twcc_stats_ctx_free (twcc->stats_ctx); + rtp_twcc_stats_manager_free (twcc->stats_manager); G_OBJECT_CLASS (rtp_twcc_manager_parent_class)->finalize (object); } @@ -712,88 +243,6 @@ rtp_twcc_manager_new (guint mtu) return twcc; } -static TWCCStatsCtx * -_get_ctx_for_pt (RTPTWCCManager * twcc, guint pt) -{ - TWCCStatsCtx *ctx = - g_hash_table_lookup (twcc->stats_ctx_by_pt, GUINT_TO_POINTER (pt)); - if (!ctx) { - ctx = twcc_stats_ctx_new (); - g_hash_table_insert (twcc->stats_ctx_by_pt, GUINT_TO_POINTER (pt), ctx); - } - return ctx; -} - - -static void -_update_stats_with_recovered (RTPTWCCManager * twcc, guint16 seqnum) -{ - TWCCStatsCtx *ctx; - StatsPacket *pkt = - twcc_stats_ctx_get_packet_for_seqnum (twcc->stats_ctx, seqnum); - - if (pkt == NULL) { - GST_INFO ("Could not find seqnum %u", seqnum); - return; - } - - pkt->recovered = TRUE; - - /* now find the equivalent packet in the payload */ - ctx = _get_ctx_for_pt (twcc, pkt->pt); - pkt = twcc_stats_ctx_get_packet_for_seqnum (ctx, seqnum); - - if (pkt) { - pkt->recovered = TRUE; - } -} - -static gint32 -_lookup_seqnum (RTPTWCCManager * twcc, guint32 ssrc, guint16 seqnum) -{ - gint32 ret = -1; - - GHashTable *seq_to_twcc = - g_hash_table_lookup (twcc->ssrc_to_seqmap, GUINT_TO_POINTER (ssrc)); - if (seq_to_twcc) { - ret = - GPOINTER_TO_UINT (g_hash_table_lookup (seq_to_twcc, - GUINT_TO_POINTER (seqnum))); - } - return ret; -} - -static void -_add_packet_to_stats (RTPTWCCManager * twcc, - ParsedPacket * parsed_pkt, SentPacket * sent_pkt) -{ - TWCCStatsCtx *ctx; - StatsPacket pkt; - - stats_packet_init_from_sent_packet (&pkt, sent_pkt); - - /* add packet to the stats context */ - twcc_stats_ctx_add_packet (twcc->stats_ctx, &pkt); - - /* add packet to the payload specific stats context */ - ctx = _get_ctx_for_pt (twcc, pkt.pt); - twcc_stats_ctx_add_packet (ctx, &pkt); - - /* extra check to see if we can confirm the arrival of a recovery packet, - and hence set the "original" packet as recovered */ - if (parsed_pkt->status != RTP_TWCC_PACKET_STATUS_NOT_RECV - && sent_pkt->rtx_osn != -1) { - gint32 recovered_seq = - _lookup_seqnum (twcc, sent_pkt->rtx_ssrc, sent_pkt->rtx_osn); - if (recovered_seq != -1) { - GST_LOG ("RTX Packet %u protects seqnum %d", sent_pkt->seqnum, - recovered_seq); - _update_stats_with_recovered (twcc, recovered_seq); - } - } -} - - static void recv_packet_init (RecvPacket * packet, guint16 seqnum, RTPPacketInfo * pinfo) { @@ -871,50 +320,14 @@ _get_twcc_seqnum_data (RTPPacketInfo * pinfo, guint8 ext_id, gpointer * data) } static void -sent_packet_init (SentPacket * packet, guint16 seqnum, RTPPacketInfo * pinfo, - GstRTPBuffer * rtp) -{ - packet->seqnum = seqnum; - packet->local_ts = pinfo->current_time; - packet->size = pinfo->bytes + 12; /* the reported wireshark size */ - packet->pt = gst_rtp_buffer_get_payload_type (rtp); - packet->remote_ts = GST_CLOCK_TIME_NONE; - packet->socket_ts = GST_CLOCK_TIME_NONE; - packet->lost = FALSE; - packet->rtx_osn = pinfo->rtx_osn; - packet->rtx_ssrc = pinfo->rtx_ssrc; -} - -static void -rtp_twcc_manager_register_seqnum (RTPTWCCManager * twcc, - guint32 ssrc, guint16 seqnum, guint16 twcc_seqnum) -{ - GHashTable *seq_to_twcc = - g_hash_table_lookup (twcc->ssrc_to_seqmap, GUINT_TO_POINTER (ssrc)); - if (!seq_to_twcc) { - seq_to_twcc = g_hash_table_new (NULL, NULL); - g_hash_table_insert (twcc->ssrc_to_seqmap, GUINT_TO_POINTER (ssrc), - seq_to_twcc); - } - g_hash_table_insert (seq_to_twcc, GUINT_TO_POINTER (seqnum), - GUINT_TO_POINTER (twcc_seqnum)); - GST_LOG ("Registering OSN: %u to twcc-twcc_seqnum: %u with ssrc: %u", seqnum, - twcc_seqnum, ssrc); -} - -/* Remove old sent packets and keep them under a maximum threshold, as we - can't accumulate them if we don't get a feedback message from the - receiver. */ -static void -_prune_old_sent_packets (RTPTWCCManager * twcc) +_structure_take_value_array (GstStructure * s, + const gchar * field_name, GValueArray * array) { - guint length; - - if (twcc->sent_packets->len <= MAX_PACKETS_PER_FEEDBACK) - return; - - length = twcc->sent_packets->len - MAX_PACKETS_PER_FEEDBACK; - g_array_remove_range (twcc->sent_packets, 0, length); + GValue value = G_VALUE_INIT; + g_value_init (&value, G_TYPE_VALUE_ARRAY); + g_value_take_boxed (&value, array); + gst_structure_take_value (s, field_name, &value); + g_value_unset (&value); } /* @@ -945,14 +358,25 @@ _get_twcc_buffer_ext_data (GstRTPBuffer * rtpbuf, guint8 ext_id) return data; } +/** + * Set TWCC seqnum and remember the packet for statistics. + * + * Fill in SentPacket structure with the seqnum and the packet info, + * and add it to the sent_packets queue, which is used by the statistics thread + * + * NB: protect_seqnum is a reference of the GstBuffer's meta! + * + * NB: at the moment protect_seqnum contains internal seqnum, + they will be replaced with twcc seqnums in-place in statistics thread! + */ static void _set_twcc_seqnum_data (RTPTWCCManager * twcc, RTPPacketInfo * pinfo, GstBuffer * buf, guint8 ext_id) { - SentPacket packet; + guint16 seqnum; + GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; gpointer data; - guint16 seqnum; if (!gst_rtp_buffer_map (buf, GST_MAP_READWRITE, &rtp)) { GST_WARNING ("Couldn't map the buffer %" GST_PTR_FORMAT, buf); @@ -967,19 +391,11 @@ _set_twcc_seqnum_data (RTPTWCCManager * twcc, RTPPacketInfo * pinfo, seqnum = twcc->send_seqnum++; GST_WRITE_UINT16_BE (data, seqnum); - sent_packet_init (&packet, seqnum, pinfo, &rtp); - gst_rtp_buffer_unmap (&rtp); - g_array_append_val (twcc->sent_packets, packet); - _prune_old_sent_packets (twcc); - rtp_twcc_manager_register_seqnum (twcc, pinfo->ssrc, pinfo->seqnum, seqnum); + rtp_twcc_stats_sent_pkt (twcc->stats_manager, pinfo, &rtp, seqnum); + gst_rtp_buffer_unmap (&rtp); gst_buffer_add_tx_feedback_meta (pinfo->data, seqnum, GST_TX_FEEDBACK_CAST (twcc)); - - GST_LOG - ("Send: twcc-seqnum: %u, seqnum: %u, pt: %u, marker: %d, size: %u, ts: %" - GST_TIME_FORMAT, packet.seqnum, pinfo->seqnum, packet.pt, pinfo->marker, - packet.size, GST_TIME_ARGS (pinfo->current_time)); } static void @@ -1684,40 +1100,6 @@ rtp_twcc_manager_send_packet (RTPTWCCManager * twcc, RTPPacketInfo * pinfo) rtp_twcc_manager_set_send_twcc_seqnum (twcc, pinfo); } -static void -rtp_twcc_manager_tx_feedback (GstTxFeedback * parent, guint64 buffer_id, - GstClockTime ts) -{ - RTPTWCCManager *twcc = RTP_TWCC_MANAGER_CAST (parent); - guint16 seqnum = (guint16) buffer_id; - SentPacket *first = NULL; - SentPacket *pkt = NULL; - gint idx; - - first = &g_array_index (twcc->sent_packets, SentPacket, 0); - if (first == NULL) { - GST_WARNING ("Received a tx-feedback without having sent any packets?!?"); - return; - } - - idx = gst_rtp_buffer_compare_seqnum (first->seqnum, seqnum); - if (idx < 0) { - pkt = - &g_array_index (twcc->sent_packets, SentPacket, - twcc->sent_packets->len - 1); - } else if (idx < twcc->sent_packets->len) { - pkt = &g_array_index (twcc->sent_packets, SentPacket, idx); - } - - if (pkt && pkt->seqnum == seqnum) { - pkt->socket_ts = ts; - GST_LOG ("packet #%u, setting socket-ts %" GST_TIME_FORMAT, - seqnum, GST_TIME_ARGS (ts)); - } else { - GST_WARNING ("Unable to update send-time for twcc-seqnum #%u", seqnum); - } -} - static void _add_parsed_packet (GArray * parsed_packets, guint16 seqnum, guint status) { @@ -1775,16 +1157,6 @@ _parse_status_vector_chunk (GstBitReader * reader, GArray * parsed_packets, return num_bits; } -/* keep the last 1000 packets... FIXME: do something smarter */ -static void -_prune_sent_packets (RTPTWCCManager * twcc) -{ - if (twcc->sent_packets->len > 1000) { - g_array_remove_range (twcc->sent_packets, 0, - twcc->sent_packets->len - 1000); - } -} - static void _check_for_lost_packets (RTPTWCCManager * twcc, GArray * parsed_packets, guint16 base_seqnum, guint16 packet_count, guint8 fb_pkt_count) @@ -1836,10 +1208,25 @@ _check_for_lost_packets (RTPTWCCManager * twcc, GArray * parsed_packets, return; } +static void +_append_structure_to_value_array (GValueArray * array, GstStructure * s) +{ + GValue *val; + g_value_array_append (array, NULL); + val = g_value_array_get_nth (array, array->n_values - 1); + g_value_init (val, GST_TYPE_STRUCTURE); + g_value_take_boxed (val, s); +} + static void _add_parsed_packet_to_value_array (GValueArray * array, ParsedPacket * pkt) { - _append_structure_to_value_array (array, + GValue *val; + + g_value_array_append (array, NULL); + val = g_value_array_get_nth (array, array->n_values - 1); + g_value_init (val, GST_TYPE_STRUCTURE); + g_value_take_boxed (val, gst_structure_new ("RTPTWCCPacket", "seqnum", G_TYPE_UINT, pkt->seqnum, "remote-ts", G_TYPE_UINT64, pkt->remote_ts, @@ -1862,8 +1249,6 @@ rtp_twcc_manager_parse_fci (RTPTWCCManager * twcc, guint packets_parsed = 0; guint fci_parsed; guint i; - SentPacket *first_sent_pkt = NULL; - GstClockTimeDiff rtt = GST_CLOCK_STIME_NONE; if (fci_length < 10) { GST_WARNING ("Malformed TWCC RTCP feedback packet"); @@ -1908,8 +1293,7 @@ rtp_twcc_manager_parse_fci (RTPTWCCManager * twcc, fci_parsed += 2; } - if (twcc->sent_packets->len > 0) - first_sent_pkt = &g_array_index (twcc->sent_packets, SentPacket, 0); + rtp_twcc_manager_tx_start_feedback (twcc->stats_manager); ts_rounded = base_time; for (i = 0; i < twcc->parsed_packets->len; i++) { @@ -1936,43 +1320,24 @@ rtp_twcc_manager_parse_fci (RTPTWCCManager * twcc, ts_rounded += delta_ts; pkt->remote_ts = ts_rounded; - GST_LOG ("pkt: #%u, remote_ts: %" GST_TIME_FORMAT + GST_DEBUG_OBJECT ( twcc, "pkt: #%u, remote_ts: %" GST_TIME_FORMAT " delta_ts: %" GST_STIME_FORMAT " status: %u", pkt->seqnum, GST_TIME_ARGS (pkt->remote_ts), GST_STIME_ARGS (delta_ts), pkt->status); _add_parsed_packet_to_value_array (array, pkt); + } else { + GST_DEBUG_OBJECT ( twcc, "pkt: #%u, remote_ts: 0 delta_ts: 0 status: %u", + pkt->seqnum, pkt->status); } - if (first_sent_pkt) { - SentPacket *found = NULL; - guint16 sent_idx = - gst_rtp_buffer_compare_seqnum (first_sent_pkt->seqnum, pkt->seqnum); - if (sent_idx < twcc->sent_packets->len) - found = &g_array_index (twcc->sent_packets, SentPacket, sent_idx); - if (found && found->seqnum == pkt->seqnum) { - found->remote_ts = pkt->remote_ts; - - GST_LOG ("matching pkt: #%u with local_ts: %" GST_TIME_FORMAT - " size: %u, remote-ts: %" GST_TIME_FORMAT, pkt->seqnum, - GST_TIME_ARGS (found->local_ts), - found->size * 8, GST_TIME_ARGS (pkt->remote_ts)); - - _add_packet_to_stats (twcc, pkt, found); - - /* calculate the round-trip time */ - rtt = GST_CLOCK_DIFF (found->local_ts, current_time); - - } - } + rtp_twcc_stats_pkt_feedback (twcc->stats_manager, pkt->seqnum, + pkt->remote_ts, current_time, + pkt->status == RTP_TWCC_PACKET_STATUS_NOT_RECV + ? RTP_TWCC_FECBLOCK_PKT_LOST : RTP_TWCC_FECBLOCK_PKT_RECEIVED); } - - if (GST_CLOCK_STIME_IS_VALID (rtt)) - twcc->avg_rtt = WEIGHT (rtt, twcc->avg_rtt, 0.1); + rtp_twcc_manager_tx_end_feedback (twcc->stats_manager); twcc->last_report_time = current_time; - - _prune_sent_packets (twcc); - _structure_take_value_array (ret, "packets", array); return ret; @@ -1982,48 +1347,8 @@ GstStructure * rtp_twcc_manager_get_windowed_stats (RTPTWCCManager * twcc, GstClockTime stats_window_size, GstClockTime stats_window_delay) { - GstStructure *ret; - GValueArray *array; - GHashTableIter iter; - gpointer key; - gpointer value; - GstClockTimeDiff start_time; - GstClockTimeDiff end_time; - - GstClockTime last_ts = twcc_stats_ctx_get_last_local_ts (twcc->stats_ctx); - if (!GST_CLOCK_TIME_IS_VALID (last_ts)) - return twcc_stats_ctx_get_structure (twcc->stats_ctx);; - - array = g_value_array_new (0); - end_time = GST_CLOCK_DIFF (stats_window_delay, last_ts); - start_time = end_time - stats_window_size; - - GST_DEBUG_OBJECT (twcc, - "Calculating windowed stats for the window %" GST_STIME_FORMAT - " starting from %" GST_STIME_FORMAT " to: %" GST_STIME_FORMAT, - GST_STIME_ARGS (stats_window_size), GST_STIME_ARGS (start_time), - GST_STIME_ARGS (end_time)); - - twcc_stats_ctx_calculate_windowed_stats (twcc->stats_ctx, start_time, - end_time); - ret = twcc_stats_ctx_get_structure (twcc->stats_ctx); - GST_LOG ("Full stats: %" GST_PTR_FORMAT, ret); - - g_hash_table_iter_init (&iter, twcc->stats_ctx_by_pt); - while (g_hash_table_iter_next (&iter, &key, &value)) { - GstStructure *s; - guint pt = GPOINTER_TO_UINT (key); - TWCCStatsCtx *ctx = value; - twcc_stats_ctx_calculate_windowed_stats (ctx, start_time, end_time); - s = twcc_stats_ctx_get_structure (ctx); - gst_structure_set (s, "pt", G_TYPE_UINT, pt, NULL); - _append_structure_to_value_array (array, s); - GST_LOG ("Stats for pt %u: %" GST_PTR_FORMAT, pt, s); - } - - _structure_take_value_array (ret, "payload-stats", array); - - return ret; + return rtp_twcc_stats_do_stats (twcc->stats_manager, stats_window_size, + stats_window_delay); } void diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtptwccstats.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtptwccstats.c new file mode 100644 index 0000000000..32fac01bd0 --- /dev/null +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtptwccstats.c @@ -0,0 +1,1393 @@ +#include "rtptwccstats.h" +#include +#include + +#define WEIGHT(a, b, w) (((a) * (w)) + ((b) * (1.0 - (w)))) + +#define PACKETS_HIST_DUR (10 * GST_SECOND) +/* How many packets should fit into the packets history by default. + Estimated bundle throughput is up to 150 per packets at maximum in average + circumstances. */ +#define PACKETS_HIST_LEN_DEFAULT (300 * PACKETS_HIST_DUR / GST_SECOND) + +#define MAX_STATS_PACKETS 1000 + +GST_DEBUG_CATEGORY_EXTERN (rtp_twcc_debug); +#define GST_CAT_DEFAULT rtp_twcc_debug + +typedef struct +{ + GstClockTime local_ts; + GstClockTime socket_ts; + GstClockTime remote_ts; + guint16 seqnum; + guint16 orig_seqnum; + guint32 ssrc; + guint8 pt; + guint size; + gboolean lost; + gint redundant_idx; /* if it's redudndant packet -- series number in a block, + -1 otherwise */ + gint redundant_num; /* if it'r a redundant packet -- how many packets are + in the block, -1 otherwise */ + guint32 protects_ssrc; /* for redundant packets: SSRC of the data stream */ + + /* For redundant packets: seqnums of the packets being protected + * by this packet. + * IMPORTANT: Once the packet is checked in before transmission, this array + * contains rtp seqnums. After receiving a feedback on the packet, the array + * is converted to TWCC seqnums. This is done to shift some work to the + * get_windowed_stats function, which should be less time-critical. + */ + GArray * protects_seqnums; + gboolean stats_processed; + + TWCCPktState state; + gint update_stats; +} SentPacket; + +typedef struct +{ + SentPacket * sentpkt; +} StatsPktPtr; + +static StatsPktPtr null_statspktptr = {.sentpkt=NULL}; + +typedef struct +{ + GstQueueArray *pt_packets; + SentPacket *last_pkt_fb; + gint64 new_packets_idx; + + /* windowed stats */ + guint packets_sent; + guint packets_recv; + guint bitrate_sent; + guint bitrate_recv; + gdouble packet_loss_pct; + gdouble recovery_pct; + gint64 avg_delta_of_delta; + gdouble delta_of_delta_growth; +} TWCCStatsCtx; + +struct _TWCCStatsManager +{ + GObject *parent; + + TWCCStatsCtx *stats_ctx; + /* The first packet in stats_ctx seqnum, valid even if there is a gap in + stats_ctx caused feedback packet loss + */ + gint32 stats_ctx_first_seqnum; + GHashTable *stats_ctx_by_pt; + GHashTable *ssrc_to_seqmap; + + /* In order to keep RingBuffer sizes under control, we assert + that the old packets we remove from the queues are older than statistics + window we use. + */ + GstClockTime prev_stat_window_beginning; + + GstQueueArray *sent_packets; + gsize sent_packets_size; + + /* Ring Buffer of pointers to SentPacket struct from sent_packets + to which we've got feedbacks, but not processed during statistics */ + GstQueueArray *sent_packets_feedbacks; + + /* Redundancy bookkeeping */ + GHashTable * redund_2_redblocks; + GHashTable * seqnum_2_redblocks; + + GstClockTimeDiff avg_rtt; + GstClockTimeDiff rtt; +}; + +/******************************************************************************/ +typedef GArray* RedBlockKey; + +typedef struct +{ + GArray *seqs; + GArray *states; + + GArray * fec_seqs; + GArray * fec_states; + + gsize num_redundant_packets; +} RedBlock; + +static RedBlock *_redblock_new(GArray* seq, guint16 fec_seq, + guint16 idx_redundant_packets, guint16 num_redundant_packets); +static void _redblock_free(RedBlock *block); +static RedBlockKey _redblock_key_new (GArray * seqs); +static void _redblock_key_free (RedBlockKey key); +static guint redblock_2_key(GArray * seq); +static guint _redund_hash (gconstpointer key); +static gboolean _redund_equal (gconstpointer a, gconstpointer b); +static gsize _redblock_reconsider (TWCCStatsManager * statsman, RedBlock * block); + + +static TWCCStatsCtx *_get_ctx_for_pt (TWCCStatsManager *statsman, guint pt); + +/******************************************************************************/ +/* Pick more definitive pkt state */ +static TWCCPktState +_better_pkt_state (TWCCPktState state1, TWCCPktState state2) +{ + if (state1 == RTP_TWCC_FECBLOCK_PKT_UNKNOWN) { + return state2; + } else if (state2 == RTP_TWCC_FECBLOCK_PKT_UNKNOWN) { + return state1; + } else if (state1 == RTP_TWCC_FECBLOCK_PKT_LOST) { + return state2; + } else if (state2 == RTP_TWCC_FECBLOCK_PKT_LOST) { + return state1; + } else if (state1 == RTP_TWCC_FECBLOCK_PKT_RECOVERED) { + return state2; + } else if (state2 == RTP_TWCC_FECBLOCK_PKT_RECOVERED) { + return state1; + } else { + return state1; + } +} + +static const gchar * +_pkt_state_s (TWCCPktState state) +{ + switch (state){ + case RTP_TWCC_FECBLOCK_PKT_UNKNOWN: + return "UNKNOWN"; + case RTP_TWCC_FECBLOCK_PKT_RECEIVED: + return "RECEIVED"; + case RTP_TWCC_FECBLOCK_PKT_RECOVERED: + return "RECOVERED"; + case RTP_TWCC_FECBLOCK_PKT_LOST: + return "LOST"; + default: + return "INVALID"; + } +} + +static StatsPktPtr* +_sent_pkt_get (GstQueueArray* pkt_array, guint idx) +{ + return (StatsPktPtr*) + gst_queue_array_peek_nth_struct (pkt_array, idx); +} + +static void +_append_structure_to_value_array (GValueArray * array, GstStructure * s) +{ + GValue *val; + g_value_array_append (array, NULL); + val = g_value_array_get_nth (array, array->n_values - 1); + g_value_init (val, GST_TYPE_STRUCTURE); + g_value_take_boxed (val, s); +} + +static void +_structure_take_value_array (GstStructure * s, + const gchar * field_name, GValueArray * array) +{ + GValue value = G_VALUE_INIT; + g_value_init (&value, G_TYPE_VALUE_ARRAY); + g_value_take_boxed (&value, array); + gst_structure_take_value (s, field_name, &value); + g_value_unset (&value); +} + +static void +_register_seqnum (TWCCStatsManager *statsman, + guint32 ssrc, guint16 seqnum, guint16 twcc_seqnum) +{ + GHashTable *seq_to_twcc = + g_hash_table_lookup (statsman->ssrc_to_seqmap, GUINT_TO_POINTER (ssrc)); + if (!seq_to_twcc) { + seq_to_twcc = g_hash_table_new (NULL, NULL); + g_hash_table_insert (statsman->ssrc_to_seqmap, GUINT_TO_POINTER (ssrc), + seq_to_twcc); + } + g_hash_table_insert (seq_to_twcc, GUINT_TO_POINTER (seqnum), + GUINT_TO_POINTER (twcc_seqnum)); + GST_LOG_OBJECT (statsman->parent, "Registering OSN: %u to statsman-twcc_seqnum: %u with ssrc: %u", seqnum, + twcc_seqnum, ssrc); +} + +static void +_sent_packet_init (SentPacket * packet, guint16 seqnum, RTPPacketInfo * pinfo, + GstRTPBuffer * rtp, gint redundant_idx, gint redundant_num, + guint32 protect_ssrc, GArray *protect_seqnums_array) +{ + packet->seqnum = seqnum; + packet->orig_seqnum = gst_rtp_buffer_get_seq (rtp); + packet->ssrc = gst_rtp_buffer_get_ssrc (rtp); + packet->local_ts = pinfo->current_time; + packet->size = pinfo->bytes + 12; /* the reported wireshark size */ + packet->pt = gst_rtp_buffer_get_payload_type (rtp); + packet->remote_ts = GST_CLOCK_TIME_NONE; + packet->socket_ts = GST_CLOCK_TIME_NONE; + packet->lost = FALSE; + packet->state = RTP_TWCC_FECBLOCK_PKT_UNKNOWN; + packet->redundant_idx = redundant_idx; + packet->redundant_num = redundant_num; + packet->protects_ssrc = protect_ssrc; + packet->protects_seqnums = protect_seqnums_array; + packet->stats_processed = FALSE; +} + +static void +_free_sentpacket (SentPacket * pkt) +{ + if (pkt && pkt->protects_seqnums) { + g_array_unref (pkt->protects_seqnums); + } +} + +static void +_sent_pkt_keep_length (TWCCStatsManager *statsman, gsize max_len, + SentPacket* new_packet) +{ + if (gst_queue_array_get_length(statsman->sent_packets) >= max_len) { + /* It could mean that statistics was not called at all, asumming that + the oldest packet was not referenced anywhere else, we can drop it. + */ + SentPacket * head = (SentPacket*)gst_queue_array_peek_head_struct (statsman->sent_packets); + GstClockTime pkt_ts = head->local_ts; + if (GST_CLOCK_TIME_IS_VALID(statsman->prev_stat_window_beginning) && + GST_CLOCK_DIFF (pkt_ts, statsman->prev_stat_window_beginning) + < 0) { + GST_WARNING_OBJECT (statsman->parent, "sent_packets FIFO overflows, dropping"); + g_assert_not_reached (); + } else if (GST_CLOCK_TIME_IS_VALID(statsman->prev_stat_window_beginning) && + GST_CLOCK_DIFF (pkt_ts, statsman->prev_stat_window_beginning) + < GST_MSECOND * 250) { + GST_WARNING_OBJECT (statsman->parent, "Risk of" + " underrun of sent_packets FIFO"); + } + GST_LOG_OBJECT (statsman->parent, "Keeping sent_packets FIFO length: %u, dropping packet #%u", + max_len, head->seqnum); + _free_sentpacket (gst_queue_array_pop_head_struct (statsman->sent_packets)); + } + gst_queue_array_push_tail_struct (statsman->sent_packets, new_packet); +} + +static TWCCStatsCtx * +twcc_stats_ctx_new (void) +{ + TWCCStatsCtx *ctx = g_new0 (TWCCStatsCtx, 1); + + ctx->pt_packets = gst_queue_array_new_for_struct (sizeof(StatsPktPtr), + MAX_STATS_PACKETS); + ctx->last_pkt_fb = NULL; + + return ctx; +} + +static void +twcc_stats_ctx_free (TWCCStatsCtx * ctx) +{ + gst_queue_array_free (ctx->pt_packets); + g_free (ctx); +} + +static GstClockTime +_pkt_stats_ts (SentPacket * pkt) +{ + return GST_CLOCK_TIME_IS_VALID (pkt->socket_ts) + ? pkt->socket_ts + : pkt->local_ts; +} + +static GstClockTime +twcc_stats_ctx_get_last_local_ts (TWCCStatsCtx * ctx) +{ + GstClockTime ret = GST_CLOCK_TIME_NONE; + SentPacket *pkt = ctx->last_pkt_fb; + if (pkt) { + ret = _pkt_stats_ts (pkt); + } + return ret; +} + +static gboolean +_get_stats_packets_window (GstQueueArray * array, + GstClockTimeDiff start_time, GstClockTimeDiff end_time, + guint * start_idx, guint * num_packets) +{ + gboolean ret = FALSE; + guint end_idx = 0; + guint i; + const guint array_length = gst_queue_array_get_length (array); + + if (array_length < 2) { + GST_DEBUG ("Not enough stats to do a window"); + return FALSE; + } + + for (i = 0; i < array_length; i++) { + SentPacket *pkt = _sent_pkt_get (array, i)->sentpkt; + if (!pkt) { + continue; + } + /* Do not process packets that were not reported about in feedbacks + yet. */ + if (pkt->state == RTP_TWCC_FECBLOCK_PKT_UNKNOWN) { + continue; + } + const GstClockTime pkt_ts = _pkt_stats_ts (pkt); + if (GST_CLOCK_TIME_IS_VALID (pkt_ts)) { + GstClockTimeDiff offset = GST_CLOCK_DIFF (pkt_ts, start_time); + *start_idx = i; + /* positive number here means it is older than our start time */ + if (offset > 0) { + GST_LOG ("Packet #%u is too old: %" + GST_TIME_FORMAT, pkt->seqnum, GST_TIME_ARGS (pkt_ts)); + } else { + GST_LOG ("Setting first packet in our window to #%u: %" + GST_TIME_FORMAT, pkt->seqnum, GST_TIME_ARGS (pkt_ts)); + ret = TRUE; + break; + } + } + } + + /* jump out early if we could not find a start_idx */ + if (!ret) { + return FALSE; + } + + ret = FALSE; + for (i = 0; i < array_length - *start_idx - 1; i++) { + guint idx = array_length - 1 - i; + SentPacket *pkt = _sent_pkt_get (array, idx)->sentpkt; + if (!pkt) { + continue; + } + const GstClockTime pkt_ts = _pkt_stats_ts (pkt); + if (pkt_ts) { + GstClockTimeDiff offset = GST_CLOCK_DIFF (pkt_ts, end_time); + if (offset >= 0) { + GST_LOG ("Setting last packet in our window to #%u: %" + GST_TIME_FORMAT, pkt->seqnum, GST_TIME_ARGS (pkt_ts)); + end_idx = idx; + ret = TRUE; + break; + } else { + GST_LOG ("Packet #%u is too new: %" + GST_TIME_FORMAT, pkt->seqnum, GST_TIME_ARGS (pkt_ts)); + } + } + } + + /* jump out early if we could not find a window */ + if (!ret) { + return FALSE; + } + + *num_packets = end_idx - *start_idx + 1; + + return ret; +} + +static void _rm_last_stats_pkt (TWCCStatsManager * ctx); + +static gboolean +twcc_stats_ctx_calculate_windowed_stats (TWCCStatsCtx * ctx, + GstClockTimeDiff start_time, GstClockTimeDiff end_time) +{ + GstQueueArray *packets = ctx->pt_packets; + guint start_idx; + guint packets_sent = 0; + guint packets_recv = 0; + guint packets_recovered = 0; + guint packets_lost = 0; + + guint i; + guint bits_sent = 0; + guint bits_recv = 0; + + GstClockTimeDiff delta_delta_sum = 0; + guint delta_delta_count = 0; + GstClockTimeDiff first_delta_delta_sum = 0; + guint first_delta_delta_count = 0; + GstClockTimeDiff last_delta_delta_sum = 0; + guint last_delta_delta_count = 0; + + SentPacket *first_local_pkt = NULL; + SentPacket *last_local_pkt = NULL; + SentPacket *first_remote_pkt = NULL; + SentPacket *last_remote_pkt = NULL; + + GstClockTimeDiff local_duration = 0; + GstClockTimeDiff remote_duration = 0; + + ctx->packet_loss_pct = 0.0; + ctx->avg_delta_of_delta = 0; + ctx->delta_of_delta_growth = 0.0; + ctx->bitrate_sent = 0; + ctx->bitrate_recv = 0; + ctx->recovery_pct = -1.0; + + gboolean ret = + _get_stats_packets_window (packets, start_time, end_time, &start_idx, + &packets_sent); + if (!ret || packets_sent < 2) { + GST_INFO ("Not enough packets to fill our window yet!"); + return FALSE; + } else { + GST_DEBUG ("Stats window: %u packets, pt: %d, %d->%d", packets_sent, + _sent_pkt_get (packets, start_idx) ->sentpkt->pt, + _sent_pkt_get (packets, start_idx)->sentpkt->seqnum, + _sent_pkt_get (packets, packets_sent + start_idx - 1)->sentpkt->seqnum); + } + + for (i = 0; i < packets_sent; i++) { + SentPacket *prev = NULL; + if (i + start_idx >= 1) + prev = _sent_pkt_get (packets, i + start_idx)->sentpkt; + + SentPacket *pkt = _sent_pkt_get(packets, i + start_idx)->sentpkt; + if (!pkt) { + continue; + } + GST_LOG ("STATS WINDOW: %u/%u: pkt #%u, pt: %u, size: %u, arrived: %s, " + "local-ts: %" GST_TIME_FORMAT ", remote-ts %" GST_TIME_FORMAT, + i + 1, packets_sent, pkt->seqnum, pkt->pt, pkt->size * 8, + GST_CLOCK_TIME_IS_VALID (pkt->remote_ts) ? "YES" : "NO", + GST_TIME_ARGS (_pkt_stats_ts (pkt)), GST_TIME_ARGS (pkt->remote_ts)); + + if (GST_CLOCK_TIME_IS_VALID (_pkt_stats_ts (pkt))) { + /* don't count the bits for the first packet in the window */ + if (first_local_pkt == NULL) { + first_local_pkt = pkt; + } else { + bits_sent += pkt->size * 8; + } + last_local_pkt = pkt; + } + + if (pkt->state == RTP_TWCC_FECBLOCK_PKT_RECEIVED) { + /* don't count the bits for the first packet in the window */ + if (first_remote_pkt == NULL) { + first_remote_pkt = pkt; + } else { + bits_recv += pkt->size * 8; + } + last_remote_pkt = pkt; + packets_recv++; + } else if (pkt->state == RTP_TWCC_FECBLOCK_PKT_RECOVERED) { + GST_LOG ("Packet #%u is lost and recovered", pkt->seqnum); + packets_lost++; + packets_recovered++; + } else if(pkt->state == RTP_TWCC_FECBLOCK_PKT_LOST) { + GST_LOG ("Packet #%u is lost", pkt->seqnum); + packets_lost++; + } + + if (!prev) { + continue; + } + + GstClockTimeDiff local_delta = GST_CLOCK_STIME_NONE; + GstClockTimeDiff remote_delta = GST_CLOCK_STIME_NONE; + GstClockTimeDiff delta_delta = GST_CLOCK_STIME_NONE; + + if (GST_CLOCK_TIME_IS_VALID (_pkt_stats_ts (pkt)) && + GST_CLOCK_TIME_IS_VALID (_pkt_stats_ts (prev))) { + local_delta = GST_CLOCK_DIFF (_pkt_stats_ts (prev), + _pkt_stats_ts (pkt)); + } + + if (GST_CLOCK_TIME_IS_VALID (pkt->remote_ts) && + GST_CLOCK_TIME_IS_VALID (prev->remote_ts)) { + remote_delta = GST_CLOCK_DIFF (prev->remote_ts, pkt->remote_ts); + } + + if (GST_CLOCK_STIME_IS_VALID (local_delta) && + GST_CLOCK_STIME_IS_VALID (remote_delta)) { + delta_delta = remote_delta - local_delta; + + delta_delta_sum += delta_delta; + delta_delta_count++; + if (i < packets_sent / 2) { + first_delta_delta_sum += delta_delta; + first_delta_delta_count++; + } else { + last_delta_delta_sum += delta_delta; + last_delta_delta_count++; + } + } + } + + ctx->packets_sent = packets_sent; + ctx->packets_recv = packets_recv; + + if (first_local_pkt && last_local_pkt) { + local_duration = + GST_CLOCK_DIFF (_pkt_stats_ts (first_local_pkt), + _pkt_stats_ts (last_local_pkt)); + } + if (first_remote_pkt && last_remote_pkt) { + remote_duration = + GST_CLOCK_DIFF (first_remote_pkt->remote_ts, + last_remote_pkt->remote_ts); + } + + if (packets_sent) + ctx->packet_loss_pct = (packets_lost * 100) / (gfloat) packets_sent; + + if (packets_lost) { + ctx->recovery_pct = (packets_recovered * 100) / (gfloat) packets_lost; + ctx->recovery_pct = MIN(ctx->recovery_pct, 100); + } + + if (delta_delta_count) { + ctx->avg_delta_of_delta = delta_delta_sum / delta_delta_count; + } + + if (first_delta_delta_count && last_delta_delta_count) { + GstClockTimeDiff first_avg = + first_delta_delta_sum / first_delta_delta_count; + GstClockTimeDiff last_avg = last_delta_delta_sum / last_delta_delta_count; + + /* filter out very small numbers */ + first_avg = MAX (first_avg, 100 * GST_USECOND); + last_avg = MAX (last_avg, 100 * GST_USECOND); + ctx->delta_of_delta_growth = (double) last_avg / (double) first_avg; + } + + if (local_duration > 0) { + ctx->bitrate_sent = + gst_util_uint64_scale (bits_sent, GST_SECOND, local_duration); + } + if (remote_duration > 0) { + ctx->bitrate_recv = + gst_util_uint64_scale (bits_recv, GST_SECOND, remote_duration); + } + + GST_INFO ("Got stats: bits_sent: %u, bits_recv: %u, packets_sent = %u, " + "packets_recv: %u, packetlost_pct = %lf, recovery_pct = %lf, " + "recovered: %u, local_duration=%" GST_TIME_FORMAT ", " + "remote_duration=%" GST_TIME_FORMAT ", " + "sent_bitrate = %u, " "recv_bitrate = %u, delta-delta-avg = %" + GST_STIME_FORMAT ", delta-delta-growth=%lf", bits_sent, bits_recv, + packets_sent, packets_recv, ctx->packet_loss_pct, ctx->recovery_pct, + packets_recovered, + GST_TIME_ARGS (local_duration), GST_TIME_ARGS (remote_duration), + ctx->bitrate_sent, ctx->bitrate_recv, + GST_STIME_ARGS (ctx->avg_delta_of_delta), ctx->delta_of_delta_growth); + + return TRUE; +} + +static GstStructure * +twcc_stats_ctx_get_structure (TWCCStatsCtx * ctx) +{ + return gst_structure_new ("RTPTWCCStats", + "packets-sent", G_TYPE_UINT, ctx->packets_sent, + "packets-recv", G_TYPE_UINT, ctx->packets_recv, + "bitrate-sent", G_TYPE_UINT, ctx->bitrate_sent, + "bitrate-recv", G_TYPE_UINT, ctx->bitrate_recv, + "packet-loss-pct", G_TYPE_DOUBLE, ctx->packet_loss_pct, + "recovery-pct", G_TYPE_DOUBLE, ctx->recovery_pct, + "avg-delta-of-delta", G_TYPE_INT64, ctx->avg_delta_of_delta, + "delta-of-delta-growth", G_TYPE_DOUBLE, ctx->delta_of_delta_growth, NULL); +} + +static gint +_idx_sentpacket (TWCCStatsManager * statsman, guint16 seqnum) +{ + const gint idx = gst_rtp_buffer_compare_seqnum ( + (guint16)statsman->stats_ctx_first_seqnum, seqnum); + if (statsman->stats_ctx_first_seqnum >= 0 && idx >= 0) { + return idx; + } else { + return -1; + } +} + +static SentPacket * +_find_stats_sentpacket (TWCCStatsManager * statsman, guint16 seqnum) +{ + gint idx = _idx_sentpacket (statsman, seqnum); + SentPacket * res = NULL; + if (idx >= 0 + && idx < gst_queue_array_get_length (statsman->stats_ctx->pt_packets)) { + res = _sent_pkt_get (statsman->stats_ctx->pt_packets, idx)->sentpkt; + } + + return res; +} + +static TWCCStatsCtx * +_get_ctx_for_pt (TWCCStatsManager *statsman, guint pt); + +static void +twcc_stats_ctx_add_packet (TWCCStatsManager *statsman, SentPacket * pkt) +{ + if (!pkt) { + return; + } else if (gst_queue_array_is_empty (statsman->stats_ctx->pt_packets)) { + gst_queue_array_push_tail_struct (statsman->stats_ctx->pt_packets, + (StatsPktPtr*)&pkt); + TWCCStatsCtx *ctx = _get_ctx_for_pt (statsman, pkt->pt); + gst_queue_array_push_tail_struct (ctx->pt_packets, (StatsPktPtr*)&pkt); + statsman->stats_ctx->last_pkt_fb = ctx->last_pkt_fb = pkt; + statsman->stats_ctx_first_seqnum = pkt->seqnum; + } else { + const gint idx = _idx_sentpacket (statsman, pkt->seqnum); + GstQueueArray * main_array = statsman->stats_ctx->pt_packets; + if (idx < 0) { + GST_WARNING_OBJECT (statsman->parent, "Packet #%u is too old for stats, dropping, latest pkt is #%u", + pkt->seqnum, statsman->stats_ctx_first_seqnum); + return; + } else if (idx >= gst_queue_array_get_length (main_array)) { + const gsize n2push = idx - gst_queue_array_get_length (main_array); + /* if n2push > 0 means that the last statsman feedback packet[s] is lost, + */ + for (gsize i = 0; i < n2push; i++) { + gst_queue_array_push_tail_struct (main_array, + &null_statspktptr); + } + gst_queue_array_push_tail_struct (main_array, (StatsPktPtr*)&pkt); + TWCCStatsCtx *ctx = _get_ctx_for_pt (statsman, pkt->pt); + gst_queue_array_push_tail_struct (ctx->pt_packets, (StatsPktPtr*)&pkt); + statsman->stats_ctx->last_pkt_fb = ctx->last_pkt_fb = pkt; + } else { + /* TWCC packets reordered -- do nothing */ + GST_WARNING_OBJECT (statsman->parent, "Packet #%u is out of order, not going to stats", pkt->seqnum); + } + } +} + +/******************************************************************************/ +/* Redundancy book keeping subpart + "Was a certain packet recovered on the receiver side?" + + * Organizes sent data packets and redundant packets into blocks + * Keeps track of redundant packets reception such as RTX and FEC packets + * Maps all packets to blocks and vice versa + * Is used to calculate redundancy statistics + */ + +static RedBlock * _redblock_new(GArray* seq, guint16 fec_seq, + guint16 idx_redundant_packets, guint16 num_redundant_packets) +{ + RedBlock *block = g_malloc (sizeof (RedBlock)); + block->seqs = g_array_ref (seq); + block->states = g_array_new (FALSE, FALSE, sizeof (TWCCPktState)); + g_array_set_size (block->states, seq->len); + for (gsize i = 0; i < seq->len; i++) + { + g_array_index (block->states, TWCCPktState, i) = + RTP_TWCC_FECBLOCK_PKT_UNKNOWN; + } + block->num_redundant_packets = num_redundant_packets; + + block->fec_seqs = g_array_new (FALSE, FALSE, sizeof (guint16)); + if (num_redundant_packets < 1 || idx_redundant_packets >= num_redundant_packets) { + GST_ERROR ("Incorrect redundant packet index or number: %hu/%hu", + idx_redundant_packets, num_redundant_packets); + g_assert_not_reached (); + } + g_array_set_size (block->fec_seqs, num_redundant_packets); + block->fec_states = g_array_new (FALSE, FALSE, sizeof (TWCCPktState)); + g_array_set_size (block->fec_states, num_redundant_packets); + for (guint16 i = 0; i < num_redundant_packets; i++) + { + g_array_index (block->fec_states, TWCCPktState, i) + = RTP_TWCC_FECBLOCK_PKT_UNKNOWN; + g_array_index (block->fec_seqs, guint16, i) = 0; + } + g_array_index (block->fec_seqs, guint16, idx_redundant_packets) + = fec_seq; + return block; +} + +static void _redblock_free(RedBlock *block) { + g_array_unref (block->seqs); + g_array_free (block->states, TRUE); + g_array_free (block->fec_seqs, TRUE); + g_array_free (block->fec_states, TRUE); + g_free (block); +} + +static RedBlockKey _redblock_key_new (GArray * seqs) +{ + return g_array_ref (seqs); +} + +static void _redblock_key_free (RedBlockKey key) +{ + g_array_unref (key); +} + +static guint redblock_2_key(GArray * seq) +{ + guint32 key = 0; + gsize i = 0; + /* In reality seq contains guint16, but we treat it as 32bits ints till + we can */ + for (; i < seq->len / 2; i += 2) { + key ^= g_array_index(seq, guint32, i / 2); + } + for (; i < seq->len; i++) { + key ^= g_array_index(seq, guint16, i); + } + return key; +} + +static guint _redund_hash (gconstpointer key) +{ + RedBlockKey bk = (RedBlockKey)key; + return redblock_2_key(bk); +} + +static gboolean _redund_equal (gconstpointer a, gconstpointer b) +{ + RedBlockKey bk1 = (RedBlockKey)a; + RedBlockKey bk2 = (RedBlockKey)b; + return bk1->len == bk2->len && + memcmp(bk1->data, bk2->data, bk1->len * sizeof(guint16)) + == 0; +} + +/* Check if the block could be recovered: + * all packets have known states + * number of lost packets is less than redundant packets were originally sent + + Returns the number of recoverd packets +*/ +static gsize _redblock_reconsider (TWCCStatsManager * statsman, RedBlock * block) +{ + gsize nreceived = 0; + gboolean recovered = FALSE; + gboolean unknowns = FALSE; + gsize nrecovered = 0; + gsize lost = 0; + + gchar states_media[48]; + gchar states_fec[16]; + + /* Special case for RTX: lost RTX introduces extra complexity which + is easier to handle separately + */ + if (block->seqs->len == 1) { + SentPacket *pkt = _find_stats_sentpacket (statsman, + g_array_index (block->seqs, guint16, 0)); + + if (pkt && pkt->state == RTP_TWCC_FECBLOCK_PKT_LOST) { + for (gsize i = 0; i < block->fec_seqs->len; ++i) { + SentPacket *pkt = _find_stats_sentpacket (statsman, + g_array_index (block->fec_seqs, guint16, i)); + if (pkt->state == RTP_TWCC_FECBLOCK_PKT_RECEIVED) { + nrecovered++; + break; + } + } + if (nrecovered == 1) { + pkt->state = RTP_TWCC_FECBLOCK_PKT_RECOVERED; + } + } + + return nrecovered; + } + + /* Walk through all the packets and check if the block could be recovered */ + for (gsize i = 0; i < block->seqs->len; ++i) { + SentPacket *pkt = _find_stats_sentpacket (statsman, + g_array_index (block->seqs, guint16, i)); + if (!pkt || pkt->state == RTP_TWCC_FECBLOCK_PKT_UNKNOWN) { + unknowns = TRUE; + if (i < G_N_ELEMENTS(states_media)) states_media[i] = 'U'; + } else if (pkt->state == RTP_TWCC_FECBLOCK_PKT_RECEIVED) { + nreceived++; + if (i < G_N_ELEMENTS(states_media)) states_media[i] = '+'; + } else if (pkt->state == RTP_TWCC_FECBLOCK_PKT_RECOVERED) { + recovered = TRUE; + if (i < G_N_ELEMENTS(states_media)) states_media[i] = 'R'; + } else if (pkt->state == RTP_TWCC_FECBLOCK_PKT_LOST) { + lost++; + if (i < G_N_ELEMENTS(states_media)) states_media[i] = '-'; + } + + if (pkt) { + pkt->update_stats = FALSE; + } + } + states_media[block->seqs->len] = '\0'; + + /* Walk through all fec packets */ + for (gsize i = 0; i < block->fec_seqs->len; ++i) { + if (g_array_index (block->fec_states, TWCCPktState, i) + == RTP_TWCC_FECBLOCK_PKT_UNKNOWN) { + unknowns = TRUE; + if (i < G_N_ELEMENTS(states_fec)) states_fec[i] = 'U'; + continue; + } + SentPacket *pkt = _find_stats_sentpacket (statsman, + g_array_index (block->fec_seqs, guint16, i)); + if (!pkt || pkt->state == RTP_TWCC_FECBLOCK_PKT_UNKNOWN) { + unknowns = TRUE; + if (i < G_N_ELEMENTS(states_fec)) states_fec[i] = 'U'; + } else if (pkt->state == RTP_TWCC_FECBLOCK_PKT_RECEIVED) { + nreceived++; + if (i < G_N_ELEMENTS(states_fec)) states_fec[i] = '+'; + } else if (pkt->state == RTP_TWCC_FECBLOCK_PKT_RECOVERED) { + recovered = TRUE; + if (i < G_N_ELEMENTS(states_fec)) states_fec[i] = 'R'; + } else if (pkt->state == RTP_TWCC_FECBLOCK_PKT_LOST) { + lost++; + if (i < G_N_ELEMENTS(states_fec)) states_fec[i] = '-'; + } + + if (pkt) { + pkt->update_stats = FALSE; + } + } + states_fec[block->fec_seqs->len] = '\0'; + + + /* We have packet[s] that was not reported about in feedbacks yet */ + if (unknowns) { + GST_INFO ("Media: %s; FEC: %s", states_media, states_fec); + GST_INFO ("The FEC block has unknown packets"); + return 0; + } + + /* Error: it's not possible to recover a part of a block */ + if ((lost + nreceived != block->seqs->len + block->fec_seqs->len) + || (lost > 0 && recovered)) { + GST_ERROR ("The FEC block is partly recovered, abort: %lu lost, %lu/%lu received", + lost, nreceived, block->seqs->len + block->fec_seqs->len); + g_assert_not_reached (); + } + + if (lost > 0 && lost <= block->fec_seqs->len) { + /* We have enough packets to recover the block */ + for (gsize i = 0; i < block->seqs->len; ++i) { + SentPacket *pkt = _find_stats_sentpacket (statsman, + g_array_index (block->seqs, guint16, i)); + if (pkt->state == RTP_TWCC_FECBLOCK_PKT_LOST) { + pkt->state = RTP_TWCC_FECBLOCK_PKT_RECOVERED; + nrecovered++; + } + } + for (gsize i = 0; i < block->fec_seqs->len; ++i) { + SentPacket *pkt = _find_stats_sentpacket (statsman, + g_array_index (block->fec_seqs, guint16, i)); + if (pkt->state == RTP_TWCC_FECBLOCK_PKT_LOST) { + pkt->state = RTP_TWCC_FECBLOCK_PKT_RECOVERED; + nrecovered++; + } + } + } + + GST_INFO ("Media: %s; FEC: %s; recovered: %lu", states_media, states_fec, + nrecovered); + return nrecovered; +} + +/******************************************************************************/ + +static TWCCStatsCtx * +_get_ctx_for_pt (TWCCStatsManager * statsman, guint pt) +{ + TWCCStatsCtx *ctx = + g_hash_table_lookup (statsman->stats_ctx_by_pt, GUINT_TO_POINTER (pt)); + if (!ctx) { + ctx = twcc_stats_ctx_new (); + g_hash_table_insert (statsman->stats_ctx_by_pt, GUINT_TO_POINTER (pt), ctx); + ctx->last_pkt_fb = NULL; + } + return ctx; +} + +static void +_rm_last_packet_from_stats_arrays (TWCCStatsManager *statsman) +{ + SentPacket * head = ((StatsPktPtr*)gst_queue_array_peek_head_struct ( + statsman->stats_ctx->pt_packets))->sentpkt; + if (head) { + TWCCStatsCtx * ctx = _get_ctx_for_pt (statsman, head->pt); + SentPacket * ctx_pkt = ((StatsPktPtr*)gst_queue_array_peek_head_struct ( + ctx->pt_packets))->sentpkt; + if (!ctx_pkt || ctx_pkt->seqnum != head->seqnum) { + GST_WARNING_OBJECT (statsman->parent, "Attempting to remove packet from pt stats context " + "which seqnum does not match the main stats context seqnum, " + "main: #%u, pt: %u, context packet: #%u, pt: %u", + head->seqnum, head->pt, + ctx_pkt ? ctx_pkt->seqnum : -1, ctx_pkt ? ctx_pkt->pt : -1); + g_assert_not_reached (); + } + if (ctx->last_pkt_fb == head) { + statsman->stats_ctx->last_pkt_fb = ctx->last_pkt_fb = NULL; + } + gst_queue_array_pop_head_struct (ctx->pt_packets); + GST_LOG_OBJECT (statsman->parent, "Removing packet #%u from stats context, ts: %" GST_STIME_FORMAT, + head->seqnum, head->local_ts); + } + gst_queue_array_pop_head_struct (statsman->stats_ctx->pt_packets); + statsman->stats_ctx_first_seqnum++; +} + +static void +_rm_last_stats_pkt (TWCCStatsManager * ctx) +{ + SentPacket * head = ((StatsPktPtr*)gst_queue_array_peek_head_struct ( + ctx->stats_ctx->pt_packets))->sentpkt; + /* If this packet maps to a block in hash tables -- remove every links + leading to this block as well as this packet: as we will remove this packet + from the context, we will not be able to use this block anyways. */ + RedBlock * block = NULL; + if (head && g_hash_table_lookup_extended (ctx->seqnum_2_redblocks, + GUINT_TO_POINTER(head->seqnum), NULL, (gpointer *)&block)) { + RedBlockKey key = _redblock_key_new (block->seqs); + for (gsize i = 0; i < block->seqs->len; i++) { + g_hash_table_remove (ctx->seqnum_2_redblocks, + GUINT_TO_POINTER(g_array_index (block->seqs, guint16, i))); + } + for (gsize i = 0; i < block->fec_seqs->len; i++) { + g_hash_table_remove (ctx->seqnum_2_redblocks, + GUINT_TO_POINTER(g_array_index (block->fec_seqs, guint16, i))); + } + g_hash_table_remove (ctx->redund_2_redblocks, key); + _redblock_key_free (key); + } + _rm_last_packet_from_stats_arrays (ctx); +} + +static gint32 +_lookup_seqnum (TWCCStatsManager *statsman, guint32 ssrc, guint16 seqnum) +{ + gint32 ret = -1; + + GHashTable *seq_to_twcc = + g_hash_table_lookup (statsman->ssrc_to_seqmap, GUINT_TO_POINTER (ssrc)); + if (seq_to_twcc) { + if (g_hash_table_lookup_extended (seq_to_twcc, GUINT_TO_POINTER (seqnum), + NULL, (gpointer *)&ret)) { + return ret; + } else { + return -1; + } + } + return ret; +} + +static SentPacket * +_find_sentpacket (TWCCStatsManager * statsman, guint16 seqnum) +{ + if (gst_queue_array_is_empty (statsman->sent_packets) == TRUE) { + return NULL; + } + + SentPacket * first = gst_queue_array_peek_head_struct (statsman->sent_packets); + SentPacket * result; + + const gint idx = gst_rtp_buffer_compare_seqnum (first->seqnum, seqnum); + if (idx < gst_queue_array_get_length (statsman->sent_packets) && idx >= 0) { + result = (SentPacket*) + gst_queue_array_peek_nth_struct (statsman->sent_packets, idx); + } else { + result = NULL; + } + + if (result && result->seqnum == seqnum) { + return result; + } + + return NULL; +} + +/* Once we've got feedback on a packet, we need to account it in the internal + structures. */ +static void +_process_pkt_feedback (SentPacket * pkt, TWCCStatsManager * statsman) +{ + if (pkt->stats_processed) { + /* This packet was already added to stats structures, but we've got + one more feedback for it + */ + RedBlock * block; + if (g_hash_table_lookup_extended (statsman->seqnum_2_redblocks, + GUINT_TO_POINTER(pkt->seqnum), NULL, (gpointer *)&block)) { + const gsize packets_recovered = _redblock_reconsider (statsman, block); + if (packets_recovered > 0) { + GST_LOG_OBJECT (statsman->parent, "Reconsider block because of packet #%u, " + "recovered %lu pckt", pkt->seqnum, packets_recovered); + } + } + return; + } + pkt->stats_processed = TRUE; + GST_LOG_OBJECT (statsman->parent, "Processing #%u packet in stats, state: %s", pkt->seqnum, + _pkt_state_s (pkt->state)); + + twcc_stats_ctx_add_packet (statsman, pkt); + + /* This is either RTX or FEC packet */ + if (pkt->protects_seqnums && pkt->protects_seqnums->len > 0) { + /* We are expecting non-twcc seqnums in the buffer's meta here, so + change them to twcc seqnums. */ + + if (pkt->redundant_idx < 0 || pkt->redundant_num <= 0 + || pkt->redundant_idx >= pkt->redundant_num) { + GST_ERROR_OBJECT (statsman->parent, "Invalid FEC packet: idx: %d, num: %d", + pkt->redundant_idx, pkt->redundant_num); + g_assert_not_reached (); + } + + for (gsize i = 0; i < pkt->protects_seqnums->len; i++) { + const guint16 prot_seqnum = g_array_index (pkt->protects_seqnums, + guint16, i); + gint32 twcc_seqnum = _lookup_seqnum (statsman, pkt->protects_ssrc, + prot_seqnum); + if (twcc_seqnum != -1) { + g_array_index (pkt->protects_seqnums, guint16, i) + = (guint16)twcc_seqnum; + } + GST_LOG_OBJECT (statsman->parent, "FEC sn: #%u covers twcc sn: #%u, orig sn: %u", + pkt->seqnum, twcc_seqnum, prot_seqnum); + } + + /* Check if this packet covers the same block that was already added. */ + RedBlockKey key = _redblock_key_new (pkt->protects_seqnums); + RedBlock * block = NULL; + if (g_hash_table_lookup_extended (statsman->redund_2_redblocks, key, NULL, + (gpointer*)&block)) { + /* This is not RTX, check this redundant pkt meta */ + if (pkt->redundant_num > 1 && + (block->fec_seqs->len != pkt->redundant_num + || block->fec_states->len != pkt->redundant_num)) { + + GST_WARNING_OBJECT (statsman->parent, "Got contradictory FEC block: " + "seqs: %u, states: %u, redundant_num: %d, redundant_idx: %d", + block->fec_seqs->len, block->fec_states->len, pkt->redundant_num, pkt->redundant_idx); + _redblock_key_free (key); + return; + /* This is 2nd or more attempt of RTX */ + } else if (pkt->redundant_num == 1) { + pkt->redundant_idx = block->fec_seqs->len; + block->num_redundant_packets++; + g_array_set_size (block->fec_seqs, block->num_redundant_packets); + g_array_set_size (block->fec_states, block->num_redundant_packets); + + GST_LOG_OBJECT (statsman->parent, "Adding redundant packet #%u to" + " an exsiting block on position %u", pkt->seqnum, pkt->redundant_idx); + } + g_array_index (block->fec_seqs, guint16, (gsize)pkt->redundant_idx) + = pkt->seqnum; + g_array_index (block->fec_states, TWCCPktState, + (gsize)pkt->redundant_idx) = pkt->state; + + _redblock_key_free (key); + + /* Link this seqnum to the block in order to be able to + release the block once this packet leave its lifetime */ + g_hash_table_insert (statsman->seqnum_2_redblocks, + GUINT_TO_POINTER(pkt->seqnum), block); + /* There is no such block, add a new one */ + } else { + /* Add every data packet into seqnum_2_redblocks */ + block = _redblock_new (pkt->protects_seqnums, pkt->seqnum, + pkt->redundant_idx, pkt->redundant_num); + g_array_index (block->fec_seqs, guint16, (gsize)pkt->redundant_idx) + = pkt->seqnum; + g_array_index (block->fec_states, TWCCPktState, + (gsize)pkt->redundant_idx) = pkt->state; + g_hash_table_insert (statsman->redund_2_redblocks, key, block); + /* Link this seqnum to the block in order to be able to + release the block once this packet leave its lifetime */ + g_hash_table_insert (statsman->seqnum_2_redblocks, + GUINT_TO_POINTER(pkt->seqnum), block); + for (gsize i = 0; i < pkt->protects_seqnums->len; ++i) { + const guint64 data_key = g_array_index (pkt->protects_seqnums, + guint16, i); + RedBlock * data_block = NULL; + if (!g_hash_table_lookup_extended (statsman->seqnum_2_redblocks, + GUINT_TO_POINTER(data_key), NULL, (gpointer*)&data_block)) { + + g_hash_table_insert (statsman->seqnum_2_redblocks, + GUINT_TO_POINTER(data_key), block); + } else if (block != data_block) { + /* Overlapped blocks are not supported yet */ + GST_WARNING_OBJECT (statsman->parent, "Data packet %ld covered by two blocks", + data_key); + g_hash_table_replace (statsman->seqnum_2_redblocks, + GUINT_TO_POINTER(data_key), block); + } + } + } + const gsize packets_recovered = _redblock_reconsider (statsman, block); + GST_LOG_OBJECT (statsman->parent, "Reconsider block because of packet #%u, recovered %lu pckt", + pkt->seqnum, packets_recovered); + /* Neither RTX nor FEC */ + } else { + RedBlock * block; + if (g_hash_table_lookup_extended (statsman->seqnum_2_redblocks, + GUINT_TO_POINTER(pkt->seqnum), NULL, (gpointer *)&block)) { + + for (gsize i = 0; i < block->seqs->len; ++i) { + if (g_array_index (block->seqs, guint16, i) == pkt->seqnum) { + g_array_index (block->states, TWCCPktState, i) = + _better_pkt_state (g_array_index (block->states, TWCCPktState, i), + pkt->state); + break; + } + } + const gsize packets_recovered = _redblock_reconsider (statsman, block); + GST_LOG_OBJECT (statsman->parent, "Reconsider block because of packet #%u, " + "recovered %lu pckt", pkt->seqnum, packets_recovered); + } + } +} + +/******************************************************************************/ + +TWCCStatsManager * +rtp_twcc_stats_manager_new (GObject *parent) +{ + TWCCStatsManager *statsman = g_new0 (TWCCStatsManager, 1); + statsman->parent = parent; + + statsman->stats_ctx = twcc_stats_ctx_new (); + statsman->ssrc_to_seqmap = g_hash_table_new_full (NULL, NULL, NULL, + (GDestroyNotify) g_hash_table_destroy); + statsman->sent_packets = gst_queue_array_new_for_struct (sizeof(SentPacket), + PACKETS_HIST_LEN_DEFAULT); + statsman->sent_packets_size = PACKETS_HIST_LEN_DEFAULT; + gst_queue_array_set_clear_func (statsman->sent_packets, (GDestroyNotify)_free_sentpacket); + statsman->sent_packets_feedbacks = gst_queue_array_new (300); + statsman->stats_ctx_first_seqnum = -1; + statsman->stats_ctx_by_pt = g_hash_table_new_full (NULL, NULL, + NULL, (GDestroyNotify) twcc_stats_ctx_free); + + statsman->prev_stat_window_beginning = GST_CLOCK_TIME_NONE; + + statsman->redund_2_redblocks = g_hash_table_new_full (_redund_hash, + _redund_equal, (GDestroyNotify)_redblock_key_free, + (GDestroyNotify)_redblock_free); + statsman->seqnum_2_redblocks = g_hash_table_new_full (g_direct_hash, + g_direct_equal, NULL, NULL); + + return statsman; +} + +void +rtp_twcc_stats_manager_free (TWCCStatsManager *statsman) +{ + g_hash_table_destroy (statsman->ssrc_to_seqmap); + gst_queue_array_free (statsman->sent_packets); + gst_queue_array_free (statsman->sent_packets_feedbacks); + g_hash_table_destroy (statsman->stats_ctx_by_pt); + g_hash_table_destroy (statsman->redund_2_redblocks); + g_hash_table_destroy (statsman->seqnum_2_redblocks); + twcc_stats_ctx_free (statsman->stats_ctx); + g_free (statsman); +} + +void rtp_twcc_stats_sent_pkt (TWCCStatsManager *statsman, + RTPPacketInfo * pinfo, GstRTPBuffer *rtp, guint16 twcc_seqnum) +{ + GstBuffer * buf = rtp->buffer; + SentPacket packet; + GArray *protect_seqnums_array = NULL; + guint32 protect_ssrc = 0; + gint redundant_pkt_idx = -1; + gint redundant_pkt_num = -1; + + /* In oreder to be able to map rtp seqnum to twcc seqnum in future, we + store it in certain hash tables (e.g. it might be needed to process + received feedback on a FEC packet) */ + _register_seqnum (statsman, pinfo->ssrc, + pinfo->seqnum, twcc_seqnum); + + /* If this packet is RTX/FEC packet, keep track of its meta */ + GstRTPRepairMeta *repair_meta = NULL; + if ((repair_meta = gst_buffer_get_rtp_repair_meta (buf)) != NULL) { + protect_ssrc = repair_meta->ssrc; + protect_seqnums_array = g_array_ref (repair_meta->seqnums); + redundant_pkt_idx = repair_meta->idx_red_packets; + redundant_pkt_num = repair_meta->num_red_packets; + } + + _sent_packet_init (&packet, twcc_seqnum, pinfo, rtp, + redundant_pkt_idx, redundant_pkt_num, + protect_ssrc, protect_seqnums_array); + /* Add packet to the sent_packets ring buffer and + make sure that it is within max_size, if not shrink by 1 pkt */ + _sent_pkt_keep_length (statsman, statsman->sent_packets_size, &packet); + + for (guint i = 0; protect_seqnums_array && i < protect_seqnums_array->len; i++) { + const guint16 prot_seqnum_ = g_array_index (protect_seqnums_array, guint16, i); + GST_DEBUG_OBJECT (statsman->parent, "%u protects seqnum: %u", twcc_seqnum, prot_seqnum_); + } + + GST_DEBUG_OBJECT + (statsman->parent, "Send: twcc-seqnum: %u, seqnum: %u, pt: %u, marker: %d, " + "redundant_idx: %d, redundant_num: %d, protected_seqnums: %u," + "size: %u, ts: %" + GST_TIME_FORMAT, packet.seqnum, pinfo->seqnum, packet.pt, pinfo->marker, + packet.redundant_idx, packet.redundant_num, + packet.protects_seqnums ? packet.protects_seqnums->len : 0, + packet.size, GST_TIME_ARGS (pinfo->current_time)); +} + +void rtp_twcc_stats_set_sock_ts (TWCCStatsManager *statsman, + guint16 seqnum, GstClockTime sock_ts) +{ + SentPacket * pkt = _find_sentpacket (statsman, seqnum); + if (pkt) { + pkt->socket_ts = sock_ts; + GST_LOG_OBJECT (statsman->parent, "packet #%u, setting socket-ts %" GST_TIME_FORMAT, + seqnum, GST_TIME_ARGS (sock_ts)); + } else { + GST_WARNING_OBJECT (statsman->parent, "Unable to update send-time for twcc-seqnum #%u", seqnum); + } +} + +void rtp_twcc_manager_tx_start_feedback (TWCCStatsManager *statsman) +{ + statsman->rtt = GST_CLOCK_TIME_NONE; +} + +void rtp_twcc_stats_pkt_feedback (TWCCStatsManager *statsman, + guint16 seqnum, GstClockTime remote_ts, GstClockTime current_time, + TWCCPktState status) +{ + SentPacket * found; + if (!!(found = _find_sentpacket (statsman, seqnum))) { + /* Do not process feedback on packets we have got feedback previously */ + if (found->state == RTP_TWCC_FECBLOCK_PKT_UNKNOWN + || found->state == RTP_TWCC_FECBLOCK_PKT_LOST) { + found->remote_ts = remote_ts; + found->state = status; + gst_queue_array_push_tail (statsman->sent_packets_feedbacks, found); + GST_LOG_OBJECT (statsman->parent, "matching pkt: #%u with local_ts: %" GST_TIME_FORMAT + " size: %u, remote-ts: %" GST_TIME_FORMAT, seqnum, + GST_TIME_ARGS (found->local_ts), + found->size * 8, GST_TIME_ARGS (remote_ts)); + + /* calculate the round-trip time */ + statsman->rtt = GST_CLOCK_DIFF (found->local_ts, current_time); + } else { + /* We've got feed back on the packet that was covered with the previous TWCC report. + Receiver could send two feedbacks on a single packet on purpose, + so we just ignore it. */ + GST_LOG_OBJECT (statsman->parent, "Rejecting second feedback on a packet #%u", seqnum); + } + } else { + GST_WARNING_OBJECT (statsman->parent, "Feedback on unknown packet #%u", seqnum); + } +} + +void rtp_twcc_manager_tx_end_feedback (TWCCStatsManager *statsman) +{ + if (GST_CLOCK_STIME_IS_VALID (statsman->rtt)) + statsman->avg_rtt = WEIGHT (statsman->rtt, statsman->avg_rtt, 0.1); +} + +GstStructure * +rtp_twcc_stats_do_stats (TWCCStatsManager *statsman, + GstClockTime stats_window_size, GstClockTime stats_window_delay) +{ + GstStructure *ret; + GValueArray *array; + GHashTableIter iter; + gpointer key; + gpointer value; + GstClockTimeDiff start_time; + GstClockTimeDiff end_time; + + while (!gst_queue_array_is_empty(statsman->sent_packets_feedbacks)) { + SentPacket * pkt = (SentPacket*)gst_queue_array_pop_head (statsman->sent_packets_feedbacks); + if (!pkt) { + continue; + } + _process_pkt_feedback (pkt, statsman); + } /* while fifo of arrays */ + + GstClockTime last_ts = twcc_stats_ctx_get_last_local_ts (statsman->stats_ctx); + if (!GST_CLOCK_TIME_IS_VALID (last_ts)) + return twcc_stats_ctx_get_structure (statsman->stats_ctx); + + /* Prune old packets in stats */ + gint last_seqnum_to_free = -1; + /* First remove all them from stats structures, and then from sent_packets + queue at once so as not to lock sent_packets for longer then necessary + */ + while (!gst_queue_array_is_empty (statsman->stats_ctx->pt_packets)) { + SentPacket * pkt = ((StatsPktPtr*)gst_queue_array_peek_head_struct ( + statsman->stats_ctx->pt_packets))->sentpkt; + if (gst_queue_array_get_length (statsman->stats_ctx->pt_packets) + >= MAX_STATS_PACKETS + || (pkt && GST_CLOCK_DIFF (pkt->local_ts, last_ts) > PACKETS_HIST_DUR)) { + if (pkt) { + if (last_seqnum_to_free >= 0 + && gst_rtp_buffer_compare_seqnum (pkt->seqnum, last_seqnum_to_free) + >= 0) { + GST_WARNING_OBJECT (statsman->parent, "Seqnum reorder in stats pkts"); + g_assert_not_reached (); + } + last_seqnum_to_free = pkt->seqnum; + } + _rm_last_stats_pkt (statsman); + } else { + break; + } + } + /* Remove old packets from sent_packets queue */ + if (last_seqnum_to_free >= 0) { + while (!gst_queue_array_is_empty (statsman->sent_packets)) { + SentPacket * pkt = gst_queue_array_peek_head_struct (statsman->sent_packets); + GST_LOG_OBJECT (statsman->parent, "Freeing sent packet #%u", pkt->seqnum); + if (gst_rtp_buffer_compare_seqnum (pkt->seqnum, last_seqnum_to_free) + >= 0) { + _free_sentpacket (pkt); + gst_queue_array_pop_head (statsman->sent_packets); + } else { + break; + } + } + } + + array = g_value_array_new (0); + end_time = GST_CLOCK_DIFF (stats_window_delay, last_ts); + start_time = end_time - stats_window_size; + + GST_DEBUG_OBJECT (statsman->parent, + "Calculating windowed stats for the window %" GST_STIME_FORMAT + " starting from %" GST_STIME_FORMAT " to: %" GST_STIME_FORMAT " overall packets: %u", + GST_STIME_ARGS (stats_window_size), GST_STIME_ARGS (start_time), + GST_STIME_ARGS (end_time), + gst_queue_array_get_length (statsman->stats_ctx->pt_packets)); + + if (!GST_CLOCK_TIME_IS_VALID(statsman->prev_stat_window_beginning) || + GST_CLOCK_DIFF (statsman->prev_stat_window_beginning, start_time) > 0) { + statsman->prev_stat_window_beginning = start_time; + } + + twcc_stats_ctx_calculate_windowed_stats (statsman->stats_ctx, start_time, + end_time); + ret = twcc_stats_ctx_get_structure (statsman->stats_ctx); + GST_LOG_OBJECT (statsman->parent, "Full stats: %" GST_PTR_FORMAT, ret); + + g_hash_table_iter_init (&iter, statsman->stats_ctx_by_pt); + while (g_hash_table_iter_next (&iter, &key, &value)) { + GstStructure *s; + guint pt = GPOINTER_TO_UINT (key); + TWCCStatsCtx *ctx = value; + twcc_stats_ctx_calculate_windowed_stats (ctx, start_time, end_time); + s = twcc_stats_ctx_get_structure (ctx); + gst_structure_set (s, "pt", G_TYPE_UINT, pt, NULL); + _append_structure_to_value_array (array, s); + GST_LOG_OBJECT (statsman->parent, "Stats for pt %u: %" GST_PTR_FORMAT, pt, s); + } + + _structure_take_value_array (ret, "payload-stats", array); + + return ret; +} + diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtptwccstats.h b/subprojects/gst-plugins-good/gst/rtpmanager/rtptwccstats.h new file mode 100644 index 0000000000..e166eab67a --- /dev/null +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtptwccstats.h @@ -0,0 +1,56 @@ +/* GStreamer + * Copyright (C) 2024 Pexip (http://pexip.com/) + * @author: Mikhail Baranov + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __RTP_TWCC_STATS_H__ +#define __RTP_TWCC_STATS_H__ + +#include +#include +#include "rtpstats.h" + +typedef enum { + RTP_TWCC_FECBLOCK_PKT_UNKNOWN, + RTP_TWCC_FECBLOCK_PKT_RECEIVED, + RTP_TWCC_FECBLOCK_PKT_RECOVERED, + RTP_TWCC_FECBLOCK_PKT_LOST +} TWCCPktState; + +struct _TWCCStatsManager; +typedef struct _TWCCStatsManager TWCCStatsManager; + +TWCCStatsManager *rtp_twcc_stats_manager_new (GObject *parent); +void rtp_twcc_stats_manager_free (TWCCStatsManager *stats_manager); + +void rtp_twcc_stats_sent_pkt (TWCCStatsManager *stats_manager, + RTPPacketInfo * pinfo, GstRTPBuffer *rtp, guint16 twcc_seqnum); + +void rtp_twcc_stats_set_sock_ts (TWCCStatsManager *stats_manager, + guint16 seqnum, GstClockTime sock_ts); + +void rtp_twcc_manager_tx_start_feedback (TWCCStatsManager *stats_manager); +void rtp_twcc_stats_pkt_feedback (TWCCStatsManager *stats_manager, + guint16 seqnum, GstClockTime remote_ts, GstClockTime current_time, + TWCCPktState status); +void rtp_twcc_manager_tx_end_feedback (TWCCStatsManager *stats_manager); + +GstStructure *rtp_twcc_stats_do_stats (TWCCStatsManager *stats_manager, + GstClockTime stats_window_size, GstClockTime stats_window_delay); + +#endif /* __RTP_TWCC_STATS_H__ */ \ No newline at end of file diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtprtx.c b/subprojects/gst-plugins-good/tests/check/elements/rtprtx.c index 0f7a5edb5b..44fc01c7f7 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtprtx.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtprtx.c @@ -21,8 +21,20 @@ #include #include #include +#include + +#define check_repairmeta(buf, ssrc_, seqnum_) \ + G_STMT_START { \ + GstRTPRepairMeta *repmeta = gst_buffer_get_rtp_repair_meta (buf); \ + fail_unless (repmeta != NULL); \ + fail_unless (repmeta->ssrc == (guint32)(ssrc_)); \ + fail_unless (repmeta->seqnums->len == 1); \ + fail_unless (g_array_index (repmeta->seqnums, guint16, 0) == (guint16)(seqnum_));\ + } G_STMT_END + -#define verify_buf(buf, is_rtx, expected_ssrc, expted_pt, expected_seqnum) \ +#define verify_buf(buf, is_rtx, expected_ssrc, expted_pt, expected_seqnum, \ + orig_ssrc, orig_seqnum) \ G_STMT_START { \ GstRTPBuffer _rtp = GST_RTP_BUFFER_INIT; \ fail_unless (gst_rtp_buffer_map (buf, GST_MAP_READ, &_rtp)); \ @@ -31,6 +43,10 @@ if (!(is_rtx)) { \ fail_unless_equals_int (gst_rtp_buffer_get_seq (&_rtp), expected_seqnum); \ } else { \ + /* Don't check rtx repair meta on stuffing */ \ + if ((gint64)(orig_ssrc)-1 >= -1 && (gint64)(orig_seqnum)-1 >= -1) { \ + check_repairmeta (buf, orig_ssrc, orig_seqnum); \ + } \ fail_unless_equals_int (GST_READ_UINT16_BE (gst_rtp_buffer_get_payload \ (&_rtp)), expected_seqnum); \ fail_unless (GST_BUFFER_FLAG_IS_SET (buf, \ @@ -39,17 +55,21 @@ gst_rtp_buffer_unmap (&_rtp); \ } G_STMT_END -#define pull_and_verify(h, is_rtx, expected_ssrc, expted_pt, expected_seqnum) \ +#define pull_and_verify(h, is_rtx, expected_ssrc, expted_pt, expected_seqnum, \ + orig_ssrc, orig_seqnum) \ G_STMT_START { \ GstBuffer *_buf = gst_harness_pull (h); \ - verify_buf (_buf, is_rtx, expected_ssrc, expted_pt, expected_seqnum); \ + verify_buf (_buf, is_rtx, expected_ssrc, expted_pt, expected_seqnum, \ + orig_ssrc, orig_seqnum); \ gst_buffer_unref (_buf); \ } G_STMT_END -#define push_pull_and_verify(h, buf, is_rtx, expected_ssrc, expted_pt, expected_seqnum) \ +#define push_pull_and_verify(h, buf, is_rtx, expected_ssrc, expted_pt, expected_seqnum, \ + orig_ssrc, orig_seqnum) \ G_STMT_START { \ gst_harness_push (h, buf); \ - pull_and_verify (h, is_rtx, expected_ssrc, expted_pt, expected_seqnum); \ + pull_and_verify (h, is_rtx, expected_ssrc, expted_pt, expected_seqnum, \ + orig_ssrc, orig_seqnum); \ } G_STMT_END static GstEvent * @@ -254,13 +274,13 @@ GST_START_TEST (test_rtxsend_basic) gst_harness_push (h, create_rtp_buffer (main_ssrc, main_pt, 0))); /* and check it came through */ - pull_and_verify (h, FALSE, main_ssrc, main_pt, 0); + pull_and_verify (h, FALSE, main_ssrc, main_pt, 0, 0, 0); /* now request this packet as rtx */ gst_harness_push_upstream_event (h, create_rtx_event (main_ssrc, main_pt, 0)); /* and verify we got an rtx-packet for it */ - pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 0); + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 0, main_ssrc, 0); gst_structure_free (ssrc_map); gst_structure_free (pt_map); @@ -293,7 +313,7 @@ GST_START_TEST (test_rtxsend_disabled_enabled_disabled) /* push, pull, request-rtx, verify nothing arrives */ fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, create_rtp_buffer (main_ssrc, main_pt, 0))); - pull_and_verify (h, FALSE, main_ssrc, main_pt, 0); + pull_and_verify (h, FALSE, main_ssrc, main_pt, 0, 0, 0); gst_harness_push_upstream_event (h, create_rtx_event (main_ssrc, main_pt, 0)); fail_unless_equals_int (0, gst_harness_buffers_in_queue (h)); /* verify there is no task on the rtxsend srcpad */ @@ -305,9 +325,9 @@ GST_START_TEST (test_rtxsend_disabled_enabled_disabled) /* push, pull, request rtx, pull rtx */ fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, create_rtp_buffer (main_ssrc, main_pt, 1))); - pull_and_verify (h, FALSE, main_ssrc, main_pt, 1); + pull_and_verify (h, FALSE, main_ssrc, main_pt, 1, 0, 0); gst_harness_push_upstream_event (h, create_rtx_event (main_ssrc, main_pt, 1)); - pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 1); + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 1, main_ssrc, 1); /* verify there is a task on the rtxsend srcpad */ fail_unless (GST_PAD_TASK (GST_PAD_PEER (h->sinkpad)) != NULL); @@ -317,7 +337,7 @@ GST_START_TEST (test_rtxsend_disabled_enabled_disabled) /* push, pull, request-rtx, verify nothing arrives */ fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, create_rtp_buffer (main_ssrc, main_pt, 2))); - pull_and_verify (h, FALSE, main_ssrc, main_pt, 2); + pull_and_verify (h, FALSE, main_ssrc, main_pt, 2, 0, 0); gst_harness_push_upstream_event (h, create_rtx_event (main_ssrc, main_pt, 2)); fail_unless_equals_int (0, gst_harness_buffers_in_queue (h)); /* verify the task is gone again */ @@ -426,7 +446,7 @@ GST_START_TEST (test_rtxsend_rtxreceive) inbufs[i] = create_rtp_buffer (master_ssrc, master_pt, 100 + i); gst_harness_push (hsend, gst_buffer_ref (inbufs[i])); gst_harness_push (hrecv, gst_harness_pull (hsend)); - pull_and_verify (hrecv, FALSE, master_ssrc, master_pt, 100 + i); + pull_and_verify (hrecv, FALSE, master_ssrc, master_pt, 100 + i, 0, 0); } /* Getting rid of reconfigure event. Preparation before the next step */ @@ -438,12 +458,14 @@ GST_START_TEST (test_rtxsend_rtxreceive) check that the packet produced out of RTX packet is the same as an original packet */ for (i = 0; i < packets_num; i++) { - GstBuffer *outbuf; + GstBuffer *outbuf, *sentbuf; gst_harness_push_upstream_event (hrecv, create_rtx_event (master_ssrc, master_pt, 100 + i)); gst_harness_push_upstream_event (hsend, gst_harness_pull_upstream_event (hrecv)); - gst_harness_push (hrecv, gst_harness_pull (hsend)); + sentbuf = gst_harness_pull (hsend); + check_repairmeta (sentbuf, master_ssrc, 100 + i); + gst_harness_push (hrecv, sentbuf); outbuf = gst_harness_pull (hrecv); compare_rtp_packets (inbufs[i], outbuf); @@ -512,7 +534,7 @@ GST_START_TEST (test_rtxsend_rtxreceive_with_packet_loss) through rtxreceive to rtxsend, and verify the packet was retransmitted */ for (drop_nth_packet = 2; drop_nth_packet < 10; drop_nth_packet++) { for (i = 0; i < packets_num; i++, seqnum++) { - GstBuffer *outbuf; + GstBuffer *outbuf, *sentbuf; GstBuffer *inbuf = create_rtp_buffer (master_ssrc, master_pt, seqnum); gboolean drop_this_packet = ((i + 1) % drop_nth_packet) == 0; @@ -526,7 +548,9 @@ GST_START_TEST (test_rtxsend_rtxreceive_with_packet_loss) gst_harness_push_upstream_event (hsend, gst_harness_pull_upstream_event (hrecv)); /* Pushing RTX packet to rtxreceive */ - gst_harness_push (hrecv, gst_harness_pull (hsend)); + sentbuf = gst_harness_pull (hsend); + check_repairmeta (sentbuf, master_ssrc, seqnum); + gst_harness_push (hrecv, sentbuf); expected_rtx_packets++; } else { gst_harness_push (hrecv, gst_harness_pull (hsend)); @@ -682,7 +706,7 @@ GST_START_TEST (test_multi_rtxsend_rtxreceive_with_packet_loss) for (i = 0; i < total_pakets_num; i++) { RtxSender *sender = &senders[i % senders_num]; gboolean drop_this_packet = ((i + 1) % drop_nth_packet) == 0; - GstBuffer *outbuf, *inbuf; + GstBuffer *outbuf, *inbuf, *sentbuf; inbuf = create_rtp_buffer (sender->master_ssrc, sender->master_pt, sender->seqnum); @@ -706,7 +730,9 @@ GST_START_TEST (test_multi_rtxsend_rtxreceive_with_packet_loss) gst_event_unref (rtxevent); /* Pushing RTX packet to rtxreceive */ - gst_harness_push (hrecv, gst_harness_pull (sender->h)); + sentbuf = gst_harness_pull (sender->h); + check_repairmeta (sentbuf, sender->master_ssrc, sender->seqnum); + gst_harness_push (hrecv, sentbuf); sender->expected_rtx_packets++; total_dropped_packets++; } else { @@ -809,7 +835,8 @@ test_rtxsender_packet_retention (gboolean test_with_time, /* Pull only the ones supposed to be retransmitted */ if (j >= i - half_buffers) - pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, rtx_seqnum); + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, rtx_seqnum, master_ssrc, + rtx_seqnum); } /* Check there no extra buffers in the harness queue */ fail_unless_equals_int (gst_harness_buffers_in_queue (h), 0); @@ -818,7 +845,7 @@ test_rtxsender_packet_retention (gboolean test_with_time, to be sure, rtprtxsend can handle it properly */ push_pull_and_verify (h, create_rtp_buffer_with_timestamp (master_ssrc, master_pt, 0x100 + i, - timestamp, pts), FALSE, master_ssrc, master_pt, 0x100 + i); + timestamp, pts), FALSE, master_ssrc, master_pt, 0x100 + i, 0, 0); } gst_structure_free (pt_map); @@ -897,14 +924,14 @@ test_rtxqueue_packet_retention (gboolean test_with_time) for (j = 0; j < i; j++) { guint rtx_seqnum = 0x100 + j; if (j >= i - half_buffers) - pull_and_verify (h, FALSE, ssrc, pt, rtx_seqnum); + pull_and_verify (h, FALSE, ssrc, pt, rtx_seqnum, 0, 0); } /* There should be only one packet remaining in the queue now */ fail_unless_equals_int (gst_harness_buffers_in_queue (h), 1); /* pull the one that we just pushed (comes after the retransmitted ones) */ - pull_and_verify (h, FALSE, ssrc, pt, 0x100 + i); + pull_and_verify (h, FALSE, ssrc, pt, 0x100 + i, 0, 0); /* Check there no extra buffers in the harness queue */ fail_unless_equals_int (gst_harness_buffers_in_queue (h), 0); @@ -1042,7 +1069,7 @@ GST_START_TEST (test_rtxsend_header_extensions_copy) gst_harness_push (hsend, gst_buffer_ref (inbufs[i])); gst_harness_push (hrecv, gst_harness_pull (hsend)); - pull_and_verify (hrecv, FALSE, master_ssrc, master_pt, 100 + i); + pull_and_verify (hrecv, FALSE, master_ssrc, master_pt, 100 + i, 0, 0); } gst_clear_object (&twcc); @@ -1123,13 +1150,13 @@ GST_START_TEST (test_rtxsender_copy_twcc_exthdr) gst_harness_push (h, create_rtp_buffer_with_twcc_ext (master_ssrc, master_pt, 0, 20, twcc_ext_id, 33)); - pull_and_verify (h, FALSE, master_ssrc, master_pt, 0); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 0, 0, 0); gst_harness_push_upstream_event (h, create_rtx_event (master_ssrc, master_pt, 0)); rtx_buf = gst_harness_pull (h); - verify_buf (rtx_buf, TRUE, rtx_ssrc, rtx_pt, 0); + verify_buf (rtx_buf, TRUE, rtx_ssrc, rtx_pt, 0, master_ssrc, 0); fail_unless (read_twcc_seqnum (rtx_buf, twcc_ext_id) == 33); gst_buffer_unref (rtx_buf); @@ -1246,7 +1273,7 @@ GST_START_TEST (test_rtxsender_copy_roi_exthdr) /* push packets through rtxsend to rtxreceive */ gst_harness_push (hsend, gst_buffer_ref (inbuf)); gst_harness_push (hrecv, gst_harness_pull (hsend)); - pull_and_verify (hrecv, FALSE, master_ssrc, master_pt, 0); + pull_and_verify (hrecv, FALSE, master_ssrc, master_pt, 0, 0, 0); /* get rid of reconfigure event in preparation for next step */ gst_event_unref (gst_harness_pull_upstream_event (hrecv)); @@ -1259,7 +1286,7 @@ GST_START_TEST (test_rtxsender_copy_roi_exthdr) gst_harness_pull_upstream_event (hrecv)); rtxbuf = gst_harness_pull (hsend); - verify_buf (rtxbuf, TRUE, rtx_ssrc, rtx_pt, 0); + verify_buf (rtxbuf, TRUE, rtx_ssrc, rtx_pt, 0, master_ssrc, 0); read_roi (rtxbuf, ext_id, &actual_x, &actual_y, &actual_w, &actual_h, &actual_id, &actual_num_faces); fail_unless_equals_int (expected_x, actual_x); @@ -1357,7 +1384,7 @@ GST_START_TEST (test_rtxsend_header_extensions) gst_rtp_buffer_unmap (&rtp); gst_harness_push (hsend, gst_buffer_ref (inbufs[i])); gst_harness_push (hrecv, gst_harness_pull (hsend)); - pull_and_verify (hrecv, FALSE, master_ssrc, master_pt, 100 + i); + pull_and_verify (hrecv, FALSE, master_ssrc, master_pt, 100 + i, 0, 0); } /* Getting rid of reconfigure event. Preparation before the next step */ @@ -1435,31 +1462,31 @@ GST_START_TEST (test_rtxsender_stuffing) gst_harness_set_time (h, 0 * GST_MSECOND); gst_harness_push (h, create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 0, 20)); - pull_and_verify (h, FALSE, master_ssrc, master_pt, 0); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 0, 0, 0); gst_harness_set_time (h, 10 * GST_MSECOND); gst_harness_push (h, create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 1, 580)); - pull_and_verify (h, FALSE, master_ssrc, master_pt, 1); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 1, 0, 0); /* budget 1000, sent 600, no stuffing (stuff with 580 will exceed budget) */ gst_harness_set_time (h, 20 * GST_MSECOND); gst_harness_push (h, create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 2, 600)); - pull_and_verify (h, FALSE, master_ssrc, master_pt, 2); - pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 2); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 2, 0, 0); + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 2, -1, -1); gst_harness_set_time (h, 30 * GST_MSECOND); gst_harness_push (h, create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 3, 200)); - pull_and_verify (h, FALSE, master_ssrc, master_pt, 3); - pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 3); - pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 3); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 3, 0, 0); + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 3, -1, -1); + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 3, -1, -1); gst_harness_set_time (h, 40 * GST_MSECOND); gst_harness_push (h, create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 4, 600)); - pull_and_verify (h, FALSE, master_ssrc, master_pt, 4); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 4, 0, 0); g_usleep (G_USEC_PER_SEC / 100); fail_if (gst_harness_try_pull (h)); @@ -1497,17 +1524,17 @@ GST_START_TEST (test_rtxsender_stuffing_toggle) gst_harness_set_time (h, 0 * GST_MSECOND); gst_harness_push (h, create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 0, 20)); - pull_and_verify (h, FALSE, master_ssrc, master_pt, 0); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 0, 0, 0); gst_harness_set_time (h, 10 * GST_MSECOND); gst_harness_push (h, create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 1, 20)); - pull_and_verify (h, FALSE, master_ssrc, master_pt, 1); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 1, 0, 0); gst_harness_set_time (h, 20 * GST_MSECOND); gst_harness_push (h, create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 2, 20)); - pull_and_verify (h, FALSE, master_ssrc, master_pt, 2); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 2, 0, 0); /* 80 kbps = 10 kBps. 10 ms distance between each packet means 10 packets * per second and 100 bytes per packet to hit target kbps. @@ -1517,17 +1544,17 @@ GST_START_TEST (test_rtxsender_stuffing_toggle) gst_harness_set_time (h, 30 * GST_MSECOND); gst_harness_push (h, create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 3, 20)); - pull_and_verify (h, FALSE, master_ssrc, master_pt, 3); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 3, 0, 0); gst_harness_set_time (h, 40 * GST_MSECOND); gst_harness_push (h, create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 4, 20)); - pull_and_verify (h, FALSE, master_ssrc, master_pt, 4); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 4, 0, 0); /* budget 100, sent 40, so stuff with #2, 3 and 4, total of 60 bytes */ - pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 2); - pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 3); - pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 4); + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 2, -1, -1); + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 3, -1, -1); + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 4, -1, -1); /* disable stuffing */ g_object_set (h->element, "stuffing-kbps", 0, NULL); @@ -1535,7 +1562,7 @@ GST_START_TEST (test_rtxsender_stuffing_toggle) gst_harness_set_time (h, 50 * GST_MSECOND); gst_harness_push (h, create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 5, 20)); - pull_and_verify (h, FALSE, master_ssrc, master_pt, 5); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 5, 0, 0); g_usleep (G_USEC_PER_SEC / 100); fail_if (gst_harness_try_pull (h)); @@ -1579,32 +1606,32 @@ GST_START_TEST (test_rtxsender_stuffing_non_rtx_packets) gst_harness_set_time (h, 0 * GST_MSECOND); gst_harness_push (h, create_rtp_buffer_with_payload_size (video_ssrc, video_pt, 0, 20)); - pull_and_verify (h, FALSE, video_ssrc, video_pt, 0); + pull_and_verify (h, FALSE, video_ssrc, video_pt, 0, 0, 0); gst_harness_set_time (h, 10 * GST_MSECOND); gst_harness_push (h, create_rtp_buffer_with_payload_size (video_ssrc, video_pt, 1, 580)); - pull_and_verify (h, FALSE, video_ssrc, video_pt, 1); + pull_and_verify (h, FALSE, video_ssrc, video_pt, 1, 0, 0); /* budget 1000, sent 600, no stuffing (stuff with 580 will exceed budget) */ gst_harness_set_time (h, 20 * GST_MSECOND); gst_harness_push (h, create_rtp_buffer_with_payload_size (video_ssrc, video_pt, 2, 600)); - pull_and_verify (h, FALSE, video_ssrc, video_pt, 2); + pull_and_verify (h, FALSE, video_ssrc, video_pt, 2, 0, 0); /* budget 2000, sent 1200, stuff with #2, 600 bytes */ - pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 2); + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 2, -1, -1); gst_harness_set_time (h, 30 * GST_MSECOND); /* audio packet comes in.. */ gst_harness_push (h, create_rtp_buffer_with_payload_size (audio_ssrc, audio_pt, 0, 200)); - pull_and_verify (h, FALSE, audio_ssrc, audio_pt, 0); + pull_and_verify (h, FALSE, audio_ssrc, audio_pt, 0, 0, 0); /* budget 3000, sent 2000, stuff with #2, 600 bytes */ - pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 2); + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 2, -1, -1); gst_harness_push (h, create_rtp_buffer_with_payload_size (video_ssrc, video_pt, 4, 600)); - pull_and_verify (h, FALSE, video_ssrc, video_pt, 4); + pull_and_verify (h, FALSE, video_ssrc, video_pt, 4, 0, 0); /* budget 4000, sent 2600, no more stuffing */ g_usleep (G_USEC_PER_SEC / 100); fail_if (gst_harness_try_pull (h)); @@ -1648,7 +1675,7 @@ GST_START_TEST (test_rtxsender_stuffing_window) GST_BUFFER_PTS (buf) = 0; gst_harness_push (h, buf); - pull_and_verify (h, FALSE, master_ssrc, master_pt, 0); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 0, 0, 0); /* budget 100, sent 80 */ gst_harness_set_time (h, 10 * GST_MSECOND); @@ -1656,18 +1683,18 @@ GST_START_TEST (test_rtxsender_stuffing_window) GST_BUFFER_PTS (buf) = 10 * GST_MSECOND; gst_harness_push (h, buf); - pull_and_verify (h, FALSE, master_ssrc, master_pt, 1); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 1, 0, 0); /* budget 200, sent 160, no stuffing */ gst_harness_set_time (h, 110 * GST_MSECOND); buf = create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 2, 80); GST_BUFFER_PTS (buf) = 110 * GST_MSECOND; gst_harness_push (h, buf); - pull_and_verify (h, FALSE, master_ssrc, master_pt, 2); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 2, 0, 0); /* we have budget, so stuff with #1 and #2, but #0 is too old */ - pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 1); - pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 2); + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 1, -1, -1); + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 2, -1, -1); g_usleep (G_USEC_PER_SEC / 100); fail_if (gst_harness_try_pull (h)); @@ -1712,14 +1739,14 @@ GST_START_TEST (test_rtxsender_stuffing_no_packets_under_window) buf = create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 0, 80); GST_BUFFER_PTS (buf) = 0; gst_harness_push (h, buf); - pull_and_verify (h, FALSE, master_ssrc, master_pt, 0); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 0, 0, 0); /* budget 100, sent 80 */ gst_harness_set_time (h, 10 * GST_MSECOND); buf = create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 1, 80); GST_BUFFER_PTS (buf) = 10 * GST_MSECOND; gst_harness_push (h, buf); - pull_and_verify (h, FALSE, master_ssrc, master_pt, 1); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 1, 0, 0); /* budget 200, sent 160, no stuffing */ /* advance far ahead, so the buffers are out of the window */ @@ -1727,12 +1754,12 @@ GST_START_TEST (test_rtxsender_stuffing_no_packets_under_window) buf = create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 2, 80); GST_BUFFER_PTS (buf) = 20 * GST_MSECOND; gst_harness_push (h, buf); - pull_and_verify (h, FALSE, master_ssrc, master_pt, 2); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 2, 0, 0); /* we have budget, non of the above are under the window, so we stuff repeatedly with the latest buffer */ for (i = 0; i < 4; i++) { - pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 2); + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 2, -1, -1); } g_usleep (G_USEC_PER_SEC / 100); @@ -1773,7 +1800,7 @@ GST_START_TEST (test_rtxsender_stuffing_sanity_when_input_rate_is_extreme) gst_harness_push (h, create_rtp_buffer_with_payload_size (master_ssrc, master_pt, i, 1000000)); - pull_and_verify (h, FALSE, master_ssrc, master_pt, i); + pull_and_verify (h, FALSE, master_ssrc, master_pt, i, 0, 0); } fail_if (gst_harness_try_pull (h)); @@ -1809,24 +1836,24 @@ GST_START_TEST (test_rtxsender_stuffing_does_not_interfer_with_rtx) gst_harness_set_time (h, 0 * GST_MSECOND); gst_harness_push (h, create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 0, 100)); - pull_and_verify (h, FALSE, master_ssrc, master_pt, 0); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 0, 0, 0); gst_harness_set_time (h, 10 * GST_MSECOND); gst_harness_push (h, create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 1, 500)); - pull_and_verify (h, FALSE, master_ssrc, master_pt, 1); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 1, 0, 0); /* budget 1000, sent 600 */ /* Request and send RTX packet. Not calculated in budget */ gst_harness_push_upstream_event (h, create_rtx_event (master_ssrc, master_pt, 1)); - pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 1); + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 1, master_ssrc, 1); /* Send media packet. Budget should still be untouched for before. */ gst_harness_set_time (h, 20 * GST_MSECOND); gst_harness_push (h, create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 2, 400)); - pull_and_verify (h, FALSE, master_ssrc, master_pt, 2); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 2, 0, 0); gst_structure_free (pt_map); gst_structure_free (ssrc_map); @@ -1866,7 +1893,7 @@ GST_START_TEST (test_rtxsender_stuffing_max_burst_packets) /* push only a very small buffer to force the creation of a lot of stuffing */ gst_harness_push (h, create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 0, 20)); - pull_and_verify (h, FALSE, master_ssrc, master_pt, 0); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 0, master_ssrc, 0); /* set the time 1 second in the future, to create a huge deficit on stuffing */ gst_harness_set_time (h, 1 * GST_SECOND); diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c b/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c index 26856f930f..7477d74d80 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c @@ -32,6 +32,7 @@ #include #include +#include #include #include #include @@ -73,7 +74,9 @@ static GstBuffer * generate_test_buffer_full (GstClockTime ts, guint seqnum, guint32 rtp_ts, guint ssrc, gboolean marker_bit, guint8 payload_type, guint8 twcc_ext_id, - guint16 twcc_seqnum) + guint16 twcc_seqnum, + gint idx, gint rpkts_num, + guint32 protects_ssrc, guint16 * protects_seqnums, guint8 protects_len) { GstBuffer *buf; guint8 *payload; @@ -103,6 +106,8 @@ generate_test_buffer_full (GstClockTime ts, } gst_rtp_buffer_unmap (&rtp); + gst_buffer_add_rtp_repair_meta (buf, idx, rpkts_num, + protects_ssrc, protects_seqnums, protects_len); return buf; } @@ -111,7 +116,8 @@ static GstBuffer * generate_test_buffer (guint seqnum, guint ssrc) { return generate_test_buffer_full (seqnum * TEST_BUF_DURATION, - seqnum, seqnum * TEST_RTP_TS_DURATION, ssrc, FALSE, TEST_BUF_PT, 0, 0); + seqnum, seqnum * TEST_RTP_TS_DURATION, ssrc, FALSE, TEST_BUF_PT, 0, 0, + -1, -1, 0, NULL, 0); } static GstBuffer * @@ -120,7 +126,7 @@ generate_twcc_recv_buffer (guint seqnum, { return generate_test_buffer_full (arrival_time, seqnum, seqnum * TEST_RTP_TS_DURATION, TEST_BUF_SSRC, marker_bit, TEST_BUF_PT, - TEST_TWCC_EXT_ID, seqnum); + TEST_TWCC_EXT_ID, seqnum, -1, -1, 0, NULL, 0); } static GstBuffer * @@ -129,7 +135,7 @@ generate_twcc_send_buffer_full (guint seqnum, gboolean marker_bit, { return generate_test_buffer_full (seqnum * TEST_BUF_DURATION, seqnum, seqnum * TEST_RTP_TS_DURATION, ssrc, marker_bit, - payload_type, 0, 0); + payload_type, 0, 0, -1, -1, 0, NULL, 0); } static GstBuffer * @@ -146,6 +152,8 @@ generate_rtx_buffer (guint rtx_seqnum, GstBuffer * buffer) GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; GstRTPBuffer new_rtp = GST_RTP_BUFFER_INIT; GstBuffer *new_buffer = gst_buffer_new (); + guint32 orig_ssrc; + guint16 orig_seqnum; GstMapInfo map; gst_rtp_buffer_map (buffer, GST_MAP_READ, &rtp); @@ -173,6 +181,8 @@ generate_rtx_buffer (guint rtx_seqnum, GstBuffer * buffer) memcpy (map.data + 2, rtp.data[2], rtp.size[2]); gst_memory_unmap (mem, &map); gst_buffer_append_memory (new_buffer, mem); + orig_ssrc = gst_rtp_buffer_get_ssrc (&rtp); + orig_seqnum = gst_rtp_buffer_get_seq (&rtp); /* everything needed is copied */ gst_rtp_buffer_unmap (&rtp); @@ -185,6 +195,7 @@ generate_rtx_buffer (guint rtx_seqnum, GstBuffer * buffer) /* Copy over timestamps */ gst_buffer_copy_into (new_buffer, buffer, GST_BUFFER_COPY_TIMESTAMPS, 0, -1); + gst_buffer_add_rtp_repair_meta (new_buffer, 0, 1, orig_ssrc, &orig_seqnum, 1); /* mark this is a RETRANSMISSION buffer */ GST_BUFFER_FLAG_SET (new_buffer, GST_RTP_BUFFER_FLAG_RETRANSMISSION); @@ -222,7 +233,8 @@ static GstBuffer * generate_test_buffer_timed (GstClockTime ts, guint seqnum, guint32 rtp_ts) { return generate_test_buffer_full (ts, - seqnum, rtp_ts, TEST_BUF_SSRC, FALSE, TEST_BUF_PT, 0, 0); + seqnum, rtp_ts, TEST_BUF_SSRC, FALSE, TEST_BUF_PT, 0, 0, + -1, -1, 0, NULL, 0); } typedef struct @@ -496,25 +508,6 @@ session_harness_set_twcc_recv_ext_id (SessionHarness * h, guint8 ext_id) g_signal_emit_by_name (h->session, "clear-pt-map"); } -static GstStructure * -create_rtx_map (const gchar * name, guint key, guint value) -{ - gchar *key_str = g_strdup_printf ("%u", key); - GstStructure *s = gst_structure_new (name, - key_str, G_TYPE_UINT, (guint) value, NULL); - g_free (key_str); - return s; -} - -static void -session_harness_enable_rtx (SessionHarness * h) -{ - GstStructure *rtx_map = - create_rtx_map ("rtx-map", TEST_BUF_SSRC, TEST_RTX_BUF_SSRC); - g_object_set (h->internal_session, "rtx-ssrc-map", rtx_map, NULL); - gst_structure_free (rtx_map); -} - static void session_harness_add_caps_for_pt (SessionHarness * h, GstCaps * caps, guint8 pt) { @@ -3195,7 +3188,7 @@ generate_stepped_ts_buffer (guint i, gboolean stepped) TEST_BUF_CLOCK_RATE)), i); buf = generate_test_buffer_full (i * GST_MSECOND, i, ts, 0xAAAA, FALSE, - TEST_BUF_PT, 0, 0); + TEST_BUF_PT, 0, 0, -1, -1, 0, NULL, 0); return buf; } @@ -4945,7 +4938,8 @@ GST_START_TEST (test_twcc_packet_event) { SessionHarness *send_h = session_harness_new (); gboolean enabled_twcc_event = __i__; - g_object_set (send_h->session, "enable-twcc-packet-event", enabled_twcc_event, NULL); + g_object_set (send_h->session, "enable-twcc-packet-event", enabled_twcc_event, + NULL); SessionHarness *recv_h = session_harness_new (); GstBuffer *buf; GstEvent *event; @@ -5284,9 +5278,6 @@ construct_initial_state_for_rtx (SessionHarness * h_send, guint window_size_ms = 300; guint num_buffers = window_size_ms / TEST_BUF_MS + 1; - session_harness_enable_rtx (h_send); - session_harness_enable_rtx (h_recv); - /* send and recv enough packets to be over the stats window */ for (i = 0; i < num_buffers; i++) { GstFlowReturn ret; @@ -5310,8 +5301,8 @@ construct_initial_state_for_rtx (SessionHarness * h_send, return i; } -static void -fail_unless_twcc_stats_recovery (SessionHarness * h, gdouble recovery_pct) +static gdouble +get_recovery_pct (SessionHarness * h) { gdouble stats_recovery_pct; GstStructure *twcc_stats; @@ -5321,9 +5312,15 @@ fail_unless_twcc_stats_recovery (SessionHarness * h, gdouble recovery_pct) fail_unless (gst_structure_get (twcc_stats, "recovery-pct", G_TYPE_DOUBLE, &stats_recovery_pct, NULL)); - fail_unless_equals_float (recovery_pct, stats_recovery_pct); gst_structure_free (twcc_stats); + return stats_recovery_pct; +} + +static void +fail_unless_twcc_stats_recovery (SessionHarness * h, gdouble recovery_pct) +{ + fail_unless_equals_float (recovery_pct, get_recovery_pct (h)); } static void @@ -5402,6 +5399,153 @@ GST_START_TEST (test_twcc_stats_no_rtx_no_recover) GST_END_TEST; +GST_START_TEST (test_twcc_stats_long_rtx_recover) +{ + gint loss_pct = 50; + guint nframes = 100; + guint nframes_to_skip = 10; + guint pkt_in_frame = 10; + + GRand *rnd = g_rand_new_with_seed (101); + + SessionHarness *h_send = session_harness_new (); + SessionHarness *h_recv = session_harness_new (); + guint next_seqnum; + + session_harness_add_twcc_caps_for_pt (h_send, TEST_BUF_PT); + session_harness_add_twcc_caps_for_pt (h_send, TEST_RTX_BUF_PT); + session_harness_set_twcc_recv_ext_id (h_recv, TEST_TWCC_EXT_ID); + + next_seqnum = construct_initial_state_for_rtx (h_send, h_recv); + + for (guint nframe = 0; nframe < nframes; nframe++) { + for (guint i = 0; i < pkt_in_frame; i++) { + GstBuffer *buf; + GstBuffer *rtx_buf; + + const gboolean lost = g_rand_int_range (rnd, 0, 100) < loss_pct; + buf = generate_twcc_send_buffer (next_seqnum++, FALSE); + rtx_buf = generate_rtx_buffer (i, buf); + send_recv_buffer (h_send, h_recv, buf, !lost); + if (lost) { + // const gboolean rtx_lost = g_rand_int_range (rnd, 0, 100) < loss_pct; + const gboolean rtx_lost = FALSE; + + /* we send a buffer but receiver doesn't get it */ + send_recv_buffer (h_send, h_recv, rtx_buf, !rtx_lost); + } else { + gst_buffer_unref (rtx_buf); + } + } + /* push a last buffer with the marker bit to trigger the report */ + send_recv_buffer (h_send, h_recv, + generate_twcc_send_buffer (next_seqnum++, TRUE), TRUE); + + fail_unless_equals_int64 (GST_FLOW_OK, + session_harness_recv_rtcp (h_send, + session_harness_produce_twcc (h_recv))); + + if (nframe > nframes_to_skip) { + const gdouble recovery_stats = get_recovery_pct (h_send); + fail_if (recovery_stats != 100. && recovery_stats != -1.); + } else { + const gdouble recovery_stats = get_recovery_pct (h_send); + GST_ERROR ("Frame %d: recovery stats: %f", nframe, recovery_stats); + } + } + + g_rand_free (rnd); + session_harness_free (h_send); + session_harness_free (h_recv); +} + +GST_END_TEST; + +GST_START_TEST (test_twcc_stats_block_fec_recover) +{ + SessionHarness *h_send = session_harness_new (); + SessionHarness *h_recv = session_harness_new (); + guint i, next_seqnum; + const guint window_size_ms = 400; + const guint num_buffers = window_size_ms / TEST_BUF_MS + 1; + + guint16 fec_seqnum = 6666; + gsize fec_num = 0; + const guint32 fec_ssrc = 0x12345678; + const guint8 fec_payload_type = 127; + const gsize block_len = 7; + const gsize block_fecs = 2; + guint16 *protects_seqnums = g_malloc0 (sizeof (guint16) * block_len); + gsize protects_seqnums_i = 0; + + + session_harness_add_twcc_caps_for_pt (h_send, TEST_BUF_PT); + session_harness_add_twcc_caps_for_pt (h_send, fec_payload_type); + session_harness_set_twcc_recv_ext_id (h_recv, TEST_TWCC_EXT_ID); + + /* send and recv enough packets to be over the stats window */ + for (next_seqnum = 0; next_seqnum < num_buffers; next_seqnum++) { + GstFlowReturn ret; + GstBuffer *buf; + const gboolean is_last = (next_seqnum == num_buffers - 1); + + buf = generate_twcc_send_buffer (next_seqnum, FALSE); + ret = session_harness_send_rtp (h_send, buf); + fail_unless_equals_int (ret, GST_FLOW_OK); + session_harness_advance_and_crank (h_send, TEST_BUF_DURATION); + + buf = session_harness_pull_send_rtp (h_send); + // Lose first data packets in the block up to number of fec packets. + if (is_last + || next_seqnum % block_len == 0 + || next_seqnum % block_len > block_fecs) { + ret = session_harness_recv_rtp (h_recv, buf); + fail_unless_equals_int (ret, GST_FLOW_OK); + } else { + gst_buffer_unref (buf); + } + if (next_seqnum % block_len == (block_fecs + 1)) { + session_harness_advance_and_crank (h_recv, + block_fecs * TEST_BUF_DURATION); + } else if (next_seqnum % block_len > (block_fecs + 1)) { + session_harness_advance_and_crank (h_recv, TEST_BUF_DURATION); + } + + protects_seqnums[protects_seqnums_i] = next_seqnum; + protects_seqnums_i += 1; + if (protects_seqnums_i >= block_len) { + protects_seqnums_i = 0; + } + // Generate FEC packets for the block + if (next_seqnum % block_len == (block_len - 1)) { + for (i = 0; i < block_fecs; i++) { + const gboolean is_fec_last = is_last && (i == block_fecs - 1); + buf = generate_test_buffer_full (fec_num * TEST_BUF_DURATION, + fec_seqnum, fec_num * TEST_RTP_TS_DURATION, fec_ssrc, is_fec_last, + fec_payload_type, 0, 0, i, block_fecs, TEST_BUF_SSRC, + protects_seqnums, block_len); + send_recv_buffer (h_send, h_recv, buf, TRUE); + + fec_num++; + fec_seqnum++; + } + } + + } + + /* produce a twcc feedback to process those packets */ + session_harness_recv_rtcp (h_send, session_harness_produce_twcc (h_recv)); + + fail_unless_twcc_stats_recovery (h_send, 100.); + + session_harness_free (h_send); + session_harness_free (h_recv); + g_free (protects_seqnums); +} + +GST_END_TEST; + + GST_START_TEST (test_twcc_feedback_max_sent_packets) { /* test is very intense for valgrind or debug build */ @@ -5521,7 +5665,7 @@ GST_START_TEST (test_twcc_overwrites_exthdr_seqnum_if_present) header extension */ buf = generate_test_buffer_full (0, 0, 0, 0xabc, FALSE, TEST_BUF_PT, - TEST_TWCC_EXT_ID, 255); + TEST_TWCC_EXT_ID, 255, -1, -1, 0, NULL, 0); fail_unless_equals_int64 (session_harness_send_rtp (h, buf), GST_FLOW_OK); buf = session_harness_pull_send_rtp (h); @@ -6279,6 +6423,8 @@ rtpsession_suite (void) tcase_add_test (tc_chain, test_twcc_feedback_old_seqnum); tcase_add_test (tc_chain, test_twcc_stats_rtx_recover_lost); tcase_add_test (tc_chain, test_twcc_stats_no_rtx_no_recover); + tcase_add_test (tc_chain, test_twcc_stats_long_rtx_recover); + tcase_add_test (tc_chain, test_twcc_stats_block_fec_recover); tcase_add_test (tc_chain, test_twcc_feedback_max_sent_packets); tcase_add_test (tc_chain, test_twcc_non_twcc_pkts_does_not_mark_loss); tcase_add_test (tc_chain, test_twcc_non_twcc_pt_no_twcc_seqnum); diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtpulpfec.c b/subprojects/gst-plugins-good/tests/check/elements/rtpulpfec.c index 5ce03facd7..153570b096 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtpulpfec.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtpulpfec.c @@ -25,6 +25,7 @@ #include #include +#include #include #define RTP_PACKET_DUR (10 * GST_MSECOND) @@ -459,6 +460,101 @@ SampleRTPPacket avmcu_fec_packets[] = { , }; +static GstElement * +request_fec_encoder (GstElement * rtpbin, guint sessid, G_GNUC_UNUSED void *user_data) +{ + gint *percentage = (gint *) user_data; + GstElement *fecenc = gst_element_factory_make_full ("rtpulpfecenc", + "pt", 100, + "multipacket", TRUE, + "percentage", *percentage, + NULL); + return fecenc; +} + +static GstHarness * +harness_rtpulpfecenc (guint32 ssrc, guint8 lost_pt, gint *percentage) +{ + GstElement *rtpbin = gst_element_factory_make ("rtpbin", NULL); + g_signal_connect (rtpbin, "request-fec-encoder", + (GCallback) request_fec_encoder, percentage); + GstPad * rtpbin_sink = gst_element_request_pad_simple (rtpbin, + "send_rtp_sink_0"); + GstPad * rtpbin_src = gst_element_get_static_pad (rtpbin, "send_rtp_src_0"); + + GstHarness *h = gst_harness_new_with_element (rtpbin, + "send_rtp_sink_0", "send_rtp_src_0"); + gst_object_unref (rtpbin_sink); + gst_object_unref (rtpbin_src); + + gchar *caps_str = + g_strdup_printf ("application/x-rtp,ssrc=(uint)%u,payload=(int)%u", + ssrc, lost_pt); + gst_harness_set_src_caps_str (h, caps_str); + g_free (caps_str); + + return h; +} + +GST_START_TEST (rtpulpfecdec_repairmeta) +{ + guint i = 0; + guint16 block_seq[8] = {0}; + guint16 seq = 12511; + const gint block_len = G_N_ELEMENTS (block_seq); + const gfloat redrate = 0.25f; + gint percentage = (gint)(redrate * 100); + GstHarness *h = harness_rtpulpfecenc (0x00000d51, 122, &percentage); + GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; + GstBuffer *buf = NULL; + GstBuffer *bufout = NULL; + + for (i = 0; i < G_N_ELEMENTS (avmcu_media_packets); ++i) { + gchar* pkt_data = g_memdup2 (avmcu_media_packets[i].data, + avmcu_media_packets[i].len); + if (i > 0 && (i+1) % block_len == 0) { + /* Set the marker bit for every block_len-th packet + so that the FEC packets are generated */ + pkt_data[1] |= 0x80; + } + buf = gst_rtp_buffer_new_take_data (pkt_data, + avmcu_media_packets[i].len); + bufout = gst_harness_push_and_pull (h, buf); + do { + fail_unless (gst_rtp_buffer_map (bufout, + GST_MAP_READ, &rtp)); + fail_unless_equals_int (gst_rtp_buffer_get_seq (&rtp), seq); + const guint8 pt = gst_rtp_buffer_get_payload_type (&rtp); + GstRTPRepairMeta *meta = gst_buffer_get_rtp_repair_meta (bufout); + if (pt == 100) { + /* Got a redundant packet from FEC encoder, check its repair meta */ + fail_unless (meta); + fail_unless_equals_int (meta->ssrc, 0x00000d51); + fail_unless_equals_int (meta->seqnums->len, block_len); + for (gint j = 0; j < block_len; ++j) { + fail_unless_equals_int (g_array_index(meta->seqnums, guint16, j), + block_seq[j]); + } + } else if (pt == 122) { + /* Data packet has just poped out, remember its seqnum */ + block_seq[i % block_len] = gst_rtp_buffer_get_seq (&rtp); + } else { + fail ("Unexpected payload type"); + } + + seq++; + gst_rtp_buffer_unmap (&rtp); + } while ((bufout = gst_harness_try_pull (h)) != NULL); + } + + /* Check that FEC encoder doesn't generate FEC packets more than expected */ + g_usleep (G_USEC_PER_SEC / 100); + i = 0; + while ((bufout = gst_harness_try_pull (h)) != NULL) i++; + fail_unless (i == 0); +} + +GST_END_TEST; GST_START_TEST (rtpulpfecdec_recovered_using_recovered_packet) { @@ -656,6 +752,7 @@ rtpfec_suite (void) tcase_add_test (tc_chain, rtpulpfecdec_recovered_from_fec); tcase_add_test (tc_chain, rtpulpfecdec_recovered_from_fec_long); tcase_add_loop_test (tc_chain, rtpulpfecdec_recovered_from_many, 0, 4); + tcase_add_test (tc_chain, rtpulpfecdec_repairmeta); tcase_add_test (tc_chain, rtpulpfecdec_recovered_using_recovered_packet); tcase_add_test (tc_chain, rtpulpfecdec_recovered_from_storage); tcase_add_test (tc_chain, rtpulpfecdec_recovered_push_failed);