feat: add audioSegmentTimingInfo event to match videoSegmentTimingInfo (#364)

Author: gesinger

lib/mp4/audio-frame-utils.js

@@ -87,6 +87,8 @@ var prefixWithSilence = function(
 
   track.baseMediaDecodeTime -=
     Math.floor(clock.videoTsToAudioTs(audioFillDuration, track.samplerate));
+
+  return audioFillDuration;
 };
 
 // If the audio segment extends before the earliest allowed dts

lib/mp4/transmuxer.js

@@ -49,7 +49,7 @@ var arrayEquals = function(a, b) {
   return true;
 };
 
-var generateVideoSegmentTimingInfo = function(
+var generateSegmentTimingInfo = function(
   baseMediaDecodeTime,
   startDts,
   startPts,
@@ -132,7 +132,9 @@ AudioSegmentStream = function(track, options) {
       moof,
       mdat,
       boxes,
-      frameDuration;
+      frameDuration,
+      segmentDuration,
+      videoClockCyclesOfSilencePrefixed;
 
     // return early if no audio data has been observed
     if (adtsFrames.length === 0) {
@@ -145,7 +147,8 @@ AudioSegmentStream = function(track, options) {
     track.baseMediaDecodeTime = trackDecodeInfo.calculateTrackBaseMediaDecodeTime(
       track, options.keepOriginalTimestamps);
 
-    audioFrameUtils.prefixWithSilence(
+    // amount of audio filled but the value is in video clock rather than audio clock
+    videoClockCyclesOfSilencePrefixed = audioFrameUtils.prefixWithSilence(
       track, frames, audioAppendStartTs, videoBaseMediaDecodeTime);
 
     // we have to build the index from byte locations to
@@ -175,9 +178,27 @@ AudioSegmentStream = function(track, options) {
     // valid use-case where an init segment/data should be triggered without associated
     // frames. Leaving for now, but should be looked into.
     if (frames.length) {
+      segmentDuration = frames.length * frameDuration;
+
+      this.trigger(
+        'segmentTimingInfo',
+        generateSegmentTimingInfo(
+          // The audio track's baseMediaDecodeTime is in audio clock cycles, but the
+          // frame info is in video clock cycles. Convert to match expectation of
+          // listeners (that all timestamps will be based on video clock cycles).
+          clock.audioTsToVideoTs(track.baseMediaDecodeTime, track.samplerate),
+          // frame times are already in video clock, as is segment duration
+          frames[0].dts,
+          frames[0].pts,
+          frames[0].dts + segmentDuration,
+          frames[0].pts + segmentDuration,
+          videoClockCyclesOfSilencePrefixed || 0
+        )
+      );
+
       this.trigger('timingInfo', {
         start: frames[0].pts,
-        end: frames[0].pts + (frames.length * frameDuration)
+        end: frames[0].pts + segmentDuration
       });
     }
     this.trigger('data', {track: track, boxes: boxes});
@@ -392,7 +413,7 @@ VideoSegmentStream = function(track, options) {
 
     this.trigger(
       'segmentTimingInfo',
-      generateVideoSegmentTimingInfo(
+      generateSegmentTimingInfo(
         track.baseMediaDecodeTime,
         firstGop.dts,
         firstGop.pts,
@@ -1051,6 +1072,8 @@ Transmuxer = function(options) {
 
           pipeline.audioSegmentStream.on('timingInfo',
             self.trigger.bind(self, 'audioTimingInfo'));
+          pipeline.audioSegmentStream.on('segmentTimingInfo',
+            self.trigger.bind(self, 'audioSegmentTimingInfo'));
 
           // Set up the final part of the audio pipeline
           pipeline.adtsStream
@@ -1180,5 +1203,5 @@ module.exports = {
   AUDIO_PROPERTIES: AUDIO_PROPERTIES,
   VIDEO_PROPERTIES: VIDEO_PROPERTIES,
   // exported for testing
-  generateVideoSegmentTimingInfo: generateVideoSegmentTimingInfo
+  generateSegmentTimingInfo: generateSegmentTimingInfo
 };

test/transmuxer.test.js

@@ -11,7 +11,7 @@ var mp2t = require('../lib/m2ts'),
     mp4Transmuxer = require('../lib/mp4/transmuxer'),
     mp4AudioProperties = mp4Transmuxer.AUDIO_PROPERTIES,
     mp4VideoProperties = mp4Transmuxer.VIDEO_PROPERTIES,
-    generateVideoSegmentTimingInfo = mp4Transmuxer.generateVideoSegmentTimingInfo,
+    generateSegmentTimingInfo = mp4Transmuxer.generateSegmentTimingInfo,
     clock = require('../lib/utils/clock'),
     utils = require('./utils'),
     TransportPacketStream = mp2t.TransportPacketStream,
@@ -2532,7 +2532,7 @@ QUnit.test('alignGopsAtEnd_ filters gops appropriately', function(assert) {
     'match with an alignment candidate');
 });
 
-QUnit.test('generateVideoSegmentTimingInfo generates correct timing info object', function(assert) {
+QUnit.test('generateSegmentTimingInfo generates correct timing info object', function(assert) {
   var
     firstFrame = {
       dts: 12,
@@ -2548,7 +2548,7 @@ QUnit.test('generateVideoSegmentTimingInfo generates correct timing info object'
     prependedContentDuration = 0;
 
   assert.deepEqual(
-    generateVideoSegmentTimingInfo(
+    generateSegmentTimingInfo(
       baseMediaDecodeTime,
       firstFrame.dts,
       firstFrame.pts,
@@ -2573,7 +2573,7 @@ QUnit.test('generateVideoSegmentTimingInfo generates correct timing info object'
     }, 'generated correct timing info object');
 });
 
-QUnit.test('generateVideoSegmentTimingInfo accounts for prepended GOPs', function(assert) {
+QUnit.test('generateSegmentTimingInfo accounts for prepended GOPs', function(assert) {
   var
     firstFrame = {
       dts: 12,
@@ -2589,7 +2589,7 @@ QUnit.test('generateVideoSegmentTimingInfo accounts for prepended GOPs', functio
     prependedContentDuration = 7;
 
   assert.deepEqual(
-    generateVideoSegmentTimingInfo(
+    generateSegmentTimingInfo(
       baseMediaDecodeTime,
       firstFrame.dts,
       firstFrame.pts,
@@ -2615,7 +2615,7 @@ QUnit.test('generateVideoSegmentTimingInfo accounts for prepended GOPs', functio
     'included prepended content duration in timing info');
 });
 
-QUnit.test('generateVideoSegmentTimingInfo handles GOPS where pts is < dts', function(assert) {
+QUnit.test('generateSegmentTimingInfo handles GOPS where pts is < dts', function(assert) {
   var
     firstFrame = {
       dts: 14,
@@ -2631,7 +2631,7 @@ QUnit.test('generateVideoSegmentTimingInfo handles GOPS where pts is < dts', fun
     prependedContentDuration = 7;
 
   assert.deepEqual(
-    generateVideoSegmentTimingInfo(
+    generateSegmentTimingInfo(
       baseMediaDecodeTime,
       firstFrame.dts,
       firstFrame.pts,
@@ -3221,6 +3221,60 @@ QUnit.test('audio track metadata takes on the value of the last metadata seen',
   assert.equal(events[0].track.channelcount, 4, 'kept the later channelcount');
 });
 
+QUnit.test('audio segment stream triggers segmentTimingInfo with timing info',
+function(assert) {
+  var
+    events = [],
+    samplerate = 48000,
+    baseMediaDecodeTimeInVideoClock = 30,
+    audioFrameDurationInVideoClock = 90000 * 1024 / samplerate,
+    firstFrame = {
+      channelcount: 2,
+      samplerate: samplerate,
+      pts: 112,
+      dts: 111,
+      data: new Uint8Array([0])
+    },
+    secondFrame = {
+      channelcount: 2,
+      samplerate: samplerate,
+      pts: firstFrame.pts + audioFrameDurationInVideoClock,
+      dts: firstFrame.dts + audioFrameDurationInVideoClock,
+      data: new Uint8Array([1])
+    };
+
+  audioSegmentStream.on('segmentTimingInfo', function(event) {
+    events.push(event);
+  });
+  audioSegmentStream.track.timelineStartInfo.baseMediaDecodeTime =
+    baseMediaDecodeTimeInVideoClock;
+
+  audioSegmentStream.push(firstFrame);
+  audioSegmentStream.push(secondFrame);
+  audioSegmentStream.flush();
+
+  assert.equal(events.length, 1, 'a segmentTimingInfo event was fired');
+  assert.deepEqual(
+    events[0],
+    {
+      start: {
+        dts: baseMediaDecodeTimeInVideoClock,
+        pts: baseMediaDecodeTimeInVideoClock + (firstFrame.pts - firstFrame.dts)
+      },
+      end: {
+        dts: baseMediaDecodeTimeInVideoClock + (secondFrame.dts - firstFrame.dts) +
+          audioFrameDurationInVideoClock,
+        pts: baseMediaDecodeTimeInVideoClock + (secondFrame.pts - firstFrame.pts) +
+          audioFrameDurationInVideoClock
+      },
+      prependedContentDuration: 0,
+      baseMediaDecodeTime: baseMediaDecodeTimeInVideoClock
+    },
+    'has correct segmentTimingInfo'
+  );
+});
+
+
 QUnit.module('Transmuxer - options');
 
 QUnit.test('no options creates combined output', function(assert) {