Changeset 5887

Timestamp:

Sep 19, 2018 8:26:41 AM (6 years ago)

Author:

nanang

Message:

Close #2152: Updated video stream to delay video frame decoding, the minimum delay is configurable via PJMEDIA_VID_STREAM_DECODE_MIN_DELAY_MSEC.

Location:

pjproject/trunk/pjmedia

Files:

• include/pjmedia/config.h (modified) (3 diffs)
• include/pjmedia/jbuf.h (modified) (1 diff)
• src/pjmedia/jbuf.c (modified) (1 diff)
• src/pjmedia/vid_stream.c (modified) (12 diffs)

pjproject/trunk/pjmedia/include/pjmedia/config.h

@@ -1494 +1494 @@
 #endif
 
+
 /**
  * Specify the number of keyframe needed to be sent after the stream is 
@@ -1504 +1505 @@
 #endif
 
+
 /**
  * Specify the interval to send keyframe after the stream is created, in msec.
@@ -1515 +1517 @@
 
 /**
+ * Specify minimum delay of video decoding, in milliseconds. Lower value may
+ * degrade video quality significantly in a bad network environment (e.g:
+ * with persistent late and out-of-order RTP packets). Note that the value
+ * must be lower than jitter buffer maximum delay (configurable via
+ * pjmedia_stream_info.jb_max or pjsua_media_config.jb_max).
+ *
+ * Default : 100
+ */
+#ifndef PJMEDIA_VID_STREAM_DECODE_MIN_DELAY_MSEC
+#   define PJMEDIA_VID_STREAM_DECODE_MIN_DELAY_MSEC         100
+#endif
+
+
+
+/**
  * @}
  */

pjproject/trunk/pjmedia/include/pjmedia/jbuf.h

@@ -103 +103 @@
     unsigned    min_prefetch;       /**< Minimum allowed prefetch, in frms. */
     unsigned    max_prefetch;       /**< Maximum allowed prefetch, in frms. */
+    unsigned    max_count;          /**< Jitter buffer capacity, in frames. */
 
     /* Status */

pjproject/trunk/pjmedia/src/pjmedia/jbuf.c

@@ -1161 +1161 @@
     state->min_prefetch = jb->jb_min_prefetch;
     state->max_prefetch = jb->jb_max_prefetch;
+    state->max_count = jb->jb_max_count;
 
     state->burst = jb->jb_eff_level;

pjproject/trunk/pjmedia/src/pjmedia/vid_stream.c

@@ -140 +140 @@
     pjmedia_ratio            dec_max_fps;   /**< Max fps of decoding dir.   */
     pjmedia_frame            dec_frame;     /**< Current decoded frame.     */
+    unsigned                 dec_delay_cnt; /**< Decoding delay (in frames).*/
     pjmedia_event            fmt_event;     /**< Buffered fmt_changed event
                                                  to avoid deadlock          */
@@ -1128 +1129 @@
 {
     pjmedia_vid_channel *channel = stream->dec;
-    pj_uint32_t last_ts = 0;
+    pj_uint32_t last_ts = 0, frm_ts = 0;
     int frm_first_seq = 0, frm_last_seq = 0;
     pj_bool_t got_frame = PJ_FALSE;
-    unsigned cnt;
+    unsigned cnt, frm_pkt_cnt = 0, frm_cnt = 0;
     pj_status_t status;
 
@@ -1139 +1140 @@
 
     /* Check if we got a decodable frame */
-    for (cnt=0; ; ++cnt) {
+    for (cnt=0; ; ) {
         char ptype;
         pj_uint32_t ts;
@@ -1148 +1149 @@
                                 &ptype, NULL, &ts, &seq);
         if (ptype == PJMEDIA_JB_NORMAL_FRAME) {
+            if (stream->last_dec_ts ==  ts) {
+                /* Remove any late packet (the frame has been decoded) */
+                pjmedia_jbuf_remove_frame(stream->jb, 1);
+                continue;
+            }
+
             if (last_ts == 0) {
                 last_ts = ts;
+
+                /* Init timestamp and first seq of the first frame */
+                frm_ts = ts;
                 frm_first_seq = seq;
             }
             if (ts != last_ts) {
-                got_frame = PJ_TRUE;
-                break;
+                last_ts = ts;
+                if (frm_pkt_cnt == 0)
+                    frm_pkt_cnt = cnt;
+
+                /* Is it time to decode? Check with minimum delay setting */
+                if (++frm_cnt == stream->dec_delay_cnt) {
+                    got_frame = PJ_TRUE;
+                    break;
+                }
             }
-            frm_last_seq = seq;
         } else if (ptype == PJMEDIA_JB_ZERO_EMPTY_FRAME) {
             /* No more packet in the jitter buffer */
             break;
         }
+
+        ++cnt;
     }
 
@@ -1166 +1184 @@
         unsigned i;
 
-        /* Generate frame bitstream from the payload */
-        if (cnt > stream->rx_frame_cnt) {
+        /* Exclude any MISSING frames in the end of the packets array, as
+         * it may be part of the next video frame (late packets).
+         */
+        for (; frm_pkt_cnt > 1; --frm_pkt_cnt) {
+            char ptype;
+            pjmedia_jbuf_peek_frame(stream->jb, frm_pkt_cnt, NULL, NULL, &ptype,
+                                    NULL, NULL, NULL);
+            if (ptype == PJMEDIA_JB_NORMAL_FRAME)
+                break;
+        }
+
+        /* Check if the packet count for this frame exceeds the limit */
+        if (frm_pkt_cnt > stream->rx_frame_cnt) {
             PJ_LOG(1,(channel->port.info.name.ptr,
                       "Discarding %u frames because array is full!",
-                      cnt - stream->rx_frame_cnt));
-            pjmedia_jbuf_remove_frame(stream->jb, cnt - stream->rx_frame_cnt);
-            cnt = stream->rx_frame_cnt;
-        }
-
-        for (i = 0; i < cnt; ++i) {
+                      frm_pkt_cnt - stream->rx_frame_cnt));
+            pjmedia_jbuf_remove_frame(stream->jb,
+                                      frm_pkt_cnt - stream->rx_frame_cnt);
+            frm_pkt_cnt = stream->rx_frame_cnt;
+        }
+
+        /* Generate frame bitstream from the payload */
+        for (i = 0; i < frm_pkt_cnt; ++i) {
             char ptype;
 
             stream->rx_frames[i].type = PJMEDIA_FRAME_TYPE_VIDEO;
-            stream->rx_frames[i].timestamp.u64 = last_ts;
+            stream->rx_frames[i].timestamp.u64 = frm_ts;
             stream->rx_frames[i].bit_info = 0;
 
@@ -1188 +1219 @@
                                     (const void**)&stream->rx_frames[i].buf,
                                     &stream->rx_frames[i].size, &ptype,
-                                    NULL, NULL, NULL);
+                                    NULL, NULL, &frm_last_seq);
 
             if (ptype != PJMEDIA_JB_NORMAL_FRAME) {
@@ -1200 +1231 @@
 
         /* Decode */
-        status = pjmedia_vid_codec_decode(stream->codec, cnt,
+        status = pjmedia_vid_codec_decode(stream->codec, frm_pkt_cnt,
                                           stream->rx_frames,
                                           (unsigned)frame->size, frame);
@@ -1210 +1241 @@
         }
 
-        pjmedia_jbuf_remove_frame(stream->jb, cnt);
+        pjmedia_jbuf_remove_frame(stream->jb, frm_pkt_cnt);
     }
 
     /* Learn remote frame rate after successful decoding */
-    if (frame->type == PJMEDIA_FRAME_TYPE_VIDEO && frame->size)
+    if (got_frame && frame->type == PJMEDIA_FRAME_TYPE_VIDEO && frame->size)
     {
         /* Only check remote frame rate when timestamp is not wrapping and
          * sequence is increased by 1.
          */
-        if (last_ts > stream->last_dec_ts &&
+        if (frm_ts > stream->last_dec_ts &&
             frm_first_seq - stream->last_dec_seq == 1)
         {
@@ -1225 +1256 @@
             pjmedia_video_format_detail *vfd;
 
-            ts_diff = last_ts - stream->last_dec_ts;
+            ts_diff = frm_ts - stream->last_dec_ts;
             vfd = pjmedia_format_get_video_format_detail(
                                     &channel->port.info.fmt, PJ_TRUE);
@@ -1242 +1273 @@
                 /* Update stream info */
                 stream->info.codec_param->dec_fmt.det.vid.fps = vfd->fps;
+
+                /* Update the decoding delay as FPS updated */
+                {
+                    pjmedia_jb_state jb_state;
+                    pjmedia_jbuf_get_state(stream->jb, &jb_state);
+
+                    stream->dec_delay_cnt = 
+                                    ((PJMEDIA_VID_STREAM_DECODE_MIN_DELAY_MSEC *
+                                      vfd->fps.num) +
+                                     (1000 * vfd->fps.denum) - 1) /
+                                    (1000 * vfd->fps.denum);
+                    if (stream->dec_delay_cnt < 1)
+                        stream->dec_delay_cnt = 1;
+                    if (stream->dec_delay_cnt > jb_state.max_count * 4/5)
+                        stream->dec_delay_cnt = jb_state.max_count * 4/5;
+                }
 
                 /* Publish PJMEDIA_EVENT_FMT_CHANGED event if frame rate
@@ -1275 +1322 @@
         /* Update last frame seq and timestamp */
         stream->last_dec_seq = frm_last_seq;
-        stream->last_dec_ts = last_ts;
+        stream->last_dec_ts = frm_ts;
     }
 
@@ -1694 +1741 @@
     else
         jb_init  = 0;
+
+    /* Calculate the decoding delay (in number of frames) based on FPS */
+    stream->dec_delay_cnt = ((PJMEDIA_VID_STREAM_DECODE_MIN_DELAY_MSEC *
+                              vfd_dec->fps.num) +
+                             (1000 * vfd_dec->fps.denum) - 1) /
+                            (1000 * vfd_dec->fps.denum);
+    if (stream->dec_delay_cnt < 1)
+        stream->dec_delay_cnt = 1;
+    if (stream->dec_delay_cnt > jb_max * 4/5)
+        stream->dec_delay_cnt = jb_max * 4/5;
 
     /* Allocate array for temporary storage for assembly of incoming