Changeset 3835

Timestamp:

Oct 23, 2011 5:23:55 AM (13 years ago)

Author:

bennylp

Message:

Fixed #1392: Immediately decode incoming frames to avoid loosing key frames when the jbuf is full and improve latency

Location:

pjproject/trunk/pjmedia

Files:

• include/pjmedia/config.h (modified) (1 diff)
• include/pjmedia/jbuf.h (modified) (1 diff)
• src/pjmedia/jbuf.c (modified) (1 diff)
• src/pjmedia/vid_stream.c (modified) (16 diffs)

pjproject/trunk/pjmedia/include/pjmedia/config.h

@@ -1046 +1046 @@
 
 /**
+ * Video stream will discard old picture from the jitter buffer as soon as
+ * new picture is received, to reduce latency.
+ *
+ * Default: 0
+ */
+#ifndef PJMEDIA_VID_STREAM_SKIP_PACKETS_TO_REDUCE_LATENCY
+#   define PJMEDIA_VID_STREAM_SKIP_PACKETS_TO_REDUCE_LATENCY    0
+#endif
+
+/**
  * @}
  */

pjproject/trunk/pjmedia/include/pjmedia/jbuf.h

@@ -390 +390 @@
                                             unsigned frame_cnt);
 
+/**
+ * Check if the jitter buffer is full.
+ *
+ * @param jb            The jitter buffer.
+ *
+ * @return              PJ_TRUE if it is full.
+ */
+PJ_DECL(pj_bool_t) pjmedia_jbuf_is_full(const pjmedia_jbuf *jb);
+
 
 /**

pjproject/trunk/pjmedia/src/pjmedia/jbuf.c

@@ -636 +636 @@
 }
 
+PJ_DEF(pj_bool_t) pjmedia_jbuf_is_full(const pjmedia_jbuf *jb)
+{
+    return jb->jb_framelist.size == jb->jb_framelist.max_count;
+}
 
 static void jbuf_calculate_jitter(pjmedia_jbuf *jb)

pjproject/trunk/pjmedia/src/pjmedia/vid_stream.c

@@ -69 +69 @@
 #endif
 
-
 /**
  * Media channel.
@@ -82 +81 @@
     void                   *buf;            /**< Output buffer.             */
     unsigned                buf_size;       /**< Size of output buffer.     */
-    unsigned                buf_len;        /**< Length of data in buffer.  */
     pjmedia_rtp_session     rtp;            /**< RTP session.               */
 } pjmedia_vid_channel;
@@ -121 +119 @@
     pj_uint32_t              rtcp_interval; /**< Interval, in timestamp.    */
     pj_bool_t                initial_rr;    /**< Initial RTCP RR sent       */
+
+    unsigned                 dec_max_size;  /**< Size of decoded/raw picture*/
+    pjmedia_frame            dec_frame;     /**< Current decoded frame.     */
 
     unsigned                 frame_size;    /**< Size of encoded base frame.*/
@@ -150 +151 @@
 };
 
+/* Prototypes */
+static pj_status_t decode_frame(pjmedia_vid_stream *stream,
+                                pjmedia_frame *frame);
 
 /*
@@ -636 +640 @@
     }
 
+    pj_mutex_lock( stream->jb_mutex );
+
+    /* Quickly see if there may be a full picture in the jitter buffer, and
+     * decode them if so. More thorough check will be done in decode_frame().
+     */
+    if ((pj_ntohl(hdr->ts) != stream->dec_frame.timestamp.u32.lo) || hdr->m) {
+        if (PJMEDIA_VID_STREAM_SKIP_PACKETS_TO_REDUCE_LATENCY) {
+            /* Always decode whenever we have picture in jb and
+             * overwrite already decoded picture if necessary
+             */
+            pj_size_t old_size = stream->dec_frame.size;
+
+            stream->dec_frame.size = stream->dec_max_size;
+            if (decode_frame(stream, &stream->dec_frame) != PJ_SUCCESS) {
+                stream->dec_frame.size = old_size;
+            }
+        } else {
+            /* Only decode if we don't already have decoded one,
+             * unless the jb is full.
+             */
+            pj_bool_t can_decode = PJ_FALSE;
+
+            if (pjmedia_jbuf_is_full(stream->jb)) {
+                can_decode = PJ_TRUE;
+            }
+            else if (stream->dec_frame.size == 0) {
+                can_decode = PJ_TRUE;
+            }
+
+            if (can_decode) {
+                stream->dec_frame.size = stream->dec_max_size;
+                if (decode_frame(stream, &stream->dec_frame) != PJ_SUCCESS) {
+                    stream->dec_frame.size = 0;
+                }
+            }
+        }
+    }
 
     /* Put "good" packet to jitter buffer, or reset the jitter buffer
      * when RTP session is restarted.
      */
-    pj_mutex_lock( stream->jb_mutex );
     if (seq_st.status.flag.restart) {
         status = pjmedia_jbuf_reset(stream->jb);
@@ -816 +856 @@
             if (stream->send_err_cnt++ == 0) {
                 LOGERR_((channel->port.info.name.ptr,
-                         "Transport send_rtp() error (repeated %d times)",
+                         "Transport send_rtp() error",
                          status));
-                if (stream->send_err_cnt > COUNT_TO_REPORT)
-                    stream->send_err_cnt = 0;
             }
+            if (stream->send_err_cnt > COUNT_TO_REPORT)
+                stream->send_err_cnt = 0;
             /* Ignore this error */
         }
@@ -874 +914 @@
 }
 
-static pj_status_t get_frame(pjmedia_port *port,
-                             pjmedia_frame *frame)
-{
-    pjmedia_vid_stream *stream = (pjmedia_vid_stream*) port->port_data.pdata;
+/* Decode one image from jitter buffer */
+static pj_status_t decode_frame(pjmedia_vid_stream *stream,
+                                pjmedia_frame *frame)
+{
     pjmedia_vid_channel *channel = stream->dec;
     pj_uint32_t last_ts = 0;
@@ -885 +925 @@
     pj_status_t status;
 
-    /* Return no frame is channel is paused */
-    if (channel->paused) {
-        frame->type = PJMEDIA_FRAME_TYPE_NONE;
-        return PJ_SUCCESS;
-    }
-
     /* Repeat get payload from the jitter buffer until all payloads with same
      * timestamp are collected.
      */
-    channel->buf_len = 0;
-
-    /* Lock jitter buffer mutex first */
-    pj_mutex_lock( stream->jb_mutex );
 
     /* Check if we got a decodable frame */
@@ -928 +958 @@
 
         /* Generate frame bitstream from the payload */
-        channel->buf_len = 0;
-
         if (cnt > stream->rx_frame_cnt) {
-            PJ_LOG(1,(port->info.name.ptr,
+            PJ_LOG(1,(channel->port.info.name.ptr,
                       "Discarding %u frames because array is full!",
                       cnt - stream->rx_frame_cnt));
@@ -967 +995 @@
                                           frame->size, frame);
         if (status != PJ_SUCCESS) {
-            LOGERR_((port->info.name.ptr, "codec decode() error",
+            LOGERR_((channel->port.info.name.ptr, "codec decode() error",
                      status));
             frame->type = PJMEDIA_FRAME_TYPE_NONE;
@@ -975 +1003 @@
         pjmedia_jbuf_remove_frame(stream->jb, cnt);
     }
-
-    /* Unlock jitter buffer mutex. */
-    pj_mutex_unlock( stream->jb_mutex );
 
     /* Learn remote frame rate after successful decoding */
@@ -1031 +1056 @@
     }
 
+    return got_frame ? PJ_SUCCESS : PJ_ENOTFOUND;
+}
+
+
+static pj_status_t get_frame(pjmedia_port *port,
+                             pjmedia_frame *frame)
+{
+    pjmedia_vid_stream *stream = (pjmedia_vid_stream*) port->port_data.pdata;
+    pjmedia_vid_channel *channel = stream->dec;
+
+    /* Return no frame is channel is paused */
+    if (channel->paused) {
+        frame->type = PJMEDIA_FRAME_TYPE_NONE;
+        frame->size = 0;
+        return PJ_SUCCESS;
+    }
+
+    pj_mutex_lock( stream->jb_mutex );
+
+    if (stream->dec_frame.size == 0) {
+        /* Don't have frame in buffer, try to decode one */
+        if (decode_frame(stream, frame) != PJ_SUCCESS) {
+            frame->type = PJMEDIA_FRAME_TYPE_NONE;
+            frame->size = 0;
+        }
+    } else {
+        if (frame->size < stream->dec_frame.size) {
+            PJ_LOG(4,(stream->dec->port.info.name.ptr,
+                      "Error: not enough buffer for decoded frame "
+                      "(supplied=%d, required=%d)",
+                      (int)frame->size, (int)stream->dec_frame.size));
+            frame->type = PJMEDIA_FRAME_TYPE_NONE;
+            frame->size = 0;
+        } else {
+            frame->type = stream->dec_frame.type;
+            frame->timestamp = stream->dec_frame.timestamp;
+            frame->size = stream->dec_frame.size;
+            pj_memcpy(frame->buf, stream->dec_frame.buf, frame->size);
+        }
+
+        stream->dec_frame.size = 0;
+    }
+
+    pj_mutex_unlock( stream->jb_mutex );
+
     return PJ_SUCCESS;
 }
-
 
 /*
@@ -1081 +1150 @@
     
     /* Allocate buffer for outgoing packet. */
-    channel->buf_size = sizeof(pjmedia_rtp_hdr) + stream->frame_size;
-
-    /* It should big enough to hold (minimally) RTCP SR with an SDES. */
-    min_out_pkt_size =  sizeof(pjmedia_rtcp_sr_pkt) +
-                        sizeof(pjmedia_rtcp_common) +
-                        (4 + stream->cname.slen) +
-                        32;
-
-    if (channel->buf_size < min_out_pkt_size)
-        channel->buf_size = min_out_pkt_size;
-
-    channel->buf = pj_pool_alloc(pool, channel->buf_size);
-    PJ_ASSERT_RETURN(channel->buf != NULL, PJ_ENOMEM);
+    if (dir == PJMEDIA_DIR_ENCODING) {
+        channel->buf_size = sizeof(pjmedia_rtp_hdr) + stream->frame_size;
+
+        /* It should big enough to hold (minimally) RTCP SR with an SDES. */
+        min_out_pkt_size =  sizeof(pjmedia_rtcp_sr_pkt) +
+                            sizeof(pjmedia_rtcp_common) +
+                            (4 + stream->cname.slen) +
+                            32;
+
+        if (channel->buf_size < min_out_pkt_size)
+            channel->buf_size = min_out_pkt_size;
+
+        channel->buf = pj_pool_alloc(pool, channel->buf_size);
+        PJ_ASSERT_RETURN(channel->buf != NULL, PJ_ENOMEM);
+    }
 
     /* Create RTP and RTCP sessions: */
@@ -1157 +1228 @@
     unsigned jb_init, jb_max, jb_min_pre, jb_max_pre, len;
     int frm_ptime, chunks_per_frm;
-    pjmedia_video_format_detail *vfd_enc;
+    pjmedia_video_format_detail *vfd_enc, *vfd_dec;
     char *p;
     unsigned mtu;
@@ -1215 +1286 @@
     vfd_enc = pjmedia_format_get_video_format_detail(
                                         &info->codec_param->enc_fmt, PJ_TRUE);
+    vfd_dec = pjmedia_format_get_video_format_detail(
+                                        &info->codec_param->dec_fmt, PJ_TRUE);
 
     /* Init stream: */
@@ -1297 +1370 @@
     if (status != PJ_SUCCESS)
         return status;
+
+    /* Create temporary buffer for immediate decoding */
+    stream->dec_max_size = vfd_dec->size.w * vfd_dec->size.h * 4;
+    stream->dec_frame.buf = pj_pool_alloc(pool, stream->dec_max_size);
 
     /* Init jitter buffer parameters: */