Ticket #1478: av_dev_synch.diff

pjsip-apps/src/pjsua/pjsua_app.c

@@ -5762 +5762 @@
 
             pjmedia_avi_dev_param_default(&avdp);
             avdp.path = app_config.avi[i].path;
+            avdp.sync_av = PJ_TRUE;
+            if (app_config.null_audio) {
+                /* null audio shouldn't have burst so we can have
+                 * better sync here.
+                 */
+                avdp.max_unsync = 80;
+            }
 
             status =  pjmedia_avi_dev_alloc(avi_factory, &avdp, &avid);
             if (status != PJ_SUCCESS) {
@@ -5778 +5785 @@
             if (app_config.avi_def_idx == PJSUA_INVALID_ID)
                 app_config.avi_def_idx = i;
 
-            strm_cnt = pjmedia_avi_streams_get_num_streams(avdp.avi_streams);
+            strm_cnt = avdp.aud_port_cnt;
             for (strm_idx=0; strm_idx<strm_cnt; ++strm_idx) {
                 pjmedia_port *aud;
                 pjmedia_format *fmt;
                 pjsua_conf_port_id slot;
                 char fmt_name[5];
 
-                aud = pjmedia_avi_streams_get_stream(avdp.avi_streams,
-                                                     strm_idx);
+                aud = avdp.aud_port[strm_idx];
                 fmt = &aud->info.fmt;
 
                 pjmedia_fourcc_name(fmt->id, fmt_name);

pjmedia/include/pjmedia-videodev/avi_dev.h

@@ -56 +56 @@
     pj_str_t    title;
 
     /**
-     * The underlying AVI streams created by the device. If the value is NULL,
-     * that means the device has not been configured yet. Application can use
-     * this field to retrieve the audio stream of the AVI. This setting is
-     * "get"-only and will be ignored in "set capability" operation.
+     * Keep audio and video synchronized.
+     *
+     * Default: PJ_FALSE
      */
-    pjmedia_avi_streams *avi_streams;
+    pj_bool_t   sync_av;
 
+    /**
+     * Maximum unsynchronization delay between audio and video that is allowed
+     * before sync algorithm kicks in. This value should be larger than the
+     * sound device burst. Setting this to low value will cause too many
+     * synchronization attempts, while setting this too high will cause unsync
+     * between audio and video. Value is in milliseconds.
+     *
+     * Default: 200
+     */
+    unsigned    max_unsync;
+
+    /**
+     * Number of audio ports from the AVI. This value will be filled by the
+     * device.
+     */
+    unsigned    aud_port_cnt;
+
+    /**
+     * Array of audio ports from the AVI. This value will be filled by the
+     * device.
+     */
+    pjmedia_port *aud_port[8];
+
 } pjmedia_avi_dev_param;
 
 

pjmedia/src/pjmedia-videodev/avi_dev.c

@@ -22 +22 @@
 #include <pj/log.h>
 #include <pj/os.h>
 #include <pj/rand.h>
+#include <pjmedia/errno.h>
 #include <pjmedia/vid_codec.h>
 
 #if defined(PJMEDIA_VIDEO_DEV_HAS_AVI) && PJMEDIA_VIDEO_DEV_HAS_AVI != 0
 
 #define THIS_FILE               "avi_dev.c"
 #define DRIVER_NAME             "AVIDev"
-#define DEFAULT_CLOCK_RATE      90000
-#define DEFAULT_WIDTH           640
-#define DEFAULT_HEIGHT          480
-#define DEFAULT_FPS             25
+#define VID_CLOCK_RATE          90000
 
+#define SYNC_BUF_SIZE_MSEC      1000
+#define SYNC_MIN_BUF_SIZE_MSEC  300 /* should be larger than snd dev burst */
+#define SYNC_MAX_UNSYNC_MSEC    200 /* should be larger than snd dev burst */
+#define SYNC_DISCARD_INTERVAL   30
+
+#define AVI_LOG_EVENT(expr)     PJ_LOG(5,expr)
+
 typedef struct avi_dev_strm avi_dev_strm;
 
 /* avi_ device info */
@@ -45 +50 @@
     pj_str_t                     title;
     pjmedia_avi_streams         *avi;
     pjmedia_port                *vid;
+    unsigned                     vid_media_time;
+    unsigned                     max_unsync;
+    unsigned                     aud_cnt;
+    pjmedia_port                *aud_port[8];
     avi_dev_strm                *strm;
     pjmedia_vid_codec           *codec;
     pj_uint8_t                  *enc_buf;
@@ -136 +145 @@
 
 
 /****************************************************************************
+ * Audio proxy object, is a "proxy" for each audio media port from the AVI,
+ * and it is (only) used when A/V synchronization is enabled in the param.
+ */
+struct aud_proxy
+{
+    pjmedia_port         base;
+    struct avi_dev_info *adi;
+    pjmedia_port        *source;
+    pj_uint8_t          *buffer;
+    pj_uint8_t          *frame_buffer;
+    unsigned             head,
+                         size,
+                         cap,           /* Capacity */
+                         samples_per_frame,
+                         avg_frame_size;
+    pj_timestamp         ts;            /* Timestamp of first sample in buf */
+    pjmedia_frame_type   frame_type;
+    pj_bool_t            skip_next;
+    unsigned             last_discard;  /* Time of last discarding frame    */
+    unsigned             last_skip;     /* Time of last skipping frame    */
+};
+
+static pj_status_t aud_proxy_fill_buffer(pjmedia_port *this_port,
+                                         unsigned vid_med_time);
+
+/* This is the media port get_frame() callback. */
+static pj_status_t aud_proxy_get_frame(pjmedia_port *this_port,
+                                       pjmedia_frame *frame)
+{
+    struct aud_proxy *audp = (struct aud_proxy*) this_port;
+    unsigned vid_media_time;
+
+    vid_media_time = audp->adi->vid_media_time;
+    aud_proxy_fill_buffer(this_port, vid_media_time);
+
+    if (audp->skip_next) {
+        /* We've been told to skip frame */
+        frame->type = PJMEDIA_FRAME_TYPE_NONE;
+        frame->size = 0;
+        audp->skip_next = PJ_FALSE;
+        return PJ_SUCCESS;
+    } else if (audp->size == 0) {
+        AVI_LOG_EVENT((THIS_FILE, "AVI audio port empty"));
+        frame->type = PJMEDIA_FRAME_TYPE_NONE;
+        frame->size = 0;
+        return PJ_SUCCESS;
+    }
+
+    if (audp->size < frame->size) {
+        AVI_LOG_EVENT((THIS_FILE, "AVI audio port underflow"));
+        frame->type = PJMEDIA_FRAME_TYPE_AUDIO;
+        frame->size = audp->size;
+        frame->timestamp = audp->ts;
+        pj_memcpy(frame->buf, audp->buffer, audp->size);
+        audp->head = audp->size = 0;
+        audp->ts.u64 += (frame->size *
+                         this_port->info.fmt.det.aud.clock_rate /
+                         (this_port->info.fmt.det.aud.avg_bps / 8));
+        return PJ_SUCCESS;
+    }
+
+    /* Copy one frame from circular buffer */
+    if (audp->head + frame->size <= audp->cap) {
+        pj_memcpy(frame->buf, audp->buffer+audp->head, frame->size);
+        audp->head += frame->size;
+    } else {
+        unsigned chunk_size = audp->cap - audp->head;
+        pj_memcpy(frame->buf, audp->buffer+audp->head, chunk_size);
+        pj_memcpy(frame->buf + chunk_size, audp->buffer,
+                  frame->size - chunk_size);
+        audp->head = frame->size - chunk_size;
+    }
+
+    /* Fix stuff */
+    audp->size -= frame->size;
+    frame->timestamp = audp->ts;
+    frame->type = audp->frame_type;
+    audp->ts.u64 += audp->samples_per_frame;
+
+    return PJ_SUCCESS;
+}
+
+/* media port callback to destroy the port */
+static pj_status_t aud_proxy_destroy(pjmedia_port *this_port)
+{
+    struct aud_proxy *audp = (struct aud_proxy*) this_port;
+    return pjmedia_port_destroy(audp->source);
+}
+
+/* internal: discard some samples from the circular buffer */
+static pj_status_t aud_proxy_discard(pjmedia_port *this_port,
+                                     unsigned size_to_discard)
+{
+    struct aud_proxy *audp = (struct aud_proxy*) this_port;
+
+    if (size_to_discard == 0)
+        size_to_discard = audp->avg_frame_size;
+    if (size_to_discard > audp->size)
+        size_to_discard = audp->size;
+
+    audp->head = (audp->head + size_to_discard) % audp->cap;
+    audp->size -= size_to_discard;
+    audp->ts.u64 += (size_to_discard *
+                     this_port->info.fmt.det.aud.clock_rate /
+                     (this_port->info.fmt.det.aud.avg_bps / 8));
+
+    return PJ_SUCCESS;
+}
+
+/* Get one frame from the AVI and put it to the circular buffer */
+static pj_status_t aud_proxy_fill_one_frame(pjmedia_port *this_port)
+{
+    struct aud_proxy *audp = (struct aud_proxy*) this_port;
+    pjmedia_frame frame;
+    unsigned tail, size_to_copy;
+    pj_status_t status;
+
+    /* Get frame from source first */
+    frame.buf = audp->frame_buffer;
+    frame.size = audp->avg_frame_size;
+
+    status = pjmedia_port_get_frame(audp->source, &frame);
+    if (status != PJ_SUCCESS)
+        return status;
+
+    if (frame.type == PJMEDIA_FRAME_TYPE_NONE)
+        return PJMEDIA_ENOTCOMPATIBLE;
+
+    /* Append to buffer */
+    if (audp->cap - audp->size < audp->avg_frame_size) {
+        /* Buffer is full, needs to throw away frame */
+        AVI_LOG_EVENT((THIS_FILE, "AVI audio port overflow"));
+        aud_proxy_discard(this_port, 0);
+    }
+
+    audp->frame_type = frame.type;
+
+    size_to_copy = frame.size;
+    tail = (audp->head + audp->size) % audp->cap;
+    if (tail >= audp->head) {
+        unsigned max = audp->cap - tail;
+        if (size_to_copy <= max) {
+            pj_memcpy(audp->buffer+tail, frame.buf, size_to_copy);
+        } else {
+            pj_memcpy(audp->buffer+tail, frame.buf, max);
+            pj_memcpy(audp->buffer, (char*)frame.buf + max,
+                      size_to_copy - max);
+        }
+    } else {
+        pj_assert(audp->head - tail >= size_to_copy);
+        pj_memcpy(audp->buffer+tail, frame.buf, size_to_copy);
+    }
+    audp->size += size_to_copy;
+
+    return PJ_SUCCESS;
+}
+
+/* Fill up the circular buffer. This function contains the logic for the
+ * sync
+ */
+static pj_status_t aud_proxy_fill_buffer(pjmedia_port *this_port,
+                                         unsigned vid_med_time)
+{
+    struct aud_proxy *audp = (struct aud_proxy*) this_port;
+    unsigned med_time, min_size;
+    pj_status_t status;
+
+
+    med_time = audp->ts.u64 * 1000 /
+               this_port->info.fmt.det.aud.clock_rate;
+
+    min_size =  this_port->info.fmt.det.aud.avg_bps * SYNC_MIN_BUF_SIZE_MSEC /
+                1000 / 8;
+
+    if (vid_med_time > med_time + audp->adi->max_unsync) {
+        /* Video is running too fast, discard some audio to keep up */
+        if (med_time - audp->last_discard > SYNC_DISCARD_INTERVAL) {
+            AVI_LOG_EVENT((THIS_FILE,
+                      "Audio is too slow by %u msec, discarding 1 frame",
+                      vid_med_time - med_time));
+            aud_proxy_discard(this_port, 0);
+            audp->last_discard = med_time;
+
+            med_time = audp->ts.u64 * 1000 /
+                       this_port->info.fmt.det.aud.clock_rate;
+        }
+
+        /* Fill in buffer with minimum size */
+        while (audp->size < min_size) {
+            status = aud_proxy_fill_one_frame(this_port);
+            if (status != PJ_SUCCESS)
+                break;
+        }
+
+    } else if (vid_med_time + audp->adi->max_unsync < med_time) {
+        /* Video is running too slow, don't fill anything to the buffer */
+        if (med_time - audp->last_skip > SYNC_DISCARD_INTERVAL) {
+            AVI_LOG_EVENT((THIS_FILE,
+                           "Audio is too fast by %u msec, skipping 1 frame",
+                           med_time - vid_med_time));
+            audp->skip_next = PJ_TRUE;
+            audp->last_skip = med_time;
+        }
+    } else {
+        /* Withing sync, fill up */
+        while (audp->size < min_size) {
+            status = aud_proxy_fill_one_frame(this_port);
+            if (status != PJ_SUCCESS)
+                break;
+        }
+    }
+
+    return status;
+}
+
+/* Create the audio proxy port for the source port */
+static pj_status_t aud_proxy_create(pj_pool_t *pool, pjmedia_port *source,
+                                    struct aud_proxy **p_audp)
+{
+    struct aud_proxy *audp;
+    pjmedia_audio_format_detail *afd;
+
+    afd = &source->info.fmt.det.aud;
+
+    audp = PJ_POOL_ZALLOC_T(pool, struct aud_proxy);
+    audp->source = source;
+    audp->cap = afd->avg_bps * SYNC_BUF_SIZE_MSEC / 1000 / 8;
+    audp->avg_frame_size = PJMEDIA_FSZ(afd->avg_bps, afd->frame_time_usec);
+    audp->samples_per_frame = PJMEDIA_SPF(afd->clock_rate,
+                                          afd->frame_time_usec,
+                                          afd->channel_count);
+
+    pj_memcpy(&audp->base.info, &source->info,
+              sizeof(pjmedia_port_info));
+    pjmedia_format_copy(&audp->base.info.fmt, &source->info.fmt);
+
+    audp->base.get_frame = &aud_proxy_get_frame;
+    audp->base.on_destroy = &aud_proxy_destroy;
+
+    audp->frame_buffer = (pj_uint8_t*) pj_pool_alloc(pool,
+                                                     audp->avg_frame_size);
+    audp->buffer = (pj_uint8_t*) pj_pool_alloc(pool, audp->cap);
+
+    *p_audp = audp;
+
+    return PJ_SUCCESS;
+}
+
+
+/****************************************************************************
  * Factory operations
  */
 
@@ -245 +504 @@
     param->cap_id = index;
     param->rend_id = PJMEDIA_VID_INVALID_DEV;
     param->flags = PJMEDIA_VID_DEV_CAP_FORMAT;
-    param->clock_rate = DEFAULT_CLOCK_RATE;
+    param->clock_rate = VID_CLOCK_RATE;
     pj_memcpy(&param->fmt, &di->info.fmt[0], sizeof(param->fmt));
 
     return PJ_SUCCESS;
@@ -326 +585 @@
 {
     pjmedia_vid_dev_factory *f;
     struct avi_factory *cf;
-    unsigned local_idx;
+    unsigned i, local_idx;
     struct avi_dev_info *adi;
     pj_status_t status;
 
@@ -346 +605 @@
     pj_bzero(prm, sizeof(*prm));
     prm->path = adi->fpath;
     prm->title = adi->title;
-    prm->avi_streams = adi->avi;
+    prm->aud_port_cnt = adi->aud_cnt;
+    for (i=0; i<adi->aud_cnt && i<PJ_ARRAY_SIZE(prm->aud_port); ++i)
+        prm->aud_port[i] = adi->aud_port[i];
 
     return PJ_SUCCESS;
 }
@@ -354 +615 @@
 PJ_DEF(void) pjmedia_avi_dev_param_default(pjmedia_avi_dev_param *p)
 {
     pj_bzero(p, sizeof(*p));
+    p->max_unsync = SYNC_MAX_UNSYNC_MSEC;
 }
 
 /* API: configure the AVI */
@@ -363 +625 @@
 {
     pjmedia_vid_dev_index id;
     struct avi_factory *cf = (struct avi_factory*)f;
-    unsigned local_idx;
+    unsigned i, cnt, local_idx;
     struct avi_dev_info *adi = NULL;
     pjmedia_format avi_fmt;
     const pjmedia_video_format_info *vfi;
@@ -399 +661 @@
     /* Reinit */
     PJ_ASSERT_RETURN(p->path.slen, PJ_EINVAL);
     adi->pool = pj_pool_create(cf->pf, "avidi%p", 512, 512, NULL);
+    adi->max_unsync = p->max_unsync;
 
 
     /* Open the AVI */
@@ -495 +758 @@
     adi->info.fmt_cnt = 1;
     pjmedia_format_copy(&adi->info.fmt[0], &avi_fmt);
 
+    /* Save the audio port in the avi, or if A/V sync is desired, create
+     * proxy the audio ports for each audio port
+     */
+    cnt = pjmedia_avi_streams_get_num_streams(adi->avi);
+    for (i=0; i<cnt && i<PJ_ARRAY_SIZE(adi->aud_port); ++i) {
+        pjmedia_avi_stream *as;
+
+        as = pjmedia_avi_streams_get_stream(adi->avi, i);
+        if (as) {
+            pjmedia_port *port;
+
+            port = pjmedia_avi_stream_get_port(as);
+            if (port->info.fmt.type == PJMEDIA_TYPE_AUDIO) {
+                if (p->sync_av) {
+                    struct aud_proxy *audp;
+                    status = aud_proxy_create(adi->pool, port, &audp);
+                    if (status == PJ_SUCCESS) {
+                        audp->adi = adi;
+                        adi->aud_port[adi->aud_cnt++] = &audp->base;
+                        p->aud_port[p->aud_port_cnt++] = &audp->base;
+                    }
+                } else {
+                    adi->aud_port[adi->aud_cnt++] = port;
+                    p->aud_port[p->aud_port_cnt++] = port;
+                }
+            }
+        }
+    }
+
     /* Set out vars */
     if (p_id)
         *p_id = id;
-    p->avi_streams = adi->avi;
     if (p->title.slen == 0)
         p->title = adi->title;
 
@@ -615 +906 @@
                                          pjmedia_frame *frame)
 {
     struct avi_dev_strm *stream = (struct avi_dev_strm*)strm;
-    
+    pj_status_t status;
+
     if (stream->adi->codec) {
         pjmedia_frame enc_frame;
-        pj_status_t status;
 
         enc_frame.buf = stream->adi->enc_buf;
         enc_frame.size = stream->adi->enc_buf_size;
@@ -626 +917 @@
         if (status != PJ_SUCCESS)
             return status;
 
-        return pjmedia_vid_codec_decode(stream->adi->codec, 1, &enc_frame,
-                                        frame->size, frame);
+        status = pjmedia_vid_codec_decode(stream->adi->codec, 1, &enc_frame,
+                                          frame->size, frame);
     } else {
-        return pjmedia_port_get_frame(stream->adi->vid, frame);
+        status = pjmedia_port_get_frame(stream->adi->vid, frame);
     }
+
+    if (status != PJ_SUCCESS)
+        return status;
+
+    /* Calculate video media time in msec. The audio ports will refer to
+     * this field for synchronization.
+     */
+    stream->adi->vid_media_time = frame->timestamp.u64 * 1000 /
+                                  VID_CLOCK_RATE;
+
+    return PJ_SUCCESS;
 }
 
 /* API: Start stream. */