Ticket #1478: av_dev_synch.diff
File av_dev_synch.diff, 15.8 KB (added by bennylp, 13 years ago) |
---|
-
pjsip-apps/src/pjsua/pjsua_app.c
5762 5762 5763 5763 pjmedia_avi_dev_param_default(&avdp); 5764 5764 avdp.path = app_config.avi[i].path; 5765 avdp.sync_av = PJ_TRUE; 5766 if (app_config.null_audio) { 5767 /* null audio shouldn't have burst so we can have 5768 * better sync here. 5769 */ 5770 avdp.max_unsync = 80; 5771 } 5765 5772 5766 5773 status = pjmedia_avi_dev_alloc(avi_factory, &avdp, &avid); 5767 5774 if (status != PJ_SUCCESS) { … … 5778 5785 if (app_config.avi_def_idx == PJSUA_INVALID_ID) 5779 5786 app_config.avi_def_idx = i; 5780 5787 5781 strm_cnt = pjmedia_avi_streams_get_num_streams(avdp.avi_streams);5788 strm_cnt = avdp.aud_port_cnt; 5782 5789 for (strm_idx=0; strm_idx<strm_cnt; ++strm_idx) { 5783 5790 pjmedia_port *aud; 5784 5791 pjmedia_format *fmt; 5785 5792 pjsua_conf_port_id slot; 5786 5793 char fmt_name[5]; 5787 5794 5788 aud = pjmedia_avi_streams_get_stream(avdp.avi_streams, 5789 strm_idx); 5795 aud = avdp.aud_port[strm_idx]; 5790 5796 fmt = &aud->info.fmt; 5791 5797 5792 5798 pjmedia_fourcc_name(fmt->id, fmt_name); -
pjmedia/include/pjmedia-videodev/avi_dev.h
56 56 pj_str_t title; 57 57 58 58 /** 59 * The underlying AVI streams created by the device. If the value is NULL, 60 * that means the device has not been configured yet. Application can use 61 * this field to retrieve the audio stream of the AVI. This setting is 62 * "get"-only and will be ignored in "set capability" operation. 59 * Keep audio and video synchronized. 60 * 61 * Default: PJ_FALSE 63 62 */ 64 pj media_avi_streams *avi_streams;63 pj_bool_t sync_av; 65 64 65 /** 66 * Maximum unsynchronization delay between audio and video that is allowed 67 * before sync algorithm kicks in. This value should be larger than the 68 * sound device burst. Setting this to low value will cause too many 69 * synchronization attempts, while setting this too high will cause unsync 70 * between audio and video. Value is in milliseconds. 71 * 72 * Default: 200 73 */ 74 unsigned max_unsync; 75 76 /** 77 * Number of audio ports from the AVI. This value will be filled by the 78 * device. 79 */ 80 unsigned aud_port_cnt; 81 82 /** 83 * Array of audio ports from the AVI. This value will be filled by the 84 * device. 85 */ 86 pjmedia_port *aud_port[8]; 87 66 88 } pjmedia_avi_dev_param; 67 89 68 90 -
pjmedia/src/pjmedia-videodev/avi_dev.c
22 22 #include <pj/log.h> 23 23 #include <pj/os.h> 24 24 #include <pj/rand.h> 25 #include <pjmedia/errno.h> 25 26 #include <pjmedia/vid_codec.h> 26 27 27 28 #if defined(PJMEDIA_VIDEO_DEV_HAS_AVI) && PJMEDIA_VIDEO_DEV_HAS_AVI != 0 28 29 29 30 #define THIS_FILE "avi_dev.c" 30 31 #define DRIVER_NAME "AVIDev" 31 #define DEFAULT_CLOCK_RATE 90000 32 #define DEFAULT_WIDTH 640 33 #define DEFAULT_HEIGHT 480 34 #define DEFAULT_FPS 25 32 #define VID_CLOCK_RATE 90000 35 33 34 #define SYNC_BUF_SIZE_MSEC 1000 35 #define SYNC_MIN_BUF_SIZE_MSEC 300 /* should be larger than snd dev burst */ 36 #define SYNC_MAX_UNSYNC_MSEC 200 /* should be larger than snd dev burst */ 37 #define SYNC_DISCARD_INTERVAL 30 38 39 #define AVI_LOG_EVENT(expr) PJ_LOG(5,expr) 40 36 41 typedef struct avi_dev_strm avi_dev_strm; 37 42 38 43 /* avi_ device info */ … … 45 50 pj_str_t title; 46 51 pjmedia_avi_streams *avi; 47 52 pjmedia_port *vid; 53 unsigned vid_media_time; 54 unsigned max_unsync; 55 unsigned aud_cnt; 56 pjmedia_port *aud_port[8]; 48 57 avi_dev_strm *strm; 49 58 pjmedia_vid_codec *codec; 50 59 pj_uint8_t *enc_buf; … … 136 145 137 146 138 147 /**************************************************************************** 148 * Audio proxy object, is a "proxy" for each audio media port from the AVI, 149 * and it is (only) used when A/V synchronization is enabled in the param. 150 */ 151 struct aud_proxy 152 { 153 pjmedia_port base; 154 struct avi_dev_info *adi; 155 pjmedia_port *source; 156 pj_uint8_t *buffer; 157 pj_uint8_t *frame_buffer; 158 unsigned head, 159 size, 160 cap, /* Capacity */ 161 samples_per_frame, 162 avg_frame_size; 163 pj_timestamp ts; /* Timestamp of first sample in buf */ 164 pjmedia_frame_type frame_type; 165 pj_bool_t skip_next; 166 unsigned last_discard; /* Time of last discarding frame */ 167 unsigned last_skip; /* Time of last skipping frame */ 168 }; 169 170 static pj_status_t aud_proxy_fill_buffer(pjmedia_port *this_port, 171 unsigned vid_med_time); 172 173 /* This is the media port get_frame() callback. */ 174 static pj_status_t aud_proxy_get_frame(pjmedia_port *this_port, 175 pjmedia_frame *frame) 176 { 177 struct aud_proxy *audp = (struct aud_proxy*) this_port; 178 unsigned vid_media_time; 179 180 vid_media_time = audp->adi->vid_media_time; 181 aud_proxy_fill_buffer(this_port, vid_media_time); 182 183 if (audp->skip_next) { 184 /* We've been told to skip frame */ 185 frame->type = PJMEDIA_FRAME_TYPE_NONE; 186 frame->size = 0; 187 audp->skip_next = PJ_FALSE; 188 return PJ_SUCCESS; 189 } else if (audp->size == 0) { 190 AVI_LOG_EVENT((THIS_FILE, "AVI audio port empty")); 191 frame->type = PJMEDIA_FRAME_TYPE_NONE; 192 frame->size = 0; 193 return PJ_SUCCESS; 194 } 195 196 if (audp->size < frame->size) { 197 AVI_LOG_EVENT((THIS_FILE, "AVI audio port underflow")); 198 frame->type = PJMEDIA_FRAME_TYPE_AUDIO; 199 frame->size = audp->size; 200 frame->timestamp = audp->ts; 201 pj_memcpy(frame->buf, audp->buffer, audp->size); 202 audp->head = audp->size = 0; 203 audp->ts.u64 += (frame->size * 204 this_port->info.fmt.det.aud.clock_rate / 205 (this_port->info.fmt.det.aud.avg_bps / 8)); 206 return PJ_SUCCESS; 207 } 208 209 /* Copy one frame from circular buffer */ 210 if (audp->head + frame->size <= audp->cap) { 211 pj_memcpy(frame->buf, audp->buffer+audp->head, frame->size); 212 audp->head += frame->size; 213 } else { 214 unsigned chunk_size = audp->cap - audp->head; 215 pj_memcpy(frame->buf, audp->buffer+audp->head, chunk_size); 216 pj_memcpy(frame->buf + chunk_size, audp->buffer, 217 frame->size - chunk_size); 218 audp->head = frame->size - chunk_size; 219 } 220 221 /* Fix stuff */ 222 audp->size -= frame->size; 223 frame->timestamp = audp->ts; 224 frame->type = audp->frame_type; 225 audp->ts.u64 += audp->samples_per_frame; 226 227 return PJ_SUCCESS; 228 } 229 230 /* media port callback to destroy the port */ 231 static pj_status_t aud_proxy_destroy(pjmedia_port *this_port) 232 { 233 struct aud_proxy *audp = (struct aud_proxy*) this_port; 234 return pjmedia_port_destroy(audp->source); 235 } 236 237 /* internal: discard some samples from the circular buffer */ 238 static pj_status_t aud_proxy_discard(pjmedia_port *this_port, 239 unsigned size_to_discard) 240 { 241 struct aud_proxy *audp = (struct aud_proxy*) this_port; 242 243 if (size_to_discard == 0) 244 size_to_discard = audp->avg_frame_size; 245 if (size_to_discard > audp->size) 246 size_to_discard = audp->size; 247 248 audp->head = (audp->head + size_to_discard) % audp->cap; 249 audp->size -= size_to_discard; 250 audp->ts.u64 += (size_to_discard * 251 this_port->info.fmt.det.aud.clock_rate / 252 (this_port->info.fmt.det.aud.avg_bps / 8)); 253 254 return PJ_SUCCESS; 255 } 256 257 /* Get one frame from the AVI and put it to the circular buffer */ 258 static pj_status_t aud_proxy_fill_one_frame(pjmedia_port *this_port) 259 { 260 struct aud_proxy *audp = (struct aud_proxy*) this_port; 261 pjmedia_frame frame; 262 unsigned tail, size_to_copy; 263 pj_status_t status; 264 265 /* Get frame from source first */ 266 frame.buf = audp->frame_buffer; 267 frame.size = audp->avg_frame_size; 268 269 status = pjmedia_port_get_frame(audp->source, &frame); 270 if (status != PJ_SUCCESS) 271 return status; 272 273 if (frame.type == PJMEDIA_FRAME_TYPE_NONE) 274 return PJMEDIA_ENOTCOMPATIBLE; 275 276 /* Append to buffer */ 277 if (audp->cap - audp->size < audp->avg_frame_size) { 278 /* Buffer is full, needs to throw away frame */ 279 AVI_LOG_EVENT((THIS_FILE, "AVI audio port overflow")); 280 aud_proxy_discard(this_port, 0); 281 } 282 283 audp->frame_type = frame.type; 284 285 size_to_copy = frame.size; 286 tail = (audp->head + audp->size) % audp->cap; 287 if (tail >= audp->head) { 288 unsigned max = audp->cap - tail; 289 if (size_to_copy <= max) { 290 pj_memcpy(audp->buffer+tail, frame.buf, size_to_copy); 291 } else { 292 pj_memcpy(audp->buffer+tail, frame.buf, max); 293 pj_memcpy(audp->buffer, (char*)frame.buf + max, 294 size_to_copy - max); 295 } 296 } else { 297 pj_assert(audp->head - tail >= size_to_copy); 298 pj_memcpy(audp->buffer+tail, frame.buf, size_to_copy); 299 } 300 audp->size += size_to_copy; 301 302 return PJ_SUCCESS; 303 } 304 305 /* Fill up the circular buffer. This function contains the logic for the 306 * sync 307 */ 308 static pj_status_t aud_proxy_fill_buffer(pjmedia_port *this_port, 309 unsigned vid_med_time) 310 { 311 struct aud_proxy *audp = (struct aud_proxy*) this_port; 312 unsigned med_time, min_size; 313 pj_status_t status; 314 315 316 med_time = audp->ts.u64 * 1000 / 317 this_port->info.fmt.det.aud.clock_rate; 318 319 min_size = this_port->info.fmt.det.aud.avg_bps * SYNC_MIN_BUF_SIZE_MSEC / 320 1000 / 8; 321 322 if (vid_med_time > med_time + audp->adi->max_unsync) { 323 /* Video is running too fast, discard some audio to keep up */ 324 if (med_time - audp->last_discard > SYNC_DISCARD_INTERVAL) { 325 AVI_LOG_EVENT((THIS_FILE, 326 "Audio is too slow by %u msec, discarding 1 frame", 327 vid_med_time - med_time)); 328 aud_proxy_discard(this_port, 0); 329 audp->last_discard = med_time; 330 331 med_time = audp->ts.u64 * 1000 / 332 this_port->info.fmt.det.aud.clock_rate; 333 } 334 335 /* Fill in buffer with minimum size */ 336 while (audp->size < min_size) { 337 status = aud_proxy_fill_one_frame(this_port); 338 if (status != PJ_SUCCESS) 339 break; 340 } 341 342 } else if (vid_med_time + audp->adi->max_unsync < med_time) { 343 /* Video is running too slow, don't fill anything to the buffer */ 344 if (med_time - audp->last_skip > SYNC_DISCARD_INTERVAL) { 345 AVI_LOG_EVENT((THIS_FILE, 346 "Audio is too fast by %u msec, skipping 1 frame", 347 med_time - vid_med_time)); 348 audp->skip_next = PJ_TRUE; 349 audp->last_skip = med_time; 350 } 351 } else { 352 /* Withing sync, fill up */ 353 while (audp->size < min_size) { 354 status = aud_proxy_fill_one_frame(this_port); 355 if (status != PJ_SUCCESS) 356 break; 357 } 358 } 359 360 return status; 361 } 362 363 /* Create the audio proxy port for the source port */ 364 static pj_status_t aud_proxy_create(pj_pool_t *pool, pjmedia_port *source, 365 struct aud_proxy **p_audp) 366 { 367 struct aud_proxy *audp; 368 pjmedia_audio_format_detail *afd; 369 370 afd = &source->info.fmt.det.aud; 371 372 audp = PJ_POOL_ZALLOC_T(pool, struct aud_proxy); 373 audp->source = source; 374 audp->cap = afd->avg_bps * SYNC_BUF_SIZE_MSEC / 1000 / 8; 375 audp->avg_frame_size = PJMEDIA_FSZ(afd->avg_bps, afd->frame_time_usec); 376 audp->samples_per_frame = PJMEDIA_SPF(afd->clock_rate, 377 afd->frame_time_usec, 378 afd->channel_count); 379 380 pj_memcpy(&audp->base.info, &source->info, 381 sizeof(pjmedia_port_info)); 382 pjmedia_format_copy(&audp->base.info.fmt, &source->info.fmt); 383 384 audp->base.get_frame = &aud_proxy_get_frame; 385 audp->base.on_destroy = &aud_proxy_destroy; 386 387 audp->frame_buffer = (pj_uint8_t*) pj_pool_alloc(pool, 388 audp->avg_frame_size); 389 audp->buffer = (pj_uint8_t*) pj_pool_alloc(pool, audp->cap); 390 391 *p_audp = audp; 392 393 return PJ_SUCCESS; 394 } 395 396 397 /**************************************************************************** 139 398 * Factory operations 140 399 */ 141 400 … … 245 504 param->cap_id = index; 246 505 param->rend_id = PJMEDIA_VID_INVALID_DEV; 247 506 param->flags = PJMEDIA_VID_DEV_CAP_FORMAT; 248 param->clock_rate = DEFAULT_CLOCK_RATE;507 param->clock_rate = VID_CLOCK_RATE; 249 508 pj_memcpy(¶m->fmt, &di->info.fmt[0], sizeof(param->fmt)); 250 509 251 510 return PJ_SUCCESS; … … 326 585 { 327 586 pjmedia_vid_dev_factory *f; 328 587 struct avi_factory *cf; 329 unsigned local_idx;588 unsigned i, local_idx; 330 589 struct avi_dev_info *adi; 331 590 pj_status_t status; 332 591 … … 346 605 pj_bzero(prm, sizeof(*prm)); 347 606 prm->path = adi->fpath; 348 607 prm->title = adi->title; 349 prm->avi_streams = adi->avi; 608 prm->aud_port_cnt = adi->aud_cnt; 609 for (i=0; i<adi->aud_cnt && i<PJ_ARRAY_SIZE(prm->aud_port); ++i) 610 prm->aud_port[i] = adi->aud_port[i]; 350 611 351 612 return PJ_SUCCESS; 352 613 } … … 354 615 PJ_DEF(void) pjmedia_avi_dev_param_default(pjmedia_avi_dev_param *p) 355 616 { 356 617 pj_bzero(p, sizeof(*p)); 618 p->max_unsync = SYNC_MAX_UNSYNC_MSEC; 357 619 } 358 620 359 621 /* API: configure the AVI */ … … 363 625 { 364 626 pjmedia_vid_dev_index id; 365 627 struct avi_factory *cf = (struct avi_factory*)f; 366 unsigned local_idx;628 unsigned i, cnt, local_idx; 367 629 struct avi_dev_info *adi = NULL; 368 630 pjmedia_format avi_fmt; 369 631 const pjmedia_video_format_info *vfi; … … 399 661 /* Reinit */ 400 662 PJ_ASSERT_RETURN(p->path.slen, PJ_EINVAL); 401 663 adi->pool = pj_pool_create(cf->pf, "avidi%p", 512, 512, NULL); 664 adi->max_unsync = p->max_unsync; 402 665 403 666 404 667 /* Open the AVI */ … … 495 758 adi->info.fmt_cnt = 1; 496 759 pjmedia_format_copy(&adi->info.fmt[0], &avi_fmt); 497 760 761 /* Save the audio port in the avi, or if A/V sync is desired, create 762 * proxy the audio ports for each audio port 763 */ 764 cnt = pjmedia_avi_streams_get_num_streams(adi->avi); 765 for (i=0; i<cnt && i<PJ_ARRAY_SIZE(adi->aud_port); ++i) { 766 pjmedia_avi_stream *as; 767 768 as = pjmedia_avi_streams_get_stream(adi->avi, i); 769 if (as) { 770 pjmedia_port *port; 771 772 port = pjmedia_avi_stream_get_port(as); 773 if (port->info.fmt.type == PJMEDIA_TYPE_AUDIO) { 774 if (p->sync_av) { 775 struct aud_proxy *audp; 776 status = aud_proxy_create(adi->pool, port, &audp); 777 if (status == PJ_SUCCESS) { 778 audp->adi = adi; 779 adi->aud_port[adi->aud_cnt++] = &audp->base; 780 p->aud_port[p->aud_port_cnt++] = &audp->base; 781 } 782 } else { 783 adi->aud_port[adi->aud_cnt++] = port; 784 p->aud_port[p->aud_port_cnt++] = port; 785 } 786 } 787 } 788 } 789 498 790 /* Set out vars */ 499 791 if (p_id) 500 792 *p_id = id; 501 p->avi_streams = adi->avi;502 793 if (p->title.slen == 0) 503 794 p->title = adi->title; 504 795 … … 615 906 pjmedia_frame *frame) 616 907 { 617 908 struct avi_dev_strm *stream = (struct avi_dev_strm*)strm; 618 909 pj_status_t status; 910 619 911 if (stream->adi->codec) { 620 912 pjmedia_frame enc_frame; 621 pj_status_t status;622 913 623 914 enc_frame.buf = stream->adi->enc_buf; 624 915 enc_frame.size = stream->adi->enc_buf_size; … … 626 917 if (status != PJ_SUCCESS) 627 918 return status; 628 919 629 returnpjmedia_vid_codec_decode(stream->adi->codec, 1, &enc_frame,630 frame->size, frame);920 status = pjmedia_vid_codec_decode(stream->adi->codec, 1, &enc_frame, 921 frame->size, frame); 631 922 } else { 632 returnpjmedia_port_get_frame(stream->adi->vid, frame);923 status = pjmedia_port_get_frame(stream->adi->vid, frame); 633 924 } 925 926 if (status != PJ_SUCCESS) 927 return status; 928 929 /* Calculate video media time in msec. The audio ports will refer to 930 * this field for synchronization. 931 */ 932 stream->adi->vid_media_time = frame->timestamp.u64 * 1000 / 933 VID_CLOCK_RATE; 934 935 return PJ_SUCCESS; 634 936 } 635 937 636 938 /* API: Start stream. */