Changeset 4836 for pjproject

Timestamp:

May 6, 2014 12:55:49 PM (10 years ago)

Author:

nanang

Message:

Re #1762: Add I420/IYUV output format to capture devices.

File:

• pjproject/trunk/pjmedia/src/pjmedia-videodev/ios_dev.m (modified) (19 diffs)

pjproject/trunk/pjmedia/src/pjmedia-videodev/ios_dev.m

@@ -46 +46 @@
 static ios_fmt_info ios_fmts[] =
 {
-    { PJMEDIA_FORMAT_BGRA, kCVPixelFormatType_32BGRA }
+    { PJMEDIA_FORMAT_BGRA, kCVPixelFormatType_32BGRA },
+    { PJMEDIA_FORMAT_I420, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange }
 };
 
@@ -87 +88 @@
 
     pjmedia_rect_size       size;
-    pj_uint8_t              bpp;
     unsigned                bytes_per_row;
-    unsigned                frame_size;
+    unsigned                frame_size;         /**< Frame size (bytes)*/
+    pj_bool_t               is_planar;
     
     AVCaptureSession            *cap_session;
@@ -95 +96 @@
     AVCaptureVideoDataOutput    *video_output;
     VOutDelegate                *vout_delegate;
+    void                        *capture_buf;
     
     void                *render_buf;
@@ -210 +212 @@
     qdi->info.dir = PJMEDIA_DIR_RENDER;
     qdi->info.has_callback = PJ_FALSE;
-    qdi->info.caps = PJMEDIA_VID_DEV_CAP_OUTPUT_WINDOW;
     
     /* Init input device */
@@ -230 +231 @@
             qdi = &qf->dev_info[qf->dev_count++];
             pj_bzero(qdi, sizeof(*qdi));
-            pj_ansi_strncpy(qdi->info.name, [[device localizedName] UTF8String],
+            pj_ansi_strncpy(qdi->info.name, [device.localizedName UTF8String],
                             sizeof(qdi->info.name));
             pj_ansi_strncpy(qdi->info.driver, "iOS", sizeof(qdi->info.driver));
@@ -236 +237 @@
             qdi->info.has_callback = PJ_TRUE;
             qdi->info.caps = PJMEDIA_VID_DEV_CAP_INPUT_PREVIEW |
-                             PJMEDIA_VID_DEV_CAP_SWITCH |
-                             PJMEDIA_VID_DEV_CAP_OUTPUT_WINDOW;
+                             PJMEDIA_VID_DEV_CAP_SWITCH;
             qdi->dev = device;
         }
@@ -252 +252 @@
     for (i = 0; i < qf->dev_count; i++) {
         qdi = &qf->dev_info[i];
-        qdi->info.fmt_cnt = PJ_ARRAY_SIZE(ios_fmts);        
         qdi->info.caps |= PJMEDIA_VID_DEV_CAP_FORMAT |
+                          PJMEDIA_VID_DEV_CAP_OUTPUT_WINDOW |
                           PJMEDIA_VID_DEV_CAP_OUTPUT_RESIZE |
                           PJMEDIA_VID_DEV_CAP_OUTPUT_POSITION |
@@ -260 +260 @@
         
         for (l = 0; l < PJ_ARRAY_SIZE(ios_fmts); l++) {
-            pjmedia_format *fmt = &qdi->info.fmt[l];
+            pjmedia_format *fmt;
+            
+            /* Simple renderer UIView only supports BGRA */
+            if (qdi->info.dir == PJMEDIA_DIR_RENDER &&
+                ios_fmts[l].pjmedia_format != PJMEDIA_FORMAT_BGRA)
+            {
+                continue;
+            }
+                
+            fmt = &qdi->info.fmt[qdi->info.fmt_cnt++];
             pjmedia_format_init_video(fmt,
                                       ios_fmts[l].pjmedia_format,
@@ -378 +387 @@
                       fromConnection:(AVCaptureConnection *)connection
 {
-    pjmedia_frame frame;
-    CVImageBufferRef imageBuffer;
+    pjmedia_frame frame = {0};
+    CVImageBufferRef img;
 
     if (!sampleBuffer)
@@ -385 +394 @@
     
     /* Get a CMSampleBuffer's Core Video image buffer for the media data */
-    imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); 
+    img = CMSampleBufferGetImageBuffer(sampleBuffer);
     
     /* Lock the base address of the pixel buffer */
-    CVPixelBufferLockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);
+    CVPixelBufferLockBaseAddress(img, kCVPixelBufferLock_ReadOnly);
     
     frame.type = PJMEDIA_FRAME_TYPE_VIDEO;
-    frame.buf = CVPixelBufferGetBaseAddress(imageBuffer);
     frame.size = stream->frame_size;
-    frame.bit_info = 0;
     frame.timestamp.u64 = stream->frame_ts.u64;
+
+    if (stream->is_planar && stream->capture_buf) {
+        if (stream->param.fmt.id == PJMEDIA_FORMAT_I420) {
+            /* kCVPixelFormatType_420YpCbCr8BiPlanar* is NV12 */
+            pj_uint8_t *p, *p_end, *Y, *U, *V;
+            pj_size_t p_len;
+            
+            p = (pj_uint8_t*)CVPixelBufferGetBaseAddressOfPlane(img, 0);
+            p_len = stream->size.w * stream->size.h;
+            Y = (pj_uint8_t*)stream->capture_buf;
+            U = Y + p_len;
+            V = U + p_len/4;
+            pj_memcpy(Y, p, p_len);
+            
+            p = (pj_uint8_t*)CVPixelBufferGetBaseAddressOfPlane(img, 1);
+            p_len >>= 1;
+            p_end = p + p_len;
+            while (p < p_end) {
+                *U++ = *p++;
+                *V++ = *p++;
+            }
+
+            frame.buf = stream->capture_buf;
+        }
+    } else {
+        frame.buf = CVPixelBufferGetBaseAddress(img);
+    }
     
     if (stream->vid_cb.capture_cb) {
@@ -411 +445 @@
     
     /* Unlock the pixel buffer */
-    CVPixelBufferUnlockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);
+    CVPixelBufferUnlockBaseAddress(img, kCVPixelBufferLock_ReadOnly);
 }
 @end
@@ -508 +542 @@
     vfd = pjmedia_format_get_video_format_detail(&strm->param.fmt, PJ_TRUE);
     pj_memcpy(&strm->size, &vfd->size, sizeof(vfd->size));
-    strm->bpp = vfi->bpp;
-    strm->bytes_per_row = strm->size.w * strm->bpp / 8;
+    strm->bytes_per_row = strm->size.w * vfi->bpp / 8;
     strm->frame_size = strm->bytes_per_row * strm->size.h;
     strm->ts_inc = PJMEDIA_SPF2(param->clock_rate, &vfd->fps, 1);
+    strm->is_planar = vfi->plane_cnt > 1;
 
     if (param->dir & PJMEDIA_DIR_CAPTURE) {
@@ -526 +560 @@
         vfd->size.h = 288;
         strm->size = vfd->size;
-        strm->bytes_per_row = strm->size.w * strm->bpp / 8;
+        strm->bytes_per_row = strm->size.w * vfi->bpp / 8;
         strm->frame_size = strm->bytes_per_row * strm->size.h;
         
@@ -578 +612 @@
                           kCVPixelBufferPixelFormatTypeKey, nil];
         
+        /* Prepare capture buffer if it's planar format */
+        if (strm->is_planar)
+            strm->capture_buf = pj_pool_alloc(strm->pool, strm->frame_size);
+        
         /* Native preview */
         if (param->flags & PJMEDIA_VID_DEV_CAP_INPUT_PREVIEW) {
@@ -677 +715 @@
             ios_init_view(strm);
             
+            CALayer *view_layer = strm->render_view.layer;
+            CGRect r = strm->render_view.bounds;
+            
             /* Preview layer instantiation should be in main thread! */
             dispatch_async(dispatch_get_main_queue(), ^{
                 /* Create preview layer */
-                AVCaptureVideoPreviewLayer *previewLayer =
-                [AVCaptureVideoPreviewLayer layerWithSession:strm->cap_session];
+                AVCaptureVideoPreviewLayer *prev_layer =
+                            [AVCaptureVideoPreviewLayer
+                             layerWithSession:strm->cap_session];
                 
                 /* Attach preview layer to a UIView */
-                CGRect r = strm->render_view.bounds;
-                previewLayer.videoGravity = AVLayerVideoGravityResizeAspectFill;
-                previewLayer.frame = r;
-                [[strm->render_view layer] addSublayer:previewLayer];
+                prev_layer.videoGravity = AVLayerVideoGravityResizeAspectFill;
+                prev_layer.frame = r;
+                [view_layer addSublayer:prev_layer];
+                PJ_LOG(4, (THIS_FILE, "Native preview initialized"));
             });
-            
-            NSLog(@"Native preview initialized.");
             
             return PJ_SUCCESS;
@@ -721 +761 @@
             /* Ok, let's do the switch */
             AVCaptureDeviceInput *cur_dev_input = strm->dev_input;
-                    //[AVCaptureDeviceInput
-                    // deviceInputWithDevice:di[strm->param.cap_id].dev
-                    // error:&error];
             AVCaptureDeviceInput *new_dev_input =
                     [AVCaptureDeviceInput
@@ -750 +787 @@
                 return PJMEDIA_EVID_BADFORMAT;
         
-            vfi = pjmedia_get_video_format_info(pjmedia_video_format_mgr_instance(),
-                                                fmt->id);
+            vfi = pjmedia_get_video_format_info(
+                                        pjmedia_video_format_mgr_instance(),
+                                        fmt->id);
             if (!vfi)
                 return PJMEDIA_EVID_BADFORMAT;
@@ -759 +797 @@
             vfd = pjmedia_format_get_video_format_detail(fmt, PJ_TRUE);
             pj_memcpy(&strm->size, &vfd->size, sizeof(vfd->size));
-            strm->bytes_per_row = strm->size.w * strm->bpp / 8;
+            strm->bytes_per_row = strm->size.w * vfi->bpp / 8;
             strm->frame_size = strm->bytes_per_row * strm->size.h;
             if (strm->render_buf_size < strm->frame_size) {
+                /* Realloc only when needed */
                 strm->render_buf = pj_pool_alloc(strm->pool, strm->frame_size);
                 strm->render_buf_size = strm->frame_size;
                 CGDataProviderRelease(strm->render_data_provider);
                 strm->render_data_provider = CGDataProviderCreateWithData(NULL,
-                                                        strm->render_buf, strm->frame_size,
+                                                        strm->render_buf,
+                                                        strm->frame_size,
                                                         NULL);
             }
@@ -777 +817 @@
             UIView *view = (UIView *)pval;
             strm->param.window.info.ios.window = (void *)pval;
-            dispatch_async(dispatch_get_main_queue(),
-                           ^{[view addSubview:strm->render_view];});
+            dispatch_async(dispatch_get_main_queue(), ^{
+                [view addSubview:strm->render_view];
+            });
             return PJ_SUCCESS;
         }