Index: /pjproject/trunk/pjmedia/src/pjmedia-codec/vid_toolbox.m
===================================================================
--- /pjproject/trunk/pjmedia/src/pjmedia-codec/vid_toolbox.m	(revision 5792)
+++ /pjproject/trunk/pjmedia/src/pjmedia-codec/vid_toolbox.m	(revision 5793)
@@ -35,4 +35,6 @@
 
 #if (defined(PJ_DARWINOS) && PJ_DARWINOS != 0 && TARGET_OS_IPHONE)
+#import <UIKit/UIKit.h>
+
 #  define DEFAULT_WIDTH		352
 #  define DEFAULT_HEIGHT	288
@@ -480,88 +482,19 @@
 }
 
-static pj_status_t vtool_codec_open(pjmedia_vid_codec *codec,
-                                    pjmedia_vid_codec_param *codec_param )
-{
-    vtool_codec_data 		*vtool_data;
-    pjmedia_vid_codec_param	*param;
-    pjmedia_h264_packetizer_cfg  pktz_cfg;
-    pjmedia_vid_codec_h264_fmtp  h264_fmtp;
-    pj_status_t		 	 status;
-    CMVideoCodecType 		 codec_type;
+
+static OSStatus create_encoder(vtool_codec_data *vtool_data)
+{
+    pjmedia_vid_codec_param	*param = vtool_data->prm;
     CFDictionaryRef		 supported_prop;
     OSStatus 			 ret;
 
-    PJ_ASSERT_RETURN(codec && codec_param, PJ_EINVAL);
-
-    PJ_LOG(5,(THIS_FILE, "Opening codec.."));
-
-    vtool_data = (vtool_codec_data*) codec->codec_data;
-    vtool_data->prm = pjmedia_vid_codec_param_clone( vtool_data->pool,
-                                                     codec_param);
-    param = vtool_data->prm;
-
-    /* Parse remote fmtp */
-    pj_bzero(&h264_fmtp, sizeof(h264_fmtp));
-    status = pjmedia_vid_codec_h264_parse_fmtp(&param->enc_fmtp, &h264_fmtp);
-    if (status != PJ_SUCCESS)
-	return status;
-
-    /* Apply SDP fmtp to format in codec param */
-    if (!param->ignore_fmtp) {
-	status = pjmedia_vid_codec_h264_apply_fmtp(param);
-	if (status != PJ_SUCCESS)
-	    return status;
-    }
-
-    pj_bzero(&pktz_cfg, sizeof(pktz_cfg));
-    pktz_cfg.mtu = param->enc_mtu;
-    pktz_cfg.unpack_nal_start = 4;
-    /* Packetization mode */
-    if (h264_fmtp.packetization_mode == 0)
-	pktz_cfg.mode = PJMEDIA_H264_PACKETIZER_MODE_SINGLE_NAL;
-    else if (h264_fmtp.packetization_mode == 1)
-	pktz_cfg.mode = PJMEDIA_H264_PACKETIZER_MODE_NON_INTERLEAVED;
-    else
-	return PJ_ENOTSUP;
-    /* Video Toolbox encoder doesn't support setting maximum slice size,
-     * so we cannot use single NAL mode since the NAL size likely
-     * exceeds the MTU.
-     */
-    pktz_cfg.mode = PJMEDIA_H264_PACKETIZER_MODE_NON_INTERLEAVED;
-
-    status = pjmedia_h264_packetizer_create(vtool_data->pool, &pktz_cfg,
-                                            &vtool_data->pktz);
-    if (status != PJ_SUCCESS)
-	return status;
-
-    vtool_data->whole = (param->packing == PJMEDIA_VID_PACKING_WHOLE);
-    if (1) {
-        /* Init format info and apply-param of encoder */
-        const pjmedia_video_format_info *enc_vfi;
-    	pjmedia_video_apply_fmt_param    enc_vafp;
-
-    	enc_vfi = pjmedia_get_video_format_info(NULL,codec_param->dec_fmt.id);
-    	if (!enc_vfi)
-            return PJ_EINVAL;
-    
-    	pj_bzero(&enc_vafp, sizeof(enc_vafp));
-    	enc_vafp.size = codec_param->enc_fmt.det.vid.size;
-    	enc_vafp.buffer = NULL;
-    	status = (*enc_vfi->apply_fmt)(enc_vfi, &enc_vafp);
-    	if (status != PJ_SUCCESS)
-            return status;
-
-	vtool_data->enc_wxh = codec_param->enc_fmt.det.vid.size.w *
-			      codec_param->enc_fmt.det.vid.size.h;
-	vtool_data->enc_input_size = enc_vafp.framebytes;
-	if (!vtool_data->whole) {
-	    vtool_data->enc_buf_size = (unsigned)enc_vafp.framebytes;
-	    vtool_data->enc_buf = pj_pool_alloc(vtool_data->pool,
-					        vtool_data->enc_buf_size);
-	}
+    /* Destroy if initialized before */
+    if (vtool_data->enc) {
+	VTCompressionSessionInvalidate(vtool_data->enc);
+	CFRelease(vtool_data->enc);
+	vtool_data->enc = NULL;
     }
 
     /* Create encoder session */
-    codec_type = kCMVideoCodecType_H264;
     ret = VTCompressionSessionCreate(NULL, (int)param->enc_fmt.det.vid.size.w,
     				     (int)param->enc_fmt.det.vid.size.h, 
@@ -571,5 +504,5 @@
     if (ret != noErr) {
 	PJ_LOG(4,(THIS_FILE, "VTCompressionCreate failed, ret=%d", ret));
-	return PJMEDIA_CODEC_EFAILED;
+	return ret;
     }
 
@@ -626,5 +559,94 @@
 
     VTCompressionSessionPrepareToEncodeFrames(vtool_data->enc);
-    
+
+    return ret;
+}
+
+
+static pj_status_t vtool_codec_open(pjmedia_vid_codec *codec,
+                                    pjmedia_vid_codec_param *codec_param )
+{
+    vtool_codec_data 		*vtool_data;
+    pjmedia_vid_codec_param	*param;
+    pjmedia_h264_packetizer_cfg  pktz_cfg;
+    pjmedia_vid_codec_h264_fmtp  h264_fmtp;
+    pj_status_t		 	 status;
+    OSStatus			 ret;
+
+    PJ_ASSERT_RETURN(codec && codec_param, PJ_EINVAL);
+
+    PJ_LOG(5,(THIS_FILE, "Opening codec.."));
+
+    vtool_data = (vtool_codec_data*) codec->codec_data;
+    vtool_data->prm = pjmedia_vid_codec_param_clone( vtool_data->pool,
+                                                     codec_param);
+    param = vtool_data->prm;
+
+    /* Parse remote fmtp */
+    pj_bzero(&h264_fmtp, sizeof(h264_fmtp));
+    status = pjmedia_vid_codec_h264_parse_fmtp(&param->enc_fmtp, &h264_fmtp);
+    if (status != PJ_SUCCESS)
+	return status;
+
+    /* Apply SDP fmtp to format in codec param */
+    if (!param->ignore_fmtp) {
+	status = pjmedia_vid_codec_h264_apply_fmtp(param);
+	if (status != PJ_SUCCESS)
+	    return status;
+    }
+
+    pj_bzero(&pktz_cfg, sizeof(pktz_cfg));
+    pktz_cfg.mtu = param->enc_mtu;
+    pktz_cfg.unpack_nal_start = 4;
+    /* Packetization mode */
+    if (h264_fmtp.packetization_mode == 0)
+	pktz_cfg.mode = PJMEDIA_H264_PACKETIZER_MODE_SINGLE_NAL;
+    else if (h264_fmtp.packetization_mode == 1)
+	pktz_cfg.mode = PJMEDIA_H264_PACKETIZER_MODE_NON_INTERLEAVED;
+    else
+	return PJ_ENOTSUP;
+    /* Video Toolbox encoder doesn't support setting maximum slice size,
+     * so we cannot use single NAL mode since the NAL size likely
+     * exceeds the MTU.
+     */
+    pktz_cfg.mode = PJMEDIA_H264_PACKETIZER_MODE_NON_INTERLEAVED;
+
+    status = pjmedia_h264_packetizer_create(vtool_data->pool, &pktz_cfg,
+                                            &vtool_data->pktz);
+    if (status != PJ_SUCCESS)
+	return status;
+
+    vtool_data->whole = (param->packing == PJMEDIA_VID_PACKING_WHOLE);
+    if (1) {
+        /* Init format info and apply-param of encoder */
+        const pjmedia_video_format_info *enc_vfi;
+    	pjmedia_video_apply_fmt_param    enc_vafp;
+
+    	enc_vfi = pjmedia_get_video_format_info(NULL,codec_param->dec_fmt.id);
+    	if (!enc_vfi)
+            return PJ_EINVAL;
+    
+    	pj_bzero(&enc_vafp, sizeof(enc_vafp));
+    	enc_vafp.size = codec_param->enc_fmt.det.vid.size;
+    	enc_vafp.buffer = NULL;
+    	status = (*enc_vfi->apply_fmt)(enc_vfi, &enc_vafp);
+    	if (status != PJ_SUCCESS)
+            return status;
+
+	vtool_data->enc_wxh = codec_param->enc_fmt.det.vid.size.w *
+			      codec_param->enc_fmt.det.vid.size.h;
+	vtool_data->enc_input_size = enc_vafp.framebytes;
+	if (!vtool_data->whole) {
+	    vtool_data->enc_buf_size = (unsigned)enc_vafp.framebytes;
+	    vtool_data->enc_buf = pj_pool_alloc(vtool_data->pool,
+					        vtool_data->enc_buf_size);
+	}
+    }
+
+    /* Create encoder */
+    ret = create_encoder(vtool_data);
+    if (ret != noErr)
+	return PJMEDIA_CODEC_EFAILED;
+
     /* If available, use the "sprop-parameter-sets" fmtp from remote SDP
      * to create the decoder.
@@ -737,7 +759,22 @@
     NSDictionary *frm_prop = NULL;
     OSStatus ret;
+#if TARGET_OS_IPHONE
+    UIApplicationState state;
+#endif
  
     PJ_ASSERT_RETURN(codec && input && out_size && output && has_more,
                      PJ_EINVAL);
+
+#if TARGET_OS_IPHONE
+    /* Skip encoding if app is not active, i.e. in the bg. */
+    state = [UIApplication sharedApplication].applicationState;
+    if (state != UIApplicationStateActive) {
+    	*has_more = PJ_FALSE;
+    	output->size = 0;
+    	output->type = PJMEDIA_FRAME_TYPE_NONE;
+
+    	return PJ_SUCCESS;
+    }
+#endif
 
     vtool_data = (vtool_codec_data*) codec->codec_data;
@@ -810,4 +847,19 @@
     				    	  (__bridge CFDictionaryRef)frm_prop,
     				    	  NULL, NULL);
+    if (ret == kVTInvalidSessionErr) {
+	/* Reset compression session */
+        ret = create_encoder(vtool_data);
+	PJ_LOG(3,(THIS_FILE, "Encoder needs to be reset [1]: %s (%d)",
+		  (ret == noErr? "success": "fail"), ret));
+        if (ret == noErr) {
+	    /* Retry encoding the frame after successful encoder reset. */
+	    ret = VTCompressionSessionEncodeFrame(vtool_data->enc, image_buf,
+    				    	  	  ts, dur,
+    				    	  	  (__bridge CFDictionaryRef)
+    				    	  	  frm_prop,
+    				    	  	  NULL, NULL);
+    	}
+    }
+
     if (ret != noErr) {
         PJ_LOG(4,(THIS_FILE, "Failed to encode frame %d", ret));
@@ -819,4 +871,15 @@
     ts.flags = kCMTimeFlags_Indefinite;
     ret = VTCompressionSessionCompleteFrames(vtool_data->enc, ts);
+    if (ret == kVTInvalidSessionErr) {
+	/* Reset compression session */
+        ret = create_encoder(vtool_data);
+	PJ_LOG(3,(THIS_FILE, "Encoder needs to be reset [2]: %s (%d)",
+		  (ret == noErr? "success": "fail"), ret));
+        if (ret == PJ_SUCCESS) {
+	    /* Retry finishing the encoding after successful encoder reset. */
+	    ret = VTCompressionSessionCompleteFrames(vtool_data->enc, ts);
+    	}
+    }
+
     if (ret != noErr) {
         PJ_LOG(4,(THIS_FILE, "Failed to complete encoding %d", ret));
@@ -990,5 +1053,5 @@
      }
 
-    if (!vtool_data->dec ||
+    if (!vtool_data->dec || !vtool_data->dec_format ||
 	!CMFormatDescriptionEqual(dec_format, vtool_data->dec_format))
     {
@@ -1049,8 +1112,22 @@
     pj_bool_t decode_whole = DECODE_WHOLE;
     OSStatus ret;
+#if TARGET_OS_IPHONE
+    UIApplicationState state;
+#endif
 
     PJ_ASSERT_RETURN(codec && count && packets && out_size && output,
                      PJ_EINVAL);
     PJ_ASSERT_RETURN(output->buf, PJ_EINVAL);
+
+#if TARGET_OS_IPHONE
+    /* Skip decoding if app is not active, i.e. in the bg. */
+    state = [UIApplication sharedApplication].applicationState;
+    if (state != UIApplicationStateActive) {
+	output->type = PJMEDIA_FRAME_TYPE_NONE;
+	output->size = 0;
+	output->timestamp = packets[0].timestamp;
+    	return PJ_SUCCESS;
+    }
+#endif
 
     vtool_data = (vtool_codec_data*) codec->codec_data;
@@ -1200,4 +1277,20 @@
 		          vtool_data->dec, sample_buf, 0,
 			  NULL, NULL);
+		if (ret == kVTInvalidSessionErr) {
+		    if (vtool_data->dec_format)
+		        CFRelease(vtool_data->dec_format);
+		    vtool_data->dec_format = NULL;
+		    ret = create_decoder(vtool_data);
+		    PJ_LOG(3,(THIS_FILE, "Decoder needs to be reset: %s (%d)",
+		  	      (ret == noErr? "success": "fail"), ret));
+
+		    if (ret == noErr) {
+	    		/* Retry decoding the frame after successful reset */
+			ret = VTDecompressionSessionDecodeFrame(
+		          	  vtool_data->dec, sample_buf, 0,
+			  	  NULL, NULL);
+		    }
+		}
+
 		if (ret != noErr) {
 		    PJ_LOG(5,(THIS_FILE, "Failed to decode frame %d of size "
