Changeset 6094 for pjproject

Timestamp:

Oct 17, 2019 7:02:50 AM (5 years ago)

Author:

ming

Message:

Fixed #1778: Support for Voice Processing IO Audio Unit on Mac

Location:

pjproject/trunk

Files:

• pjmedia/include/pjmedia/audiodev.h (modified) (1 diff)
• pjmedia/src/pjmedia-audiodev/coreaudio_dev.m (modified) (30 diffs)
• pjsip/include/pjsua-lib/pjsua.h (modified) (1 diff)
• pjsip/include/pjsua2/endpoint.hpp (modified) (1 diff)

pjproject/trunk/pjmedia/include/pjmedia/audiodev.h

@@ -38 +38 @@
  * @defgroup s2_audio_device_reference Audio Device API Reference
  * @ingroup audio_device_api
- * @brief API Reference
+ * @brief Documentation and API Reference
  * @{
+ *
+ * @section ec_sec Hardware/Built-in Echo Cancellation
+ *
+ * On some platforms, audio device comes with built-in echo cancellation
+ * feature. This is usually done based on specific hardware configuration,
+ * such as the use of multiple microphones and/or a known fixed distance
+ * between the capture and playback device, in order to precalculate the
+ * echo time distance. Because of this, when using the hardware EC,
+ * users may not get the freedom to select their own audio devices.
+ * This is applicable for Mac (users must use default audio devices) and
+ * iOS (users must use the same built-in audio device).
+ *
+ * In PJMEDIA, applications wishing to use sofware echo instead can pass
+ * PJMEDIA_ECHO_USE_SW_ECHO when calling pjmedia_snd_port_create2().
  */
 

pjproject/trunk/pjmedia/src/pjmedia-audiodev/coreaudio_dev.m

@@ -101 +101 @@
 
     AudioComponent               io_comp;
+    pj_bool_t                    has_vpio;
     struct stream_list           streams;
 };
@@ -271 +272 @@
     if (cf->io_comp == NULL)
         return PJMEDIA_EAUD_INIT; // cannot find IO unit;
+
+    desc.componentSubType = kAudioUnitSubType_VoiceProcessingIO;
+    if (AudioComponentFindNext(NULL, &desc) != NULL)
+        cf->has_vpio = PJ_TRUE;
 
     status = ca_factory_refresh(f);
@@ -632 +637 @@
             }
         }
+        if (cf->has_vpio) {
+            cdi->info.caps |= PJMEDIA_AUD_DEV_CAP_EC;
+        }
 
         cf->dev_count++;
 
         PJ_LOG(4, (THIS_FILE, " dev_id %d: %s  (in=%d, out=%d) %dHz",
-               i,
+               cdi->dev_id,
                cdi->info.name,
                cdi->info.input_count,
@@ -1001 +1009 @@
 }
 
+/* Copy 16-bit signed int samples to a destination buffer, which can be
+ * either 16-bit signed int, or float.
+ */
+static void copy_samples(void *dst, unsigned *dst_pos, unsigned dst_elmt_size,
+                         pj_int16_t *src, unsigned nsamples)
+{
+   if (dst_elmt_size == sizeof(pj_int16_t)) {
+        /* Destination is also 16-bit signed int. */
+        pjmedia_copy_samples((pj_int16_t*)dst + *dst_pos, src, nsamples);
+   } else {
+        /* Convert it first to float. */
+        unsigned i;
+        float *fdst = (float *)dst;
+
+        pj_assert(dst_elmt_size == sizeof(Float32));
+
+        for (i = 0; i< nsamples; i++) {
+            /* Value needs to be between -1.0 to 1.0 */
+            fdst[*dst_pos + i] = (float)src[i] / 32768.0f;
+        }
+   }
+   *dst_pos += nsamples;
+}
+
 static OSStatus output_renderer(void                       *inRefCon,
                                 AudioUnitRenderActionFlags *ioActionFlags,
@@ -1011 +1043 @@
     pj_status_t status = 0;
     unsigned nsamples_req = inNumberFrames * stream->param.channel_count;
-    pj_int16_t *output = ioData->mBuffers[0].mData;
+    void *output = ioData->mBuffers[0].mData;
+    unsigned elmt_size = ioData->mBuffers[0].mDataByteSize / inNumberFrames;
+    unsigned output_pos = 0;
 
     pj_assert(!stream->quit_flag);
@@ -1037 +1071 @@
         /* samples buffered >= requested by sound device */
         if (stream->play_buf_count >= nsamples_req) {
-            pjmedia_copy_samples((pj_int16_t*)output, stream->play_buf,
-                                 nsamples_req);
+            copy_samples(output, &output_pos, elmt_size,
+                         stream->play_buf, nsamples_req);
             stream->play_buf_count -= nsamples_req;
             pjmedia_move_samples(stream->play_buf,
@@ -1049 +1083 @@
 
         /* samples buffered < requested by sound device */
-        pjmedia_copy_samples((pj_int16_t*)output, stream->play_buf,
-                             stream->play_buf_count);
+        copy_samples(output, &output_pos, elmt_size,
+                     stream->play_buf, stream->play_buf_count);
         nsamples_req -= stream->play_buf_count;
-        output = (pj_int16_t*)output + stream->play_buf_count;
         stream->play_buf_count = 0;
     }
@@ -1067 +1100 @@
 
         if (nsamples_req >= stream->param.samples_per_frame) {
-            frame.buf = output;
+            /* If the output buffer is 16-bit signed int, we can
+             * directly use the supplied buffer.
+             */
+            if (elmt_size == sizeof(pj_int16_t)) {
+                frame.buf = (pj_int16_t *)output + output_pos;
+            } else {
+                frame.buf = stream->play_buf;
+            }
             status = (*stream->play_cb)(stream->user_data, &frame);
             if (status != PJ_SUCCESS)
@@ -1076 +1116 @@
 
             nsamples_req -= stream->param.samples_per_frame;
-            output = (pj_int16_t*)output + stream->param.samples_per_frame;
+            if (elmt_size == sizeof(pj_int16_t)) {
+                output_pos += stream->param.samples_per_frame;
+            } else {
+                copy_samples(output, &output_pos, elmt_size, stream->play_buf,
+                             stream->param.samples_per_frame);
+            }
         } else {
             frame.buf = stream->play_buf;
@@ -1086 +1131 @@
                 pj_bzero(frame.buf, frame.size);
 
-            pjmedia_copy_samples((pj_int16_t*)output, stream->play_buf,
-                                 nsamples_req);
+            copy_samples(output, &output_pos, elmt_size,
+                         stream->play_buf, nsamples_req);
             stream->play_buf_count = stream->param.samples_per_frame -
                                      nsamples_req;
@@ -1293 +1338 @@
                                        &enable,
                                        sizeof(enable));
-        if (ostatus != noErr) {
+        if (ostatus != noErr && !strm->param.ec_enabled) {
             PJ_LOG(4, (THIS_FILE,
                    "Warning: cannot enable IO of capture device %d",
@@ -1308 +1353 @@
                                            &enable,
                                            sizeof(enable));
-            if (ostatus != noErr) {
+            if (ostatus != noErr && !strm->param.ec_enabled) {
                 PJ_LOG(4, (THIS_FILE,
                        "Warning: cannot disable IO of capture device %d",
@@ -1327 +1372 @@
                                        &enable,
                                        sizeof(enable));
-        if (ostatus != noErr) {
+        if (ostatus != noErr && !strm->param.ec_enabled)
+        {
             PJ_LOG(4, (THIS_FILE,
                    "Warning: cannot enable IO of playback device %d",
@@ -1336 +1382 @@
 
 #if COREAUDIO_MAC
-    PJ_LOG(5, (THIS_FILE, "Opening device %d", dev_id));
-    ostatus = AudioUnitSetProperty(*io_unit,
-                                   kAudioOutputUnitProperty_CurrentDevice,
-                                   kAudioUnitScope_Global,
-                                   0,
-                                   &dev_id,
-                                   sizeof(dev_id));
-    if (ostatus != noErr) {
-        return PJMEDIA_AUDIODEV_ERRNO_FROM_COREAUDIO(ostatus);
+    if (!strm->param.ec_enabled) {
+        /* When using VPIO on Mac, we shouldn't set the current device.
+         * Doing so will cause getting the buffer size later to fail
+         * with kAudioUnitErr_InvalidProperty error (-10879).
+         */
+        PJ_LOG(4, (THIS_FILE, "Setting current device %d", dev_id));
+        ostatus = AudioUnitSetProperty(*io_unit,
+                                       kAudioOutputUnitProperty_CurrentDevice,
+                                       kAudioUnitScope_Global,
+                                       0,
+                                       &dev_id,
+                                       sizeof(dev_id));
+        if (ostatus != noErr) {
+            PJ_LOG(3, (THIS_FILE, "Failed setting current device %d, "
+                       "error: %d", dev_id, ostatus));
+            return PJMEDIA_AUDIODEV_ERRNO_FROM_COREAUDIO(ostatus);
+        }
     }
 #endif
@@ -1353 +1407 @@
         UInt32 size;
 
-        /*
-         * Keep the sample rate from the device, otherwise we will confuse
-         * AUHAL
-         */
-        size = sizeof(AudioStreamBasicDescription);
-        ostatus = AudioUnitGetProperty(*io_unit,
-                                       kAudioUnitProperty_StreamFormat,
-                                       kAudioUnitScope_Input,
-                                       1,
-                                       &deviceFormat,
-                                       &size);
-        if (ostatus != noErr) {
-            return PJMEDIA_AUDIODEV_ERRNO_FROM_COREAUDIO(ostatus);
-        }
-        strm->streamFormat.mSampleRate = deviceFormat.mSampleRate;
+        if (!strm->param.ec_enabled) {
+            /* Keep the sample rate from the device, otherwise we will confuse
+             * AUHAL.
+             */
+            size = sizeof(AudioStreamBasicDescription);
+            ostatus = AudioUnitGetProperty(*io_unit,
+                                           kAudioUnitProperty_StreamFormat,
+                                           kAudioUnitScope_Input,
+                                           1,
+                                           &deviceFormat,
+                                           &size);
+            if (ostatus != noErr) {
+                PJ_LOG(3, (THIS_FILE, "Failed getting stream format of device"
+                                      " %d, error: %d", dev_id, ostatus));
+                return PJMEDIA_AUDIODEV_ERRNO_FROM_COREAUDIO(ostatus);
+            }
+            strm->streamFormat.mSampleRate = deviceFormat.mSampleRate;
+        }
 #endif
 
@@ -1381 +1438 @@
                                        sizeof(strm->streamFormat));
         if (ostatus != noErr) {
+            PJ_LOG(3, (THIS_FILE, "Failed setting stream format of device %d"
+                                  ", error: %d", dev_id, ostatus));
             return PJMEDIA_AUDIODEV_ERRNO_FROM_COREAUDIO(ostatus);
         }
@@ -1395 +1454 @@
         if (ostatus == noErr) {
             if (strm->streamFormat.mSampleRate != deviceFormat.mSampleRate) {
+                PJ_LOG(4, (THIS_FILE, "Creating audio resample from %d to %d",
+                           (int)deviceFormat.mSampleRate,
+                           (int)strm->streamFormat.mSampleRate));
                 pj_status_t rc = create_audio_resample(strm, &deviceFormat);
-                if (PJ_SUCCESS != rc)
+                if (PJ_SUCCESS != rc) {
+                    PJ_LOG(3, (THIS_FILE, "Failed creating resample %d",
+                               rc));
                     return rc;
+                }
             }
         } else {
+            PJ_LOG(3, (THIS_FILE, "Failed getting stream format of device %d"
+                                  " (2), error: %d", dev_id, ostatus));
             return PJMEDIA_AUDIODEV_ERRNO_FROM_COREAUDIO(ostatus);
         }
@@ -1407 +1474 @@
     if (dir & PJMEDIA_DIR_PLAYBACK) {
         AURenderCallbackStruct output_cb;
+        AudioStreamBasicDescription streamFormat = strm->streamFormat;
 
         /* Set the stream format */
+#if COREAUDIO_MAC
+        if (strm->param.ec_enabled) {
+            /* When using VPIO on Mac, we need to use float data. Using
+             * signed integer will generate no errors, but strangely,
+             * no sound will be played.
+             */
+            streamFormat.mFormatFlags      = kLinearPCMFormatFlagIsFloat |
+                                             kLinearPCMFormatFlagIsPacked;
+            streamFormat.mBitsPerChannel   = sizeof(float) * 8;
+            streamFormat.mBytesPerFrame    = streamFormat.mChannelsPerFrame *
+                                             streamFormat.mBitsPerChannel / 8;
+            streamFormat.mBytesPerPacket   = streamFormat.mBytesPerFrame *
+                                             streamFormat.mFramesPerPacket;
+        }
+#endif
         ostatus = AudioUnitSetProperty(*io_unit,
                                        kAudioUnitProperty_StreamFormat,
                                        kAudioUnitScope_Input,
                                        0,
-                                       &strm->streamFormat,
-                                       sizeof(strm->streamFormat));
+                                       &streamFormat,
+                                       sizeof(streamFormat));
         if (ostatus != noErr) {
             PJ_LOG(4, (THIS_FILE,
@@ -1431 +1514 @@
                                        sizeof(output_cb));
         if (ostatus != noErr) {
+            PJ_LOG(3, (THIS_FILE, "Failed setting render callback %d",
+                       ostatus));
             return PJMEDIA_AUDIODEV_ERRNO_FROM_COREAUDIO(ostatus);
         }
@@ -1437 +1522 @@
         strm->play_buf = (pj_int16_t*)pj_pool_alloc(strm->pool,
                          strm->param.samples_per_frame *
+                         (strm->param.ec_enabled? 2: 1) *
                          strm->param.bits_per_sample >> 3);
         if (!strm->play_buf)
@@ -1458 +1544 @@
                       kAudioOutputUnitProperty_SetInputCallback,
                       kAudioUnitScope_Global,
-                      0,
+                      1,
                       &input_cb,
                       sizeof(input_cb));
         if (ostatus != noErr) {
+            PJ_LOG(3, (THIS_FILE, "Failed setting input callback %d",
+                       ostatus));
             return PJMEDIA_AUDIODEV_ERRNO_FROM_COREAUDIO(ostatus);
         }
 
 #if COREAUDIO_MAC
+        /* Set device's buffer frame size */
+        size = sizeof(UInt32);
+        buf_size = (20 * strm->streamFormat.mSampleRate) / 1000;
+        ostatus = AudioUnitSetProperty (*io_unit,
+                                        kAudioDevicePropertyBufferFrameSize,
+                                        kAudioUnitScope_Global,
+                                        0,
+                                        &buf_size,
+                                        size);
+        if (ostatus != noErr) {
+            PJ_LOG(3, (THIS_FILE, "Failed setting buffer size %d",
+                       ostatus));
+            return PJMEDIA_AUDIODEV_ERRNO_FROM_COREAUDIO(ostatus);
+        }
+
         /* Get device's buffer frame size */
-        size = sizeof(UInt32);
         ostatus = AudioUnitGetProperty(*io_unit,
                                        kAudioDevicePropertyBufferFrameSize,
@@ -1474 +1576 @@
                                        &buf_size,
                                        &size);
-        if (ostatus != noErr)
-        {
+        if (ostatus != noErr) {
+            PJ_LOG(3, (THIS_FILE, "Failed getting buffer size %d",
+                       ostatus));
             return PJMEDIA_AUDIODEV_ERRNO_FROM_COREAUDIO(ostatus);
         }
@@ -1489 +1592 @@
         ab->mNumberChannels = strm->streamFormat.mChannelsPerFrame;
         ab->mDataByteSize = buf_size * ab->mNumberChannels *
-                            strm->param.bits_per_sample >> 3;
+                            strm->streamFormat.mBitsPerChannel >> 3;
         ab->mData = pj_pool_alloc(strm->pool,
                                   ab->mDataByteSize);
@@ -1522 +1625 @@
     ostatus = AudioUnitInitialize(*io_unit);
     if (ostatus != noErr) {
+        PJ_LOG(3, (THIS_FILE, "Failed initializing audio unit %d", ostatus));
         return PJMEDIA_AUDIODEV_ERRNO_FROM_COREAUDIO(ostatus);
     }
@@ -1563 +1667 @@
     strm->streamFormat.mChannelsPerFrame = param->channel_count;
     strm->streamFormat.mBytesPerFrame    = strm->streamFormat.mChannelsPerFrame
-                                           * strm->param.bits_per_sample >> 3;
+                                           * strm->streamFormat.mBitsPerChannel
+                                           / 8;
     strm->streamFormat.mFramesPerPacket  = 1;
     strm->streamFormat.mBytesPerPacket   = strm->streamFormat.mBytesPerFrame *
@@ -1580 +1685 @@
     }
     if (param->flags & PJMEDIA_AUD_DEV_CAP_EC) {
-        ca_stream_set_cap(&strm->base,
-                          PJMEDIA_AUD_DEV_CAP_EC,
-                          &param->ec_enabled);
+        status = ca_stream_set_cap(&strm->base,
+                                   PJMEDIA_AUD_DEV_CAP_EC,
+                                   &param->ec_enabled);
+        if (status != PJ_SUCCESS)
+            strm->param.ec_enabled = PJ_FALSE;
     } else {
         pj_bool_t ec = PJ_FALSE;
@@ -1589 +1696 @@
     }
 
+#if !TARGET_OS_IPHONE
+    if (strm->param.ec_enabled &&
+        param->rec_id != PJMEDIA_AUD_DEFAULT_CAPTURE_DEV &&
+        param->play_id != PJMEDIA_AUD_DEFAULT_PLAYBACK_DEV)
+    {
+        PJ_LOG(4, (THIS_FILE, "Warning: audio device id settings are "
+                              "ignored when using VPIO"));
+    }
+#endif
+
     strm->io_units[0] = strm->io_units[1] = NULL;
-    if (param->dir == PJMEDIA_DIR_CAPTURE_PLAYBACK &&
-        param->rec_id == param->play_id)
+    if ((param->dir == PJMEDIA_DIR_CAPTURE_PLAYBACK &&
+         param->rec_id == param->play_id) ||
+        (param->flags & PJMEDIA_AUD_DEV_CAP_EC && param->ec_enabled))
     {
-        /* If both input and output are on the same device, only create
-         * one audio unit to interface with the device.
+        /* If both input and output are on the same device or if EC is enabled,
+         * only create one audio unit to interface with the device(s).
          */
         status = create_audio_unit(cf->io_comp,
@@ -1891 +2009 @@
     PJ_ASSERT_RETURN(s && pval, PJ_EINVAL);
 
-#if COREAUDIO_MAC
-    if (cap==PJMEDIA_AUD_DEV_CAP_OUTPUT_VOLUME_SETTING &&
-        (strm->param.dir & PJMEDIA_DIR_PLAYBACK))
-    {
-        OSStatus ostatus;
-        Float32 volume = *(unsigned*)pval;
-
-        /* Output volume setting */
-        volume /= 100.0;
-        ostatus = AudioUnitSetProperty (strm->io_units[1] ? strm->io_units[1] :
-                                        strm->io_units[0],
-                                        kAudioDevicePropertyVolumeScalar,
-                                        kAudioUnitScope_Output,
-                                        0,
-                                        &volume,
-                                        sizeof(Float32));
-        if (ostatus != noErr) {
-            return PJMEDIA_AUDIODEV_ERRNO_FROM_COREAUDIO(ostatus);
-        }
-        strm->param.output_vol = *(unsigned*)pval;
-        return PJ_SUCCESS;
-    }
-
-#else
     if (cap==PJMEDIA_AUD_DEV_CAP_EC) {
         AudioComponentDescription desc;
@@ -1921 +2015 @@
         desc.componentType = kAudioUnitType_Output;
         desc.componentSubType = (*(pj_bool_t*)pval)?
-        kAudioUnitSubType_VoiceProcessingIO :
-        kAudioUnitSubType_RemoteIO;
+                                kAudioUnitSubType_VoiceProcessingIO :
+#if COREAUDIO_MAC
+                                kAudioUnitSubType_HALOutput;
+#else
+                                kAudioUnitSubType_RemoteIO;
+#endif
         desc.componentManufacturer = kAudioUnitManufacturer_Apple;
         desc.componentFlags = 0;
@@ -1935 +2033 @@
         PJ_LOG(4, (THIS_FILE, "Using %s audio unit",
                    (desc.componentSubType ==
-                    kAudioUnitSubType_RemoteIO? "RemoteIO":
-                    "VoiceProcessingIO")));
+                    kAudioUnitSubType_VoiceProcessingIO?
+                    "VoiceProcessingIO": "default")));
         
         return PJ_SUCCESS;
-    } else if ((cap==PJMEDIA_AUD_DEV_CAP_INPUT_LATENCY &&
+    }
+#if COREAUDIO_MAC
+    else if (cap==PJMEDIA_AUD_DEV_CAP_OUTPUT_VOLUME_SETTING &&
+        (strm->param.dir & PJMEDIA_DIR_PLAYBACK))
+    {
+        OSStatus ostatus;
+        Float32 volume = *(unsigned*)pval;
+
+        /* Output volume setting */
+        volume /= 100.0;
+        ostatus = AudioUnitSetProperty (strm->io_units[1] ? strm->io_units[1] :
+                                        strm->io_units[0],
+                                        kAudioDevicePropertyVolumeScalar,
+                                        kAudioUnitScope_Output,
+                                        0,
+                                        &volume,
+                                        sizeof(Float32));
+        if (ostatus != noErr) {
+            return PJMEDIA_AUDIODEV_ERRNO_FROM_COREAUDIO(ostatus);
+        }
+        strm->param.output_vol = *(unsigned*)pval;
+        return PJ_SUCCESS;
+    }
+
+#else
+    else if ((cap==PJMEDIA_AUD_DEV_CAP_INPUT_LATENCY &&
          (strm->param.dir & PJMEDIA_DIR_CAPTURE)) ||
         (cap==PJMEDIA_AUD_DEV_CAP_OUTPUT_LATENCY &&

pjproject/trunk/pjsip/include/pjsua-lib/pjsua.h

@@ -6546 +6546 @@
 
     /**
-     * Echo canceller options (see #pjmedia_echo_create())
+     * Echo canceller options (see #pjmedia_echo_create()).
+     * Specify PJMEDIA_ECHO_USE_SW_ECHO here if application wishes
+     * to use software echo canceller instead of device EC.
      *
      * Default: 0.

pjproject/trunk/pjsip/include/pjsua2/endpoint.hpp

@@ -808 +808 @@
 
     /**
-     * Echo canceller options (see pjmedia_echo_create())
+     * Echo canceller options (see pjmedia_echo_create()).
+     * Specify PJMEDIA_ECHO_USE_SW_ECHO here if application wishes
+     * to use software echo canceller instead of device EC.
      *
      * Default: 0.