Changeset 2063

Timestamp:

Jun 26, 2008 6:52:16 PM (16 years ago)

Author:

nanang

Message:

Ticket #543: Updated PESQ to start/stop manually stream/record instead of using auto-rec/play/hangup, and also updated thresholds

Location:

pjproject/trunk/pjsip-apps/src/test-pjsua

Files:

• mod_pesq.py (modified) (5 diffs)
• scripts-pesq/100_defaults.py (modified) (1 diff)
• scripts-pesq/101_defaults.py (modified) (2 diffs)
• scripts-pesq/200_codec_g711a.py (modified) (1 diff)
• scripts-pesq/200_codec_g711u.py (modified) (1 diff)
• scripts-pesq/200_codec_g722.py (modified) (1 diff)
• scripts-pesq/200_codec_gsm.py (modified) (1 diff)
• scripts-pesq/200_codec_ilbc.py (modified) (1 diff)
• scripts-pesq/200_codec_speex_16000.py (modified) (1 diff)
• scripts-pesq/200_codec_speex_8000.py (modified) (1 diff)
• scripts-pesq/201_codec_g711a.py (modified) (2 diffs)
• scripts-pesq/201_codec_g711u.py (modified) (2 diffs)
• scripts-pesq/201_codec_g722.py (modified) (2 diffs)
• scripts-pesq/201_codec_gsm.py (modified) (2 diffs)
• scripts-pesq/201_codec_ilbc.py (modified) (2 diffs)
• scripts-pesq/201_codec_speex_16000.py (modified) (2 diffs)
• scripts-pesq/201_codec_speex_8000.py (modified) (2 diffs)

pjproject/trunk/pjsip-apps/src/test-pjsua/mod_pesq.py

@@ -16 +16 @@
 import re
 import subprocess
+import wave
 import inc_const as const
 
@@ -26 +27 @@
 # PESQ_THRESHOLD specifies the minimum acceptable PESQ MOS value, so test can be declared successful
 PESQ = "tools/pesq.exe"
-PESQ_THRESHOLD = 3.0
+PESQ_DEFAULT_THRESHOLD = 3.4
 
 # UserData
@@ -39 +40 @@
 # Test body function
 def test_func(t, user_data):
-
-        if len(t.process) == 0:
-                return
 
         ua1 = t.process[0]
@@ -65 +63 @@
         user_data.pesq_sample_rate_opt = "+" + clock_rate + "000"
 
+        # Get WAV input length, in seconds
+        fin = wave.open(user_data.input_filename, "r")
+        if fin == None:
+                raise TestError("Failed opening input WAV file")
+        inwavlen = fin.getnframes() // fin.getframerate()
+        if (fin.getnframes() % fin.getframerate()) > 0:
+                inwavlen = inwavlen + 1
+        fin.close()
+
         # UA1 making call
         ua1.send("m")
         ua1.send(t.inst_params[1].uri)
         ua1.expect(const.STATE_CALLING)
+
+        # UA2 wait until call established
         ua2.expect(const.STATE_CONFIRMED)
 
-        # Disconnect mic -> rec file to avoid echo recorded when using sound device
-        ua2.send("cd 0 1")
+        # Disconnect mic -> rec file, to avoid echo recorded when using sound device
+        # Disconnect stream -> spk, make it silent
+        # Connect stream -> rec file, start recording
+        ua2.send("cd 0 1\ncd 4 0\ncc 4 1")
 
-        # Auto answer, auto play, auto hangup
-        # Just wait for call disconnected
-        # Assumed WAV input is no more than 30 secs
-        while 1:
-                line = ua2.proc.stdout.readline()
-                if line == "":
-                        raise TestError(ua2.name + ": Premature EOF")
+        # Disconnect mic -> stream, make stream purely sending from file
+        # Disconnect stream -> spk, make it silent
+        # Connect file -> stream, start sending
+        ua1.send("cd 0 4\ncd 4 0\ncc 1 4")
 
-                # Search for disconnected text
-                if re.search(const.STATE_DISCONNECTED, line) != None:
-                        break
-        
+        time.sleep(inwavlen)
+
+        # Disconnect files from bridge
+        ua2.send("cd 4 1")
+        ua2.expect(const.MEDIA_DISCONN_PORT_SUCCESS)
+        ua1.send("cd 1 4")
+        ua1.expect(const.MEDIA_DISCONN_PORT_SUCCESS)
+
 
 # Post body function
@@ -102 +114 @@
                 raise TestError("Failed to fetch PESQ result")
 
-        # Evaluate the similarity value
+        # Get threshold
+        if (cfg_file.pesq_threshold != None) | (cfg_file.pesq_threshold > -0.5 ):
+                threshold = cfg_file.pesq_threshold
+        else:
+                threshold = PESQ_DEFAULT_THRESHOLD
+
+        # Evaluate the PESQ MOS value
         pesq_res = mo_pesq_out.group(1)
-        if (float(pesq_res) >= PESQ_THRESHOLD):
+        if (float(pesq_res) >= threshold):
                 endpt.trace("Success, PESQ result = " + pesq_res)
         else:

pjproject/trunk/pjsip-apps/src/test-pjsua/scripts-pesq/100_defaults.py

@@ -12 +12 @@
                 "PESQ defaults pjsua settings",
                 [
-                        InstanceParam("UA1", ADD_PARAM + " --max-calls=1 --play-file wavs/input.16.wav --auto-play-hangup"),
-                        InstanceParam("UA2", "--null-audio --max-calls=1 --rec-file  wavs/tmp.16.wav --clock-rate 16000 --auto-answer 200 --auto-rec")
+                        InstanceParam("UA1", ADD_PARAM + " --max-calls=1 --play-file wavs/input.16.wav"),
+                        InstanceParam("UA2", "--null-audio --max-calls=1 --rec-file  wavs/tmp.16.wav --clock-rate 16000 --auto-answer 200")
                 ]
                 )
+
+pesq_threshold = None

pjproject/trunk/pjsip-apps/src/test-pjsua/scripts-pesq/101_defaults.py

@@ -7 +7 @@
                 "PESQ defaults pjsua settings (RX side uses snd dev)",
                 [
-                        InstanceParam("UA1", "--max-calls=1 --play-file wavs/input.16.wav --auto-play-hangup --null-audio"),
-                        InstanceParam("UA2", "--max-calls=1 --rec-file  wavs/tmp.16.wav --clock-rate 16000 --auto-answer 200 --auto-rec")
+                        InstanceParam("UA1", "--max-calls=1 --play-file wavs/input.16.wav --null-audio"),
+                        InstanceParam("UA2", "--max-calls=1 --rec-file  wavs/tmp.16.wav --clock-rate 16000 --auto-answer 200")
                 ]
                 )
@@ -16 +16 @@
         test_param.skip = True
 
+pesq_threshold = None

pjproject/trunk/pjsip-apps/src/test-pjsua/scripts-pesq/200_codec_g711a.py

@@ -12 +12 @@
                 "PESQ codec PCMA",
                 [
-                        InstanceParam("UA1", ADD_PARAM + " --max-calls=1 --add-codec pcma --clock-rate 8000 --play-file wavs/input.8.wav --auto-play-hangup"),
-                        InstanceParam("UA2", "--null-audio --max-calls=1 --add-codec pcma --clock-rate 8000 --rec-file  wavs/tmp.8.wav   --auto-answer 200 --auto-rec")
+                        InstanceParam("UA1", ADD_PARAM + " --max-calls=1 --add-codec pcma --clock-rate 8000 --play-file wavs/input.8.wav"),
+                        InstanceParam("UA2", "--null-audio --max-calls=1 --add-codec pcma --clock-rate 8000 --rec-file  wavs/tmp.8.wav --auto-answer 200")
                 ]
                 )
+
+pesq_threshold = 3.5

pjproject/trunk/pjsip-apps/src/test-pjsua/scripts-pesq/200_codec_g711u.py

@@ -12 +12 @@
                 "PESQ codec PCMU",
                 [
-                        InstanceParam("UA1", ADD_PARAM + " --max-calls=1 --add-codec pcmu --clock-rate 8000 --play-file wavs/input.8.wav --auto-play-hangup"),
-                        InstanceParam("UA2", "--null-sound --max-calls=1 --add-codec pcmu --clock-rate 8000 --rec-file  wavs/tmp.8.wav   --auto-answer 200 --auto-rec")
+                        InstanceParam("UA1", ADD_PARAM + " --max-calls=1 --add-codec pcmu --clock-rate 8000 --play-file wavs/input.8.wav"),
+                        InstanceParam("UA2", "--null-audio --max-calls=1 --add-codec pcmu --clock-rate 8000 --rec-file  wavs/tmp.8.wav --auto-answer 200")
                 ]
                 )
+
+pesq_threshold = 3.5

pjproject/trunk/pjsip-apps/src/test-pjsua/scripts-pesq/200_codec_g722.py

@@ -12 +12 @@
                 "PESQ codec G722",
                 [
-                        InstanceParam("UA1", ADD_PARAM + " --max-calls=1 --add-codec g722 --clock-rate 16000 --play-file wavs/input.16.wav --auto-play-hangup"),
-                        InstanceParam("UA2", "--null-sound --max-calls=1 --add-codec g722 --clock-rate 16000 --rec-file  wavs/tmp.16.wav   --auto-answer 200 --auto-rec")
+                        InstanceParam("UA1", ADD_PARAM + " --max-calls=1 --add-codec g722 --clock-rate 16000 --play-file wavs/input.16.wav"),
+                        InstanceParam("UA2", "--null-audio --max-calls=1 --add-codec g722 --clock-rate 16000 --rec-file  wavs/tmp.16.wav --auto-answer 200")
                 ]
                 )
+
+pesq_threshold = 3.7

pjproject/trunk/pjsip-apps/src/test-pjsua/scripts-pesq/200_codec_gsm.py

@@ -12 +12 @@
                 "PESQ codec GSM",
                 [
-                        InstanceParam("UA1", ADD_PARAM + " --max-calls=1 --add-codec gsm --clock-rate 8000 --play-file wavs/input.8.wav --auto-play-hangup"),
-                        InstanceParam("UA2", "--null-sound --max-calls=1 --add-codec gsm --clock-rate 8000 --rec-file  wavs/tmp.8.wav   --auto-answer 200 --auto-rec")
+                        InstanceParam("UA1", ADD_PARAM + " --max-calls=1 --add-codec gsm --clock-rate 8000 --play-file wavs/input.8.wav"),
+                        InstanceParam("UA2", "--null-audio --max-calls=1 --add-codec gsm --clock-rate 8000 --rec-file  wavs/tmp.8.wav   --auto-answer 200")
                 ]
                 )
+
+pesq_threshold = 3.0

pjproject/trunk/pjsip-apps/src/test-pjsua/scripts-pesq/200_codec_ilbc.py

@@ -12 +12 @@
                 "PESQ codec iLBC",
                 [
-                        InstanceParam("UA1", ADD_PARAM + " --max-calls=1 --add-codec ilbc --clock-rate 8000 --play-file wavs/input.8.wav --auto-play-hangup"),
-                        InstanceParam("UA2", "--null-sound --max-calls=1 --add-codec ilbc --clock-rate 8000 --rec-file  wavs/tmp.8.wav   --auto-answer 200 --auto-rec")
+                        InstanceParam("UA1", ADD_PARAM + " --max-calls=1 --add-codec ilbc --clock-rate 8000 --play-file wavs/input.8.wav"),
+                        InstanceParam("UA2", "--null-audio --max-calls=1 --add-codec ilbc --clock-rate 8000 --rec-file  wavs/tmp.8.wav   --auto-answer 200")
                 ]
                 )
+
+pesq_threshold = 3.0

pjproject/trunk/pjsip-apps/src/test-pjsua/scripts-pesq/200_codec_speex_16000.py

@@ -12 +12 @@
                 "PESQ codec Speex WB",
                 [
-                        InstanceParam("UA1", ADD_PARAM + " --max-calls=1 --clock-rate 16000 --add-codec speex/16000 --play-file wavs/input.16.wav --auto-play-hangup"),
-                        InstanceParam("UA2", "--null-sound --max-calls=1 --clock-rate 16000 --add-codec speex/16000 --rec-file  wavs/tmp.16.wav   --auto-answer 200 --auto-rec")
+                        InstanceParam("UA1", ADD_PARAM + " --max-calls=1 --clock-rate 16000 --add-codec speex/16000 --play-file wavs/input.16.wav"),
+                        InstanceParam("UA2", "--null-audio --max-calls=1 --clock-rate 16000 --add-codec speex/16000 --rec-file  wavs/tmp.16.wav --auto-answer 200")
                 ]
                 )
+
+pesq_threshold = 3.7

pjproject/trunk/pjsip-apps/src/test-pjsua/scripts-pesq/200_codec_speex_8000.py

@@ -12 +12 @@
                 "PESQ codec Speex NB",
                 [
-                        InstanceParam("UA1", ADD_PARAM + " --max-calls=1 --add-codec speex/8000 --clock-rate 8000 --play-file wavs/input.8.wav --auto-play-hangup"),
-                        InstanceParam("UA2", "--null-sound --max-calls=1 --add-codec speex/8000 --clock-rate 8000 --rec-file  wavs/tmp.8.wav   --auto-answer 200 --auto-rec")
+                        InstanceParam("UA1", ADD_PARAM + " --max-calls=1 --add-codec speex/8000 --clock-rate 8000 --play-file wavs/input.8.wav"),
+                        InstanceParam("UA2", "--null-audio --max-calls=1 --add-codec speex/8000 --clock-rate 8000 --rec-file  wavs/tmp.8.wav --auto-answer 200")
                 ]
                 )
+
+pesq_threshold = 3.0

pjproject/trunk/pjsip-apps/src/test-pjsua/scripts-pesq/201_codec_g711a.py

@@ -7 +7 @@
                 "PESQ codec PCMA (RX side uses snd dev)",
                 [
-                        InstanceParam("UA1", "--max-calls=1 --add-codec pcma --clock-rate 8000 --play-file wavs/input.8.wav --auto-play-hangup --null-audio"),
-                        InstanceParam("UA2", "--max-calls=1 --add-codec pcma --clock-rate 8000 --rec-file  wavs/tmp.8.wav   --auto-answer 200 --auto-rec")
+                        InstanceParam("UA1", "--max-calls=1 --add-codec pcma --clock-rate 8000 --play-file wavs/input.8.wav --null-audio"),
+                        InstanceParam("UA2", "--max-calls=1 --add-codec pcma --clock-rate 8000 --rec-file  wavs/tmp.8.wav   --auto-answer 200")
                 ]
                 )
@@ -14 +14 @@
 if (HAS_SND_DEV == 0):
         test_param.skip = True
+
+pesq_threshold = 3.5

pjproject/trunk/pjsip-apps/src/test-pjsua/scripts-pesq/201_codec_g711u.py

@@ -7 +7 @@
                 "PESQ codec PCMU (RX side uses snd dev)",
                 [
-                        InstanceParam("UA1", "--max-calls=1 --add-codec pcmu --clock-rate 8000 --play-file wavs/input.8.wav --auto-play-hangup --null-audio"),
-                        InstanceParam("UA2", "--max-calls=1 --add-codec pcmu --clock-rate 8000 --rec-file  wavs/tmp.8.wav   --auto-answer 200 --auto-rec")
+                        InstanceParam("UA1", "--max-calls=1 --add-codec pcmu --clock-rate 8000 --play-file wavs/input.8.wav --null-audio"),
+                        InstanceParam("UA2", "--max-calls=1 --add-codec pcmu --clock-rate 8000 --rec-file  wavs/tmp.8.wav   --auto-answer 200")
                 ]
                 )
@@ -14 +14 @@
 if (HAS_SND_DEV == 0):
         test_param.skip = True
+
+pesq_threshold = 3.5

pjproject/trunk/pjsip-apps/src/test-pjsua/scripts-pesq/201_codec_g722.py

@@ -7 +7 @@
                 "PESQ codec G722 (RX side uses snd dev)",
                 [
-                        InstanceParam("UA1", "--max-calls=1 --add-codec g722 --clock-rate 16000 --play-file wavs/input.16.wav --auto-play-hangup --null-audio"),
-                        InstanceParam("UA2", "--max-calls=1 --add-codec g722 --clock-rate 16000 --rec-file  wavs/tmp.16.wav   --auto-answer 200 --auto-rec")
+                        InstanceParam("UA1", "--max-calls=1 --add-codec g722 --clock-rate 16000 --play-file wavs/input.16.wav --null-audio"),
+                        InstanceParam("UA2", "--max-calls=1 --add-codec g722 --clock-rate 16000 --rec-file  wavs/tmp.16.wav   --auto-answer 200")
                 ]
                 )
@@ -14 +14 @@
 if (HAS_SND_DEV == 0):
         test_param.skip = True
+
+pesq_threshold = 3.7

pjproject/trunk/pjsip-apps/src/test-pjsua/scripts-pesq/201_codec_gsm.py

@@ -7 +7 @@
                 "PESQ codec GSM (RX side uses snd dev)",
                 [
-                        InstanceParam("UA1", "--max-calls=1 --add-codec gsm --clock-rate 8000 --play-file wavs/input.8.wav --auto-play-hangup --null-audio"),
-                        InstanceParam("UA2", "--max-calls=1 --add-codec gsm --clock-rate 8000 --rec-file  wavs/tmp.8.wav   --auto-answer 200 --auto-rec")
+                        InstanceParam("UA1", "--max-calls=1 --add-codec gsm --clock-rate 8000 --play-file wavs/input.8.wav --null-audio"),
+                        InstanceParam("UA2", "--max-calls=1 --add-codec gsm --clock-rate 8000 --rec-file  wavs/tmp.8.wav   --auto-answer 200")
                 ]
                 )
@@ -14 +14 @@
 if (HAS_SND_DEV == 0):
         test_param.skip = True
+
+pesq_threshold = 3.0

pjproject/trunk/pjsip-apps/src/test-pjsua/scripts-pesq/201_codec_ilbc.py

@@ -7 +7 @@
                 "PESQ codec iLBC (RX side uses snd dev)",
                 [
-                        InstanceParam("UA1", "--max-calls=1 --add-codec ilbc --clock-rate 8000 --play-file wavs/input.8.wav --auto-play-hangup --null-audio"),
-                        InstanceParam("UA2", "--max-calls=1 --add-codec ilbc --clock-rate 8000 --rec-file  wavs/tmp.8.wav   --auto-answer 200 --auto-rec")
+                        InstanceParam("UA1", "--max-calls=1 --add-codec ilbc --clock-rate 8000 --play-file wavs/input.8.wav --null-audio"),
+                        InstanceParam("UA2", "--max-calls=1 --add-codec ilbc --clock-rate 8000 --rec-file  wavs/tmp.8.wav   --auto-answer 200")
                 ]
                 )
@@ -14 +14 @@
 if (HAS_SND_DEV == 0):
         test_param.skip = True
+
+pesq_threshold = 3.0

pjproject/trunk/pjsip-apps/src/test-pjsua/scripts-pesq/201_codec_speex_16000.py

@@ -7 +7 @@
                 "PESQ codec Speex WB (RX side uses snd dev)",
                 [
-                        InstanceParam("UA1", "--max-calls=1 --clock-rate 16000 --add-codec speex/16000 --play-file wavs/input.16.wav --auto-play-hangup --null-audio"),
-                        InstanceParam("UA2", "--max-calls=1 --clock-rate 16000 --add-codec speex/16000 --rec-file  wavs/tmp.16.wav   --auto-answer 200 --auto-rec")
+                        InstanceParam("UA1", "--max-calls=1 --clock-rate 16000 --add-codec speex/16000 --play-file wavs/input.16.wav --null-audio"),
+                        InstanceParam("UA2", "--max-calls=1 --clock-rate 16000 --add-codec speex/16000 --rec-file  wavs/tmp.16.wav   --auto-answer 200")
                 ]
                 )
@@ -14 +14 @@
 if (HAS_SND_DEV == 0):
         test_param.skip = True
+
+pesq_threshold = 3.7

pjproject/trunk/pjsip-apps/src/test-pjsua/scripts-pesq/201_codec_speex_8000.py

@@ -7 +7 @@
                 "PESQ codec Speex NB (RX side uses snd dev)",
                 [
-                        InstanceParam("UA1", "--max-calls=1 --add-codec speex/8000 --clock-rate 8000 --play-file wavs/input.8.wav --auto-play-hangup --null-audio"),
-                        InstanceParam("UA2", "--max-calls=1 --add-codec speex/8000 --clock-rate 8000 --rec-file  wavs/tmp.8.wav   --auto-answer 200 --auto-rec")
+                        InstanceParam("UA1", "--max-calls=1 --add-codec speex/8000 --clock-rate 8000 --play-file wavs/input.8.wav --null-audio"),
+                        InstanceParam("UA2", "--max-calls=1 --add-codec speex/8000 --clock-rate 8000 --rec-file  wavs/tmp.8.wav   --auto-answer 200")
                 ]
                 )
@@ -14 +14 @@
 if (HAS_SND_DEV == 0):
         test_param.skip = True
+
+pesq_threshold = 3.0