Changeset 5699

Timestamp:

Nov 21, 2017 9:25:11 AM (7 years ago)

Author:

riza

Message:

Close #2065: Update libyuv to fix linker error when building libyuv as dll on Visual Studio 2015.

Location:

pjproject/trunk/third_party

Files:

• build/yuv/Notes.txt (modified) (1 diff)
• yuv/include/libyuv/basic_types.h (modified) (1 diff)
• yuv/include/libyuv/compare_row.h (modified) (6 diffs)
• yuv/include/libyuv/convert_from.h (modified) (1 diff)
• yuv/include/libyuv/cpu_id.h (modified) (1 diff)
• yuv/include/libyuv/planar_functions.h (modified) (2 diffs)
• yuv/include/libyuv/rotate_row.h (modified) (2 diffs)
• yuv/include/libyuv/row.h (modified) (19 diffs)
• yuv/include/libyuv/scale_row.h (modified) (5 diffs)
• yuv/include/libyuv/version.h (modified) (1 diff)
• yuv/source/compare.cc (modified) (5 diffs)
• yuv/source/compare_common.cc (modified) (2 diffs)
• yuv/source/compare_gcc.cc (modified) (1 diff)
• yuv/source/compare_neon.cc (modified) (2 diffs)
• yuv/source/compare_neon64.cc (modified) (2 diffs)
• yuv/source/compare_win.cc (modified) (9 diffs)
• yuv/source/convert_from.cc (modified) (2 diffs)
• yuv/source/cpu_id.cc (modified) (3 diffs)
• yuv/source/mjpeg_decoder.cc (modified) (1 diff)
• yuv/source/mjpeg_validate.cc (modified) (1 diff)
• yuv/source/planar_functions.cc (modified) (12 diffs)
• yuv/source/rotate.cc (modified) (1 diff)
• yuv/source/rotate_neon64.cc (modified) (3 diffs)
• yuv/source/rotate_win.cc (modified) (3 diffs)
• yuv/source/row_any.cc (modified) (5 diffs)
• yuv/source/row_common.cc (modified) (2 diffs)
• yuv/source/row_gcc.cc (modified) (7 diffs)
• yuv/source/row_msa.cc (modified) (2 diffs)
• yuv/source/row_neon.cc (modified) (92 diffs)
• yuv/source/row_neon64.cc (modified) (81 diffs)
• yuv/source/row_win.cc (modified) (39 diffs)
• yuv/source/scale.cc (modified) (6 diffs)
• yuv/source/scale_any.cc (modified) (3 diffs)
• yuv/source/scale_argb.cc (modified) (6 diffs)
• yuv/source/scale_common.cc (modified) (1 diff)
• yuv/source/scale_msa.cc (modified) (2 diffs)
• yuv/source/scale_neon.cc (modified) (29 diffs)
• yuv/source/scale_neon64.cc (modified) (22 diffs)
• yuv/source/scale_win.cc (modified) (8 diffs)

pjproject/trunk/third_party/build/yuv/Notes.txt

@@ -1 +1 @@
 Notes:
 
-* Source code for libyuv from https://chromium.googlesource.com/libyuv/libyuv/ dated 27 July 2017.
+* Source code for libyuv from https://chromium.googlesource.com/libyuv/libyuv/ dated 17 November 2017.
 
-* All code is compilable, except for compare_win.cc
-  - Use older version (https://chromium.googlesource.com/libyuv/libyuv/+/baf6a3c1bd385e7ffe6b7634560e71fb49e4f589%5E%21/)
-    Since there's a compiler error on:
-    --------------------------------------------------------------------------------------
-    pmulld      xmm0,xmm6
-    --------------------------------------------------------------------------------------
-
-  - On VS2015, error C2024: 'alignas' attribute applies to variables, data members and tag types only
-    --------------------------------------------------------------------------------------
-    __declspec(naked) __declspec(align(16))
-
-    Change to :
-
-    __declspec(naked)
-    --------------------------------------------------------------------------------------
-
-* Added these lines to file include/libyuv/basic_types.h:
-  --
-  #if _MSC_VER==1400
-  #   include <stdint.h>  // for uint8_t
-  #endif
-  ...
-  #if defined(_MSC_VER)
-  #  pragma warning(disable:4996) // This function or variable may be unsafe.
-  #endif
-  --

pjproject/trunk/third_party/yuv/include/libyuv/basic_types.h

@@ -15 +15 @@
 
 #if defined(_MSC_VER) && (_MSC_VER < 1600)
-#if _MSC_VER==1400
-#   include <stdint.h>  // for uint8_t
-#endif
 #include <sys/types.h>  // for uintptr_t on x86
 #else
 #include <stdint.h>  // for uintptr_t
-#endif
-
-#if defined(_MSC_VER)
-#  pragma warning(disable:4996) // This function or variable may be unsafe.
 #endif
 

pjproject/trunk/third_party/yuv/include/libyuv/compare_row.h

@@ -20 +20 @@
 
 #if defined(__pnacl__) || defined(__CLR_VER) || \
-    (defined(__i386__) && !defined(__SSE2__))
+    (defined(__i386__) && !defined(__SSE__) && !defined(__clang__))
 #define LIBYUV_DISABLE_X86
 #endif
@@ -43 +43 @@
 #endif  // __clang__
 
+// The following are available for Visual C:
 #if !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86) && \
     (defined(VISUALC_HAS_AVX2) || defined(CLANG_HAS_AVX2))
@@ -53 +54 @@
 #define HAS_HASHDJB2_SSE41
 #define HAS_SUMSQUAREERROR_SSE2
-#define HAS_HAMMINGDISTANCE_X86
+#define HAS_HAMMINGDISTANCE_SSE42
 #endif
 
@@ -63 +64 @@
 #endif
 
+// The following are available for GCC and clangcl 64 bit:
+#if !defined(LIBYUV_DISABLE_X86) && \
+    (defined(__x86_64__) || (defined(__i386__) && !defined(_MSC_VER)))
+#define HAS_HAMMINGDISTANCE_SSSE3
+#endif
+
+// The following are available for GCC and clangcl 64 bit:
+#if !defined(LIBYUV_DISABLE_X86) && defined(CLANG_HAS_AVX2) && \
+    (defined(__x86_64__) || (defined(__i386__) && !defined(_MSC_VER)))
+#define HAS_HAMMINGDISTANCE_AVX2
+#endif
+
 // The following are available for Neon:
 #if !defined(LIBYUV_DISABLE_NEON) && \
@@ -70 +83 @@
 #endif
 
+#if !defined(LIBYUV_DISABLE_MSA) && defined(__mips_msa)
+#define HAS_HAMMINGDISTANCE_MSA
+#define HAS_SUMSQUAREERROR_MSA
+#endif
+
 uint32 HammingDistance_C(const uint8* src_a, const uint8* src_b, int count);
-uint32 HammingDistance_X86(const uint8* src_a, const uint8* src_b, int count);
+uint32 HammingDistance_SSE42(const uint8* src_a, const uint8* src_b, int count);
+uint32 HammingDistance_SSSE3(const uint8* src_a, const uint8* src_b, int count);
+uint32 HammingDistance_AVX2(const uint8* src_a, const uint8* src_b, int count);
 uint32 HammingDistance_NEON(const uint8* src_a, const uint8* src_b, int count);
+uint32 HammingDistance_MSA(const uint8* src_a, const uint8* src_b, int count);
 
 uint32 SumSquareError_C(const uint8* src_a, const uint8* src_b, int count);
@@ -78 +99 @@
 uint32 SumSquareError_AVX2(const uint8* src_a, const uint8* src_b, int count);
 uint32 SumSquareError_NEON(const uint8* src_a, const uint8* src_b, int count);
+uint32 SumSquareError_MSA(const uint8* src_a, const uint8* src_b, int count);
 
 uint32 HashDjb2_C(const uint8* src, int count, uint32 seed);

pjproject/trunk/third_party/yuv/include/libyuv/convert_from.h

@@ -179 +179 @@
 LIBYUV_API
 int I420ToRAW(const uint8* src_y,
+              int src_stride_y,
+              const uint8* src_u,
+              int src_stride_u,
+              const uint8* src_v,
+              int src_stride_v,
+              uint8* dst_frame,
+              int dst_stride_frame,
+              int width,
+              int height);
+
+LIBYUV_API
+int H420ToRGB24(const uint8* src_y,
+                int src_stride_y,
+                const uint8* src_u,
+                int src_stride_u,
+                const uint8* src_v,
+                int src_stride_v,
+                uint8* dst_frame,
+                int dst_stride_frame,
+                int width,
+                int height);
+
+LIBYUV_API
+int H420ToRAW(const uint8* src_y,
               int src_stride_y,
               const uint8* src_u,

pjproject/trunk/third_party/yuv/include/libyuv/cpu_id.h

@@ -37 +37 @@
 static const int kCpuHasERMS = 0x800;
 static const int kCpuHasFMA3 = 0x1000;
-static const int kCpuHasAVX3 = 0x2000;
-static const int kCpuHasF16C = 0x4000;
-
-// 0x8000 reserved for future X86 flags.
+static const int kCpuHasF16C = 0x2000;
+static const int kCpuHasGFNI = 0x4000;
+static const int kCpuHasAVX512BW = 0x8000;
+static const int kCpuHasAVX512VL = 0x10000;
+static const int kCpuHasAVX512VBMI = 0x20000;
+static const int kCpuHasAVX512VBMI2 = 0x40000;
+static const int kCpuHasAVX512VBITALG = 0x80000;
+static const int kCpuHasAVX512VPOPCNTDQ = 0x100000;
 
 // These flags are only valid on MIPS processors.
-static const int kCpuHasMIPS = 0x10000;
-static const int kCpuHasDSPR2 = 0x20000;
-static const int kCpuHasMSA = 0x40000;
+static const int kCpuHasMIPS = 0x200000;
+static const int kCpuHasDSPR2 = 0x400000;
+static const int kCpuHasMSA = 0x800000;
 
 // Optional init function. TestCpuFlag does an auto-init.

pjproject/trunk/third_party/yuv/include/libyuv/planar_functions.h

@@ -69 +69 @@
                   int width,
                   int height);
+
+// Split interleaved RGB plane into separate R, G and B planes.
+LIBYUV_API
+void SplitRGBPlane(const uint8* src_rgb,
+                   int src_stride_rgb,
+                   uint8* dst_r,
+                   int dst_stride_r,
+                   uint8* dst_g,
+                   int dst_stride_g,
+                   uint8* dst_b,
+                   int dst_stride_b,
+                   int width,
+                   int height);
+
+// Merge separate R, G and B planes into one interleaved RGB plane.
+LIBYUV_API
+void MergeRGBPlane(const uint8* src_r,
+                   int src_stride_r,
+                   const uint8* src_g,
+                   int src_stride_g,
+                   const uint8* src_b,
+                   int src_stride_b,
+                   uint8* dst_rgb,
+                   int dst_stride_rgb,
+                   int width,
+                   int height);
 
 // Copy I400.  Supports inverting.
@@ -721 +747 @@
 
 #if defined(__pnacl__) || defined(__CLR_VER) || \
-    (defined(__i386__) && !defined(__SSE2__))
+    (defined(__i386__) && !defined(__SSE__) && !defined(__clang__))
 #define LIBYUV_DISABLE_X86
 #endif

pjproject/trunk/third_party/yuv/include/libyuv/rotate_row.h

@@ -20 +20 @@
 
 #if defined(__pnacl__) || defined(__CLR_VER) || \
-    (defined(__i386__) && !defined(__SSE2__))
+    (defined(__i386__) && !defined(__SSE__) && !defined(__clang__))
 #define LIBYUV_DISABLE_X86
 #endif
@@ -30 +30 @@
 #endif
 // The following are available for Visual C and clangcl 32 bit:
-#if !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86)
+#if !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86) && defined(_MSC_VER)
 #define HAS_TRANSPOSEWX8_SSSE3
 #define HAS_TRANSPOSEUVWX8_SSE2

pjproject/trunk/third_party/yuv/include/libyuv/row.h

@@ -32 +32 @@
 
 #if defined(__pnacl__) || defined(__CLR_VER) || \
-    (defined(__i386__) && !defined(__SSE2__))
+    (defined(__i386__) && !defined(__SSE__) && !defined(__clang__))
 #define LIBYUV_DISABLE_X86
 #endif
@@ -265 +265 @@
 #endif
 
+// The following are available for gcc/clang x86 platforms:
+// TODO(fbarchard): Port to Visual C
+#if !defined(LIBYUV_DISABLE_X86) && \
+    (defined(__x86_64__) || (defined(__i386__) && !defined(_MSC_VER)))
+#define HAS_MERGERGBROW_SSSE3
+#define HAS_SPLITRGBROW_SSSE3
+#endif
+
+// The following are available for AVX2 gcc/clang x86 platforms:
+// TODO(fbarchard): Port to Visual C
+#if !defined(LIBYUV_DISABLE_X86) &&                                       \
+    (defined(__x86_64__) || (defined(__i386__) && !defined(_MSC_VER))) && \
+    (defined(CLANG_HAS_AVX2) || defined(GCC_HAS_AVX2))
+#define HAS_MERGEUVROW_16_AVX2
+#define HAS_MULTIPLYROW_16_AVX2
+#endif
+
 // The following are available on Neon platforms:
 #if !defined(LIBYUV_DISABLE_NEON) && \
@@ -324 +341 @@
 #define HAS_RGBATOYROW_NEON
 #define HAS_SETROW_NEON
+#define HAS_SPLITRGBROW_NEON
 #define HAS_SPLITUVROW_NEON
 #define HAS_UYVYTOARGBROW_NEON
@@ -353 +371 @@
 #define HAS_SOBELXYROW_NEON
 #define HAS_SOBELYROW_NEON
+#endif
+
+// The following are available on AArch64 platforms:
+#if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
+#define HAS_SCALESUMSAMPLES_NEON
 #endif
 
@@ -386 +409 @@
 
 #if !defined(LIBYUV_DISABLE_MSA) && defined(__mips_msa)
+#define HAS_ABGRTOUVROW_MSA
+#define HAS_ABGRTOYROW_MSA
+#define HAS_ARGB1555TOARGBROW_MSA
+#define HAS_ARGB1555TOUVROW_MSA
+#define HAS_ARGB1555TOYROW_MSA
+#define HAS_ARGB4444TOARGBROW_MSA
+#define HAS_ARGBADDROW_MSA
+#define HAS_ARGBATTENUATEROW_MSA
+#define HAS_ARGBBLENDROW_MSA
+#define HAS_ARGBCOLORMATRIXROW_MSA
+#define HAS_ARGBEXTRACTALPHAROW_MSA
+#define HAS_ARGBGRAYROW_MSA
 #define HAS_ARGBMIRRORROW_MSA
+#define HAS_ARGBMULTIPLYROW_MSA
+#define HAS_ARGBQUANTIZEROW_MSA
+#define HAS_ARGBSEPIAROW_MSA
+#define HAS_ARGBSETROW_MSA
+#define HAS_ARGBSHADEROW_MSA
+#define HAS_ARGBSHUFFLEROW_MSA
+#define HAS_ARGBSUBTRACTROW_MSA
+#define HAS_ARGBTOARGB1555ROW_MSA
+#define HAS_ARGBTOARGB4444ROW_MSA
+#define HAS_ARGBTORAWROW_MSA
+#define HAS_ARGBTORGB24ROW_MSA
+#define HAS_ARGBTORGB565DITHERROW_MSA
+#define HAS_ARGBTORGB565ROW_MSA
+#define HAS_ARGBTOUV444ROW_MSA
+#define HAS_ARGBTOUVJROW_MSA
+#define HAS_ARGBTOUVROW_MSA
+#define HAS_ARGBTOYJROW_MSA
+#define HAS_ARGBTOYROW_MSA
+#define HAS_BGRATOUVROW_MSA
+#define HAS_BGRATOYROW_MSA
+#define HAS_HALFFLOATROW_MSA
+#define HAS_I400TOARGBROW_MSA
+#define HAS_I422ALPHATOARGBROW_MSA
+#define HAS_I422TOARGBROW_MSA
+#define HAS_I422TORGB24ROW_MSA
+#define HAS_I422TORGBAROW_MSA
 #define HAS_I422TOUYVYROW_MSA
 #define HAS_I422TOYUY2ROW_MSA
+#define HAS_I444TOARGBROW_MSA
+#define HAS_INTERPOLATEROW_MSA
+#define HAS_J400TOARGBROW_MSA
+#define HAS_MERGEUVROW_MSA
 #define HAS_MIRRORROW_MSA
+#define HAS_MIRRORUVROW_MSA
+#define HAS_NV12TOARGBROW_MSA
+#define HAS_NV12TORGB565ROW_MSA
+#define HAS_NV21TOARGBROW_MSA
+#define HAS_RAWTOARGBROW_MSA
+#define HAS_RAWTORGB24ROW_MSA
+#define HAS_RAWTOUVROW_MSA
+#define HAS_RAWTOYROW_MSA
+#define HAS_RGB24TOARGBROW_MSA
+#define HAS_RGB24TOUVROW_MSA
+#define HAS_RGB24TOYROW_MSA
+#define HAS_RGB565TOARGBROW_MSA
+#define HAS_RGB565TOUVROW_MSA
+#define HAS_RGB565TOYROW_MSA
+#define HAS_RGBATOUVROW_MSA
+#define HAS_RGBATOYROW_MSA
+#define HAS_SETROW_MSA
+#define HAS_SOBELROW_MSA
+#define HAS_SOBELTOPLANEROW_MSA
+#define HAS_SOBELXROW_MSA
+#define HAS_SOBELXYROW_MSA
+#define HAS_SOBELYROW_MSA
+#define HAS_SPLITUVROW_MSA
+#define HAS_UYVYTOARGBROW_MSA
 #define HAS_UYVYTOUVROW_MSA
 #define HAS_UYVYTOYROW_MSA
+#define HAS_YUY2TOARGBROW_MSA
 #define HAS_YUY2TOUV422ROW_MSA
 #define HAS_YUY2TOUVROW_MSA
 #define HAS_YUY2TOYROW_MSA
-#define HAS_ARGB4444TOARGBROW_MSA
-#define HAS_ARGBTOYROW_MSA
-#define HAS_ARGBTOUVROW_MSA
-#define HAS_I422TOARGBROW_MSA
-#define HAS_I422TORGBAROW_MSA
-#define HAS_I422ALPHATOARGBROW_MSA
-#define HAS_I422TORGB24ROW_MSA
-#define HAS_ARGBTORGB24ROW_MSA
-#define HAS_ARGBTORAWROW_MSA
-#define HAS_ARGBTORGB565ROW_MSA
-#define HAS_ARGBTOARGB1555ROW_MSA
-#define HAS_ARGBTOARGB4444ROW_MSA
-#define HAS_ARGBTOUV444ROW_MSA
-#define HAS_ARGBMULTIPLYROW_MSA
-#define HAS_ARGBADDROW_MSA
-#define HAS_ARGBSUBTRACTROW_MSA
-#define HAS_ARGBATTENUATEROW_MSA
-#define HAS_ARGBTORGB565DITHERROW_MSA
-#define HAS_ARGBSHUFFLEROW_MSA
-#define HAS_ARGBSHADEROW_MSA
-#define HAS_ARGBGRAYROW_MSA
-#define HAS_ARGBSEPIAROW_MSA
-#define HAS_ARGB1555TOARGBROW_MSA
-#define HAS_RGB565TOARGBROW_MSA
-#define HAS_RGB24TOARGBROW_MSA
-#define HAS_RAWTOARGBROW_MSA
-#define HAS_ARGB1555TOYROW_MSA
-#define HAS_RGB565TOYROW_MSA
-#define HAS_RGB24TOYROW_MSA
-#define HAS_RAWTOYROW_MSA
-#define HAS_ARGB1555TOUVROW_MSA
-#define HAS_RGB565TOUVROW_MSA
-#define HAS_RGB24TOUVROW_MSA
-#define HAS_RAWTOUVROW_MSA
-#define HAS_NV12TOARGBROW_MSA
-#define HAS_NV12TORGB565ROW_MSA
-#define HAS_NV21TOARGBROW_MSA
-#define HAS_SOBELROW_MSA
-#define HAS_SOBELTOPLANEROW_MSA
-#define HAS_SOBELXYROW_MSA
-#define HAS_ARGBTOYJROW_MSA
-#define HAS_BGRATOYROW_MSA
-#define HAS_ABGRTOYROW_MSA
-#define HAS_RGBATOYROW_MSA
-#define HAS_ARGBTOUVJROW_MSA
-#define HAS_BGRATOUVROW_MSA
-#define HAS_ABGRTOUVROW_MSA
-#define HAS_RGBATOUVROW_MSA
-#define HAS_I444TOARGBROW_MSA
-#define HAS_I400TOARGBROW_MSA
-#define HAS_J400TOARGBROW_MSA
-#define HAS_YUY2TOARGBROW_MSA
-#define HAS_UYVYTOARGBROW_MSA
-#define HAS_INTERPOLATEROW_MSA
-#define HAS_ARGBSETROW_MSA
-#define HAS_RAWTORGB24ROW_MSA
-#define HAS_MERGEUVROW_MSA
 #endif
 
@@ -1346 +1379 @@
                        uint8* dst_v,
                        int width);
+void MirrorUVRow_MSA(const uint8* src_uv,
+                     uint8* dst_u,
+                     uint8* dst_v,
+                     int width);
 void MirrorUVRow_C(const uint8* src_uv, uint8* dst_u, uint8* dst_v, int width);
 
@@ -1375 +1412 @@
                       uint8* dst_v,
                       int width);
+void SplitUVRow_MSA(const uint8* src_uv, uint8* dst_u, uint8* dst_v, int width);
 void SplitUVRow_Any_SSE2(const uint8* src_uv,
                          uint8* dst_u,
@@ -1391 +1429 @@
                           uint8* dst_v,
                           int width);
+void SplitUVRow_Any_MSA(const uint8* src_uv,
+                        uint8* dst_u,
+                        uint8* dst_v,
+                        int width);
 
 void MergeUVRow_C(const uint8* src_u,
@@ -1429 +1471 @@
                         int width);
 
+void SplitRGBRow_C(const uint8* src_rgb,
+                   uint8* dst_r,
+                   uint8* dst_g,
+                   uint8* dst_b,
+                   int width);
+void SplitRGBRow_SSSE3(const uint8* src_rgb,
+                       uint8* dst_r,
+                       uint8* dst_g,
+                       uint8* dst_b,
+                       int width);
+void SplitRGBRow_NEON(const uint8* src_rgb,
+                      uint8* dst_r,
+                      uint8* dst_g,
+                      uint8* dst_b,
+                      int width);
+void SplitRGBRow_Any_SSSE3(const uint8* src_rgb,
+                           uint8* dst_r,
+                           uint8* dst_g,
+                           uint8* dst_b,
+                           int width);
+void SplitRGBRow_Any_NEON(const uint8* src_rgb,
+                          uint8* dst_r,
+                          uint8* dst_g,
+                          uint8* dst_b,
+                          int width);
+
+void MergeRGBRow_C(const uint8* src_r,
+                   const uint8* src_g,
+                   const uint8* src_b,
+                   uint8* dst_rgb,
+                   int width);
+void MergeRGBRow_SSSE3(const uint8* src_r,
+                       const uint8* src_g,
+                       const uint8* src_b,
+                       uint8* dst_rgb,
+                       int width);
+void MergeRGBRow_NEON(const uint8* src_r,
+                      const uint8* src_g,
+                      const uint8* src_b,
+                      uint8* dst_rgb,
+                      int width);
+void MergeRGBRow_Any_SSSE3(const uint8* src_r,
+                           const uint8* src_g,
+                           const uint8* src_b,
+                           uint8* dst_rgb,
+                           int width);
+void MergeRGBRow_Any_NEON(const uint8* src_r,
+                          const uint8* src_g,
+                          const uint8* src_b,
+                          uint8* dst_rgb,
+                          int width);
+
+void MergeUVRow_16_C(const uint16* src_u,
+                     const uint16* src_v,
+                     uint16* dst_uv,
+                     int scale, /* 64 for 10 bit */
+                     int width);
+void MergeUVRow_16_AVX2(const uint16* src_u,
+                        const uint16* src_v,
+                        uint16* dst_uv,
+                        int scale,
+                        int width);
+
+void MultiplyRow_16_AVX2(const uint16* src_y,
+                         uint16* dst_y,
+                         int scale,
+                         int width);
+void MultiplyRow_16_C(const uint16* src_y, uint16* dst_y, int scale, int width);
+
 void CopyRow_SSE2(const uint8* src, uint8* dst, int count);
 void CopyRow_AVX(const uint8* src, uint8* dst, int count);
@@ -1455 +1566 @@
 void ARGBExtractAlphaRow_AVX2(const uint8* src_argb, uint8* dst_a, int width);
 void ARGBExtractAlphaRow_NEON(const uint8* src_argb, uint8* dst_a, int width);
+void ARGBExtractAlphaRow_MSA(const uint8* src_argb, uint8* dst_a, int width);
 void ARGBExtractAlphaRow_Any_SSE2(const uint8* src_argb,
                                   uint8* dst_a,
@@ -1464 +1576 @@
                                   uint8* dst_a,
                                   int width);
+void ARGBExtractAlphaRow_Any_MSA(const uint8* src_argb,
+                                 uint8* dst_a,
+                                 int width);
 
 void ARGBCopyYToAlphaRow_C(const uint8* src_y, uint8* dst_argb, int width);
@@ -1476 +1591 @@
 
 void SetRow_C(uint8* dst, uint8 v8, int count);
+void SetRow_MSA(uint8* dst, uint8 v8, int count);
 void SetRow_X86(uint8* dst, uint8 v8, int count);
 void SetRow_ERMS(uint8* dst, uint8 v8, int count);
@@ -2123 +2239 @@
                        uint8* dst_argb,
                        int width);
+void ARGBBlendRow_MSA(const uint8* src_argb,
+                      const uint8* src_argb1,
+                      uint8* dst_argb,
+                      int width);
 void ARGBBlendRow_C(const uint8* src_argb,
                     const uint8* src_argb1,
@@ -2836 +2956 @@
                              const int8* matrix_argb,
                              int width);
+void ARGBColorMatrixRow_MSA(const uint8* src_argb,
+                            uint8* dst_argb,
+                            const int8* matrix_argb,
+                            int width);
 
 void ARGBColorTableRow_C(uint8* dst_argb, const uint8* table_argb, int width);
@@ -2858 +2982 @@
                           int interval_offset,
                           int width);
+void ARGBQuantizeRow_MSA(uint8* dst_argb,
+                         int scale,
+                         int interval_size,
+                         int interval_offset,
+                         int width);
 
 void ARGBShadeRow_C(const uint8* src_argb,
@@ -2991 +3120 @@
                     uint8* dst_sobelx,
                     int width);
+void SobelXRow_MSA(const uint8* src_y0,
+                   const uint8* src_y1,
+                   const uint8* src_y2,
+                   uint8* dst_sobelx,
+                   int width);
 void SobelYRow_C(const uint8* src_y0,
                  const uint8* src_y1,
@@ -3003 +3137 @@
                     uint8* dst_sobely,
                     int width);
+void SobelYRow_MSA(const uint8* src_y0,
+                   const uint8* src_y1,
+                   uint8* dst_sobely,
+                   int width);
 void SobelRow_C(const uint8* src_sobelx,
                 const uint8* src_sobely,
@@ -3133 +3271 @@
                             float scale,
                             int width);
+void HalfFloatRow_MSA(const uint16* src, uint16* dst, float scale, int width);
+void HalfFloatRow_Any_MSA(const uint16* src,
+                          uint16* dst,
+                          float scale,
+                          int width);
 
 void ARGBLumaColorTableRow_C(const uint8* src_argb,
@@ -3145 +3288 @@
                                  uint32 lumacoeff);
 
+float ScaleMaxSamples_C(const float* src, float* dst, float scale, int width);
+float ScaleMaxSamples_NEON(const float* src,
+                           float* dst,
+                           float scale,
+                           int width);
+float ScaleSumSamples_C(const float* src, float* dst, float scale, int width);
+float ScaleSumSamples_NEON(const float* src,
+                           float* dst,
+                           float scale,
+                           int width);
+void ScaleSamples_C(const float* src, float* dst, float scale, int width);
+void ScaleSamples_NEON(const float* src, float* dst, float scale, int width);
+
 #ifdef __cplusplus
 }  // extern "C"

pjproject/trunk/third_party/yuv/include/libyuv/scale_row.h

@@ -21 +21 @@
 
 #if defined(__pnacl__) || defined(__CLR_VER) || \
-    (defined(__i386__) && !defined(__SSE2__))
+    (defined(__i386__) && !defined(__SSE__) && !defined(__clang__))
 #define LIBYUV_DISABLE_X86
 #endif
@@ -106 +106 @@
 
 #if !defined(LIBYUV_DISABLE_MSA) && defined(__mips_msa)
+#define HAS_SCALEADDROW_MSA
+#define HAS_SCALEARGBCOLS_MSA
+#define HAS_SCALEARGBFILTERCOLS_MSA
 #define HAS_SCALEARGBROWDOWN2_MSA
 #define HAS_SCALEARGBROWDOWNEVEN_MSA
+#define HAS_SCALEFILTERCOLS_MSA
 #define HAS_SCALEROWDOWN2_MSA
+#define HAS_SCALEROWDOWN34_MSA
+#define HAS_SCALEROWDOWN38_MSA
 #define HAS_SCALEROWDOWN4_MSA
-#define HAS_SCALEROWDOWN38_MSA
-#define HAS_SCALEADDROW_MSA
 #endif
 
@@ -547 +551 @@
                             int x,
                             int dx);
+void ScaleARGBFilterCols_MSA(uint8* dst_argb,
+                             const uint8* src_argb,
+                             int dst_width,
+                             int x,
+                             int dx);
+void ScaleARGBCols_MSA(uint8* dst_argb,
+                       const uint8* src_argb,
+                       int dst_width,
+                       int x,
+                       int dx);
+void ScaleARGBFilterCols_Any_MSA(uint8* dst_argb,
+                                 const uint8* src_argb,
+                                 int dst_width,
+                                 int x,
+                                 int dx);
+void ScaleARGBCols_Any_MSA(uint8* dst_argb,
+                           const uint8* src_argb,
+                           int dst_width,
+                           int x,
+                           int dx);
 
 // ARGB Row functions
@@ -886 +910 @@
                               int dst_width);
 void ScaleAddRow_MSA(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width);
+void ScaleFilterCols_MSA(uint8* dst_ptr,
+                         const uint8* src_ptr,
+                         int dst_width,
+                         int x,
+                         int dx);
+void ScaleRowDown34_MSA(const uint8* src_ptr,
+                        ptrdiff_t src_stride,
+                        uint8* dst_ptr,
+                        int dst_width);
+void ScaleRowDown34_0_Box_MSA(const uint8* src_ptr,
+                              ptrdiff_t src_stride,
+                              uint8* dst_ptr,
+                              int dst_width);
+void ScaleRowDown34_1_Box_MSA(const uint8* src_ptr,
+                              ptrdiff_t src_stride,
+                              uint8* dst_ptr,
+                              int dst_width);
+
 void ScaleRowDown2_Any_MSA(const uint8_t* src_ptr,
                            ptrdiff_t src_stride,
@@ -921 +963 @@
                          uint16_t* dst_ptr,
                          int src_width);
+void ScaleFilterCols_Any_MSA(uint8* dst_ptr,
+                             const uint8* src_ptr,
+                             int dst_width,
+                             int x,
+                             int dx);
+void ScaleRowDown34_Any_MSA(const uint8* src_ptr,
+                            ptrdiff_t src_stride,
+                            uint8* dst_ptr,
+                            int dst_width);
+void ScaleRowDown34_0_Box_Any_MSA(const uint8* src_ptr,
+                                  ptrdiff_t src_stride,
+                                  uint8* dst_ptr,
+                                  int dst_width);
+void ScaleRowDown34_1_Box_Any_MSA(const uint8* src_ptr,
+                                  ptrdiff_t src_stride,
+                                  uint8* dst_ptr,
+                                  int dst_width);
 
 #ifdef __cplusplus

pjproject/trunk/third_party/yuv/include/libyuv/version.h

@@ -12 +12 @@
 #define INCLUDE_LIBYUV_VERSION_H_
 
-#define LIBYUV_VERSION 1662
+#define LIBYUV_VERSION 1678
 
 #endif  // INCLUDE_LIBYUV_VERSION_H_

pjproject/trunk/third_party/yuv/source/compare.cc

@@ -111 +111 @@
 }
 
+// NEON version accumulates in 16 bit shorts which overflow at 65536 bytes.
+// So actual maximum is 1 less loop, which is 64436 - 32 bytes.
+
 LIBYUV_API
 uint64 ComputeHammingDistance(const uint8* src_a,
                               const uint8* src_b,
                               int count) {
-  const int kBlockSize = 65536;
-  int remainder = count & (kBlockSize - 1) & ~31;
+  const int kBlockSize = 1 << 15;  // 32768;
+  const int kSimdSize = 64;
+  // SIMD for multiple of 64, and C for remainder
+  int remainder = count & (kBlockSize - 1) & ~(kSimdSize - 1);
   uint64 diff = 0;
   int i;
@@ -126 +131 @@
   }
 #endif
-#if defined(HAS_HAMMINGDISTANCE_X86)
-  if (TestCpuFlag(kCpuHasX86)) {
-    HammingDistance = HammingDistance_X86;
+#if defined(HAS_HAMMINGDISTANCE_SSSE3)
+  if (TestCpuFlag(kCpuHasSSSE3)) {
+    HammingDistance = HammingDistance_SSSE3;
+  }
+#endif
+#if defined(HAS_HAMMINGDISTANCE_SSE42)
+  if (TestCpuFlag(kCpuHasSSE42)) {
+    HammingDistance = HammingDistance_SSE42;
   }
 #endif
@@ -136 +146 @@
   }
 #endif
+#if defined(HAS_HAMMINGDISTANCE_MSA)
+  if (TestCpuFlag(kCpuHasMSA)) {
+    HammingDistance = HammingDistance_MSA;
+  }
+#endif
 #ifdef _OPENMP
 #pragma omp parallel for reduction(+ : diff)
@@ -149 +164 @@
     src_b += remainder;
   }
-  remainder = count & 31;
+  remainder = count & (kSimdSize - 1);
   if (remainder) {
     diff += HammingDistance_C(src_a, src_b, remainder);
@@ -185 +200 @@
     // Note only used for multiples of 32 so count is not checked.
     SumSquareError = SumSquareError_AVX2;
+  }
+#endif
+#if defined(HAS_SUMSQUAREERROR_MSA)
+  if (TestCpuFlag(kCpuHasMSA)) {
+    SumSquareError = SumSquareError_MSA;
   }
 #endif

pjproject/trunk/third_party/yuv/source/compare_common.cc

@@ -19 +19 @@
 
 #if ORIGINAL_OPT
-uint32 HammingDistance_C(const uint8* src_a, const uint8* src_b, int count) {
+uint32 HammingDistance_C1(const uint8* src_a, const uint8* src_b, int count) {
   uint32 diff = 0u;
 
@@ -59 +59 @@
     src_b += 4;
   }
+
+  for (; i < count; ++i) {
+    uint32 x = *src_a ^ *src_b;
+    uint32 u = x - ((x >> 1) & 0x55);
+    u = ((u >> 2) & 0x33) + (u & 0x33);
+    diff += (u + (u >> 4)) & 0x0f;
+    src_a += 1;
+    src_b += 1;
+  }
+
   return diff;
 }

pjproject/trunk/third_party/yuv/source/compare_gcc.cc

@@ -23 +23 @@
     (defined(__x86_64__) || (defined(__i386__) && !defined(_MSC_VER)))
 
-uint32 HammingDistance_X86(const uint8* src_a, const uint8* src_b, int count) {
+#if defined(__x86_64__)
+uint32 HammingDistance_SSE42(const uint8* src_a,
+                             const uint8* src_b,
+                             int count) {
+  uint64 diff = 0u;
+
+  asm volatile(
+      "xor        %3,%3                          \n"
+      "xor        %%r8,%%r8                      \n"
+      "xor        %%r9,%%r9                      \n"
+      "xor        %%r10,%%r10                    \n"
+
+      // Process 32 bytes per loop.
+      LABELALIGN
+      "1:                                        \n"
+      "mov        (%0),%%rcx                     \n"
+      "mov        0x8(%0),%%rdx                  \n"
+      "xor        (%1),%%rcx                     \n"
+      "xor        0x8(%1),%%rdx                  \n"
+      "popcnt     %%rcx,%%rcx                    \n"
+      "popcnt     %%rdx,%%rdx                    \n"
+      "mov        0x10(%0),%%rsi                 \n"
+      "mov        0x18(%0),%%rdi                 \n"
+      "xor        0x10(%1),%%rsi                 \n"
+      "xor        0x18(%1),%%rdi                 \n"
+      "popcnt     %%rsi,%%rsi                    \n"
+      "popcnt     %%rdi,%%rdi                    \n"
+      "add        $0x20,%0                       \n"
+      "add        $0x20,%1                       \n"
+      "add        %%rcx,%3                       \n"
+      "add        %%rdx,%%r8                     \n"
+      "add        %%rsi,%%r9                     \n"
+      "add        %%rdi,%%r10                    \n"
+      "sub        $0x20,%2                       \n"
+      "jg         1b                             \n"
+
+      "add        %%r8, %3                       \n"
+      "add        %%r9, %3                       \n"
+      "add        %%r10, %3                      \n"
+      : "+r"(src_a),  // %0
+        "+r"(src_b),  // %1
+        "+r"(count),  // %2
+        "=r"(diff)    // %3
+      :
+      : "memory", "cc", "rcx", "rdx", "rsi", "rdi", "r8", "r9", "r10");
+
+  return static_cast<uint32>(diff);
+}
+#else
+uint32 HammingDistance_SSE42(const uint8* src_a,
+                             const uint8* src_b,
+                             int count) {
   uint32 diff = 0u;
 
-  int i;
-  for (i = 0; i < count - 7; i += 8) {
-    uint64 x = *((uint64*)src_a) ^ *((uint64*)src_b);
-    src_a += 8;
-    src_b += 8;
-    diff += __builtin_popcountll(x);
-  }
+  asm volatile(
+      // Process 16 bytes per loop.
+      LABELALIGN
+      "1:                                        \n"
+      "mov        (%0),%%ecx                     \n"
+      "mov        0x4(%0),%%edx                  \n"
+      "xor        (%1),%%ecx                     \n"
+      "xor        0x4(%1),%%edx                  \n"
+      "popcnt     %%ecx,%%ecx                    \n"
+      "add        %%ecx,%3                       \n"
+      "popcnt     %%edx,%%edx                    \n"
+      "add        %%edx,%3                       \n"
+      "mov        0x8(%0),%%ecx                  \n"
+      "mov        0xc(%0),%%edx                  \n"
+      "xor        0x8(%1),%%ecx                  \n"
+      "xor        0xc(%1),%%edx                  \n"
+      "popcnt     %%ecx,%%ecx                    \n"
+      "add        %%ecx,%3                       \n"
+      "popcnt     %%edx,%%edx                    \n"
+      "add        %%edx,%3                       \n"
+      "add        $0x10,%0                       \n"
+      "add        $0x10,%1                       \n"
+      "sub        $0x10,%2                       \n"
+      "jg         1b                             \n"
+      : "+r"(src_a),  // %0
+        "+r"(src_b),  // %1
+        "+r"(count),  // %2
+        "+r"(diff)    // %3
+      :
+      : "memory", "cc", "ecx", "edx");
+
   return diff;
 }
+#endif
+
+static vec8 kNibbleMask = {15, 15, 15, 15, 15, 15, 15, 15,
+                           15, 15, 15, 15, 15, 15, 15, 15};
+static vec8 kBitCount = {0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4};
+
+uint32 HammingDistance_SSSE3(const uint8* src_a,
+                             const uint8* src_b,
+                             int count) {
+  uint32 diff = 0u;
+
+  asm volatile(
+      "movdqa     %4,%%xmm2                      \n"
+      "movdqa     %5,%%xmm3                      \n"
+      "pxor       %%xmm0,%%xmm0                  \n"
+      "pxor       %%xmm1,%%xmm1                  \n"
+      "sub        %0,%1                          \n"
+
+      LABELALIGN
+      "1:                                        \n"
+      "movdqa     (%0),%%xmm4                    \n"
+      "movdqa     0x10(%0), %%xmm5               \n"
+      "pxor       (%0,%1), %%xmm4                \n"
+      "movdqa     %%xmm4,%%xmm6                  \n"
+      "pand       %%xmm2,%%xmm6                  \n"
+      "psrlw      $0x4,%%xmm4                    \n"
+      "movdqa     %%xmm3,%%xmm7                  \n"
+      "pshufb     %%xmm6,%%xmm7                  \n"
+      "pand       %%xmm2,%%xmm4                  \n"
+      "movdqa     %%xmm3,%%xmm6                  \n"
+      "pshufb     %%xmm4,%%xmm6                  \n"
+      "paddb      %%xmm7,%%xmm6                  \n"
+      "pxor       0x10(%0,%1),%%xmm5             \n"
+      "add        $0x20,%0                       \n"
+      "movdqa     %%xmm5,%%xmm4                  \n"
+      "pand       %%xmm2,%%xmm5                  \n"
+      "psrlw      $0x4,%%xmm4                    \n"
+      "movdqa     %%xmm3,%%xmm7                  \n"
+      "pshufb     %%xmm5,%%xmm7                  \n"
+      "pand       %%xmm2,%%xmm4                  \n"
+      "movdqa     %%xmm3,%%xmm5                  \n"
+      "pshufb     %%xmm4,%%xmm5                  \n"
+      "paddb      %%xmm7,%%xmm5                  \n"
+      "paddb      %%xmm5,%%xmm6                  \n"
+      "psadbw     %%xmm1,%%xmm6                  \n"
+      "paddd      %%xmm6,%%xmm0                  \n"
+      "sub        $0x20,%2                       \n"
+      "jg         1b                             \n"
+
+      "pshufd     $0xaa,%%xmm0,%%xmm1            \n"
+      "paddd      %%xmm1,%%xmm0                  \n"
+      "movd       %%xmm0, %3                     \n"
+      : "+r"(src_a),       // %0
+        "+r"(src_b),       // %1
+        "+r"(count),       // %2
+        "=r"(diff)         // %3
+      : "m"(kNibbleMask),  // %4
+        "m"(kBitCount)     // %5
+      : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+        "xmm7");
+
+  return diff;
+}
+
+#ifdef HAS_HAMMINGDISTANCE_AVX2
+uint32 HammingDistance_AVX2(const uint8* src_a, const uint8* src_b, int count) {
+  uint32 diff = 0u;
+
+  asm volatile(
+      "vbroadcastf128 %4,%%ymm2                  \n"
+      "vbroadcastf128 %5,%%ymm3                  \n"
+      "vpxor      %%ymm0,%%ymm0,%%ymm0           \n"
+      "vpxor      %%ymm1,%%ymm1,%%ymm1           \n"
+      "sub        %0,%1                          \n"
+
+      LABELALIGN
+      "1:                                        \n"
+      "vmovdqa    (%0),%%ymm4                    \n"
+      "vmovdqa    0x20(%0), %%ymm5               \n"
+      "vpxor      (%0,%1), %%ymm4, %%ymm4        \n"
+      "vpand      %%ymm2,%%ymm4,%%ymm6           \n"
+      "vpsrlw     $0x4,%%ymm4,%%ymm4             \n"
+      "vpshufb    %%ymm6,%%ymm3,%%ymm6           \n"
+      "vpand      %%ymm2,%%ymm4,%%ymm4           \n"
+      "vpshufb    %%ymm4,%%ymm3,%%ymm4           \n"
+      "vpaddb     %%ymm4,%%ymm6,%%ymm6           \n"
+      "vpxor      0x20(%0,%1),%%ymm5,%%ymm4      \n"
+      "add        $0x40,%0                       \n"
+      "vpand      %%ymm2,%%ymm4,%%ymm5           \n"
+      "vpsrlw     $0x4,%%ymm4,%%ymm4             \n"
+      "vpshufb    %%ymm5,%%ymm3,%%ymm5           \n"
+      "vpand      %%ymm2,%%ymm4,%%ymm4           \n"
+      "vpshufb    %%ymm4,%%ymm3,%%ymm4           \n"
+      "vpaddb     %%ymm5,%%ymm4,%%ymm4           \n"
+      "vpaddb     %%ymm6,%%ymm4,%%ymm4           \n"
+      "vpsadbw    %%ymm1,%%ymm4,%%ymm4           \n"
+      "vpaddd     %%ymm0,%%ymm4,%%ymm0           \n"
+      "sub        $0x40,%2                       \n"
+      "jg         1b                             \n"
+
+      "vpermq     $0xb1,%%ymm0,%%ymm1            \n"
+      "vpaddd     %%ymm1,%%ymm0,%%ymm0           \n"
+      "vpermq     $0xaa,%%ymm0,%%ymm1            \n"
+      "vpaddd     %%ymm1,%%ymm0,%%ymm0           \n"
+      "vmovd      %%xmm0, %3                     \n"
+      "vzeroupper                                \n"
+      : "+r"(src_a),       // %0
+        "+r"(src_b),       // %1
+        "+r"(count),       // %2
+        "=r"(diff)         // %3
+      : "m"(kNibbleMask),  // %4
+        "m"(kBitCount)     // %5
+      : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
+
+  return diff;
+}
+#endif  // HAS_HAMMINGDISTANCE_AVX2
 
 uint32 SumSquareError_SSE2(const uint8* src_a, const uint8* src_b, int count) {

pjproject/trunk/third_party/yuv/source/compare_neon.cc

@@ -27 +27 @@
   uint32 diff;
 
-  asm volatile (
-    "vmov.u16   q4, #0                         \n"  // accumulator
+  asm volatile(
+      "vmov.u16   q4, #0                         \n"  // accumulator
 
-  "1:                                          \n"
-    "vld1.8     {q0, q1}, [%0]!                \n"
-    "vld1.8     {q2, q3}, [%1]!                \n"
-    "veor.32    q0, q0, q2                     \n"
-    "veor.32    q1, q1, q3                     \n"
-    "vcnt.i8    q0, q0                         \n"
-    "vcnt.i8    q1, q1                         \n"
-    "subs       %2, %2, #32                    \n"
-    "vadd.u8    q0, q0, q1                     \n"  // 16 byte counts
-    "vpadal.u8  q4, q0                         \n"  // 8 shorts
-    "bgt        1b                             \n"
+      "1:                                        \n"
+      "vld1.8     {q0, q1}, [%0]!                \n"
+      "vld1.8     {q2, q3}, [%1]!                \n"
+      "veor.32    q0, q0, q2                     \n"
+      "veor.32    q1, q1, q3                     \n"
+      "vcnt.i8    q0, q0                         \n"
+      "vcnt.i8    q1, q1                         \n"
+      "subs       %2, %2, #32                    \n"
+      "vadd.u8    q0, q0, q1                     \n"  // 16 byte counts
+      "vpadal.u8  q4, q0                         \n"  // 8 shorts
+      "bgt        1b                             \n"
 
-    "vpaddl.u16 q0, q4                         \n"  // 4 ints
-    "vpadd.u32  d0, d0, d1                     \n"
-    "vpadd.u32  d0, d0, d0                     \n"
-    "vmov.32    %3, d0[0]                      \n"
- 
-    : "+r"(src_a),
-      "+r"(src_b),
-      "+r"(count),
-      "=r"(diff)
-    :
-    :  "cc", "q0", "q1", "q2", "q3", "q4");
+      "vpaddl.u16 q0, q4                         \n"  // 4 ints
+      "vpadd.u32  d0, d0, d1                     \n"
+      "vpadd.u32  d0, d0, d0                     \n"
+      "vmov.32    %3, d0[0]                      \n"
+
+      : "+r"(src_a), "+r"(src_b), "+r"(count), "=r"(diff)
+      :
+      : "cc", "q0", "q1", "q2", "q3", "q4");
   return diff;
 }
@@ -58 +55 @@
 uint32 SumSquareError_NEON(const uint8* src_a, const uint8* src_b, int count) {
   uint32 sse;
-  asm volatile (
-    "vmov.u8    q8, #0                         \n"
-    "vmov.u8    q10, #0                        \n"
-    "vmov.u8    q9, #0                         \n"
-    "vmov.u8    q11, #0                        \n"
+  asm volatile(
+      "vmov.u8    q8, #0                         \n"
+      "vmov.u8    q10, #0                        \n"
+      "vmov.u8    q9, #0                         \n"
+      "vmov.u8    q11, #0                        \n"
 
-  "1:                                          \n"
-    "vld1.8     {q0}, [%0]!                    \n"
-    "vld1.8     {q1}, [%1]!                    \n"
-    "subs       %2, %2, #16                    \n"
-    "vsubl.u8   q2, d0, d2                     \n"
-    "vsubl.u8   q3, d1, d3                     \n"
-    "vmlal.s16  q8, d4, d4                     \n"
-    "vmlal.s16  q9, d6, d6                     \n"
-    "vmlal.s16  q10, d5, d5                    \n"
-    "vmlal.s16  q11, d7, d7                    \n"
-    "bgt        1b                             \n"
+      "1:                                        \n"
+      "vld1.8     {q0}, [%0]!                    \n"
+      "vld1.8     {q1}, [%1]!                    \n"
+      "subs       %2, %2, #16                    \n"
+      "vsubl.u8   q2, d0, d2                     \n"
+      "vsubl.u8   q3, d1, d3                     \n"
+      "vmlal.s16  q8, d4, d4                     \n"
+      "vmlal.s16  q9, d6, d6                     \n"
+      "vmlal.s16  q10, d5, d5                    \n"
+      "vmlal.s16  q11, d7, d7                    \n"
+      "bgt        1b                             \n"
 
-    "vadd.u32   q8, q8, q9                     \n"
-    "vadd.u32   q10, q10, q11                  \n"
-    "vadd.u32   q11, q8, q10                   \n"
-    "vpaddl.u32 q1, q11                        \n"
-    "vadd.u64   d0, d2, d3                     \n"
-    "vmov.32    %3, d0[0]                      \n"
-    : "+r"(src_a),
-      "+r"(src_b),
-      "+r"(count),
-      "=r"(sse)
-    :
-    : "memory", "cc", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11");
+      "vadd.u32   q8, q8, q9                     \n"
+      "vadd.u32   q10, q10, q11                  \n"
+      "vadd.u32   q11, q8, q10                   \n"
+      "vpaddl.u32 q1, q11                        \n"
+      "vadd.u64   d0, d2, d3                     \n"
+      "vmov.32    %3, d0[0]                      \n"
+      : "+r"(src_a), "+r"(src_b), "+r"(count), "=r"(sse)
+      :
+      : "memory", "cc", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11");
   return sse;
 }

pjproject/trunk/third_party/yuv/source/compare_neon64.cc

@@ -25 +25 @@
 uint32 HammingDistance_NEON(const uint8* src_a, const uint8* src_b, int count) {
   uint32 diff;
-  asm volatile (
-    "movi       v4.8h, #0                      \n"
+  asm volatile(
+      "movi       v4.8h, #0                      \n"
 
-  "1:                                          \n"
-    "ld1        {v0.16b, v1.16b}, [%0], #32    \n"
-    "ld1        {v2.16b, v3.16b}, [%1], #32    \n"
-    "eor        v0.16b, v0.16b, v2.16b         \n"
-    "eor        v1.16b, v1.16b, v3.16b         \n" 
-    "cnt        v0.16b, v0.16b                 \n"
-    "cnt        v1.16b, v1.16b                 \n"
-    "subs       %w2, %w2, #32                  \n"
-    "add        v0.16b, v0.16b, v1.16b         \n"
-    "uadalp     v4.8h, v0.16b                  \n"
-    "b.gt       1b                             \n"
+      "1:                                        \n"
+      "ld1        {v0.16b, v1.16b}, [%0], #32    \n"
+      "ld1        {v2.16b, v3.16b}, [%1], #32    \n"
+      "eor        v0.16b, v0.16b, v2.16b         \n"
+      "eor        v1.16b, v1.16b, v3.16b         \n"
+      "cnt        v0.16b, v0.16b                 \n"
+      "cnt        v1.16b, v1.16b                 \n"
+      "subs       %w2, %w2, #32                  \n"
+      "add        v0.16b, v0.16b, v1.16b         \n"
+      "uadalp     v4.8h, v0.16b                  \n"
+      "b.gt       1b                             \n"
 
-    "uaddlv     s4, v4.8h                      \n"
-    "fmov       %w3, s4                        \n"
-    : "+r"(src_a),
-      "+r"(src_b),
-      "+r"(count),
-      "=r"(diff)
-    :
-    : "cc", "v0", "v1", "v2", "v3", "v4");
+      "uaddlv     s4, v4.8h                      \n"
+      "fmov       %w3, s4                        \n"
+      : "+r"(src_a), "+r"(src_b), "+r"(count), "=r"(diff)
+      :
+      : "cc", "v0", "v1", "v2", "v3", "v4");
   return diff;
 }
@@ -53 +50 @@
 uint32 SumSquareError_NEON(const uint8* src_a, const uint8* src_b, int count) {
   uint32 sse;
-  asm volatile (
-    "eor        v16.16b, v16.16b, v16.16b      \n"
-    "eor        v18.16b, v18.16b, v18.16b      \n"
-    "eor        v17.16b, v17.16b, v17.16b      \n"
-    "eor        v19.16b, v19.16b, v19.16b      \n"
+  asm volatile(
+      "eor        v16.16b, v16.16b, v16.16b      \n"
+      "eor        v18.16b, v18.16b, v18.16b      \n"
+      "eor        v17.16b, v17.16b, v17.16b      \n"
+      "eor        v19.16b, v19.16b, v19.16b      \n"
 
-  "1:                                          \n"
-    "ld1        {v0.16b}, [%0], #16            \n"
-    "ld1        {v1.16b}, [%1], #16            \n"
-    "subs       %w2, %w2, #16                  \n"
-    "usubl      v2.8h, v0.8b, v1.8b            \n"
-    "usubl2     v3.8h, v0.16b, v1.16b          \n"
-    "smlal      v16.4s, v2.4h, v2.4h           \n"
-    "smlal      v17.4s, v3.4h, v3.4h           \n"
-    "smlal2     v18.4s, v2.8h, v2.8h           \n"
-    "smlal2     v19.4s, v3.8h, v3.8h           \n"
-    "b.gt       1b                             \n"
+      "1:                                        \n"
+      "ld1        {v0.16b}, [%0], #16            \n"
+      "ld1        {v1.16b}, [%1], #16            \n"
+      "subs       %w2, %w2, #16                  \n"
+      "usubl      v2.8h, v0.8b, v1.8b            \n"
+      "usubl2     v3.8h, v0.16b, v1.16b          \n"
+      "smlal      v16.4s, v2.4h, v2.4h           \n"
+      "smlal      v17.4s, v3.4h, v3.4h           \n"
+      "smlal2     v18.4s, v2.8h, v2.8h           \n"
+      "smlal2     v19.4s, v3.8h, v3.8h           \n"
+      "b.gt       1b                             \n"
 
-    "add        v16.4s, v16.4s, v17.4s         \n"
-    "add        v18.4s, v18.4s, v19.4s         \n"
-    "add        v19.4s, v16.4s, v18.4s         \n"
-    "addv       s0, v19.4s                     \n"
-    "fmov       %w3, s0                        \n"
-    : "+r"(src_a),
-      "+r"(src_b),
-      "+r"(count),
-      "=r"(sse)
-    :
-    : "cc", "v0", "v1", "v2", "v3", "v16", "v17", "v18", "v19");
+      "add        v16.4s, v16.4s, v17.4s         \n"
+      "add        v18.4s, v18.4s, v19.4s         \n"
+      "add        v19.4s, v16.4s, v18.4s         \n"
+      "addv       s0, v19.4s                     \n"
+      "fmov       %w3, s0                        \n"
+      : "+r"(src_a), "+r"(src_b), "+r"(count), "=r"(sse)
+      :
+      : "cc", "v0", "v1", "v2", "v3", "v16", "v17", "v18", "v19");
   return sse;
 }

pjproject/trunk/third_party/yuv/source/compare_win.cc

@@ -10 +10 @@
 
 #include "libyuv/basic_types.h"
+
+#include "libyuv/compare_row.h"
 #include "libyuv/row.h"
+
+#if defined(_MSC_VER)
+#include <intrin.h>  // For __popcnt
+#endif
 
 #ifdef __cplusplus
@@ -17 +23 @@
 #endif
 
+// This module is for 32 bit Visual C x86 and clangcl
 #if !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86) && defined(_MSC_VER)
-#if (_MSC_VER >= 1900)
-__declspec(naked)
-#else
-__declspec(naked) __declspec(align(16))
-#endif
-uint32 SumSquareError_SSE2(const uint8* src_a, const uint8* src_b, int count) {
-  __asm {
-    mov        eax, [esp + 4]    // src_a
-    mov        edx, [esp + 8]    // src_b
-    mov        ecx, [esp + 12]   // count
+
+uint32 HammingDistance_SSE42(const uint8* src_a,
+                             const uint8* src_b,
+                             int count) {
+  uint32 diff = 0u;
+
+  int i;
+  for (i = 0; i < count - 3; i += 4) {
+    uint32 x = *((uint32*)src_a) ^ *((uint32*)src_b);
+    src_a += 4;
+    src_b += 4;
+    diff += __popcnt(x);
+  }
+  return diff;
+}
+
+__declspec(naked) uint32
+    SumSquareError_SSE2(const uint8* src_a, const uint8* src_b, int count) {
+  __asm {
+    mov        eax, [esp + 4]  // src_a
+    mov        edx, [esp + 8]  // src_b
+    mov        ecx, [esp + 12]  // count
     pxor       xmm0, xmm0
     pxor       xmm5, xmm5
 
-    align      4
-  wloop:
-    movdqa     xmm1, [eax]
+  wloop:
+    movdqu     xmm1, [eax]
     lea        eax,  [eax + 16]
-    movdqa     xmm2, [edx]
+    movdqu     xmm2, [edx]
     lea        edx,  [edx + 16]
-    sub        ecx, 16
     movdqa     xmm3, xmm1  // abs trick
     psubusb    xmm1, xmm2
@@ -49 +66 @@
     paddd      xmm0, xmm1
     paddd      xmm0, xmm2
+    sub        ecx, 16
     jg         wloop
 
@@ -63 +81 @@
 #if _MSC_VER >= 1700
 // C4752: found Intel(R) Advanced Vector Extensions; consider using /arch:AVX.
-#pragma warning(disable: 4752)
-#if (_MSC_VER >= 1900)
-__declspec(naked)
-#else
-__declspec(naked) __declspec(align(16))
-#endif
-uint32 SumSquareError_AVX2(const uint8* src_a, const uint8* src_b, int count) {
-  __asm {
-    mov        eax, [esp + 4]    // src_a
-    mov        edx, [esp + 8]    // src_b
-    mov        ecx, [esp + 12]   // count
+#pragma warning(disable : 4752)
+__declspec(naked) uint32
+    SumSquareError_AVX2(const uint8* src_a, const uint8* src_b, int count) {
+  __asm {
+    mov        eax, [esp + 4]  // src_a
+    mov        edx, [esp + 8]  // src_b
+    mov        ecx, [esp + 12]  // count
     vpxor      ymm0, ymm0, ymm0  // sum
     vpxor      ymm5, ymm5, ymm5  // constant 0 for unpck
     sub        edx, eax
 
-    align      4
   wloop:
     vmovdqu    ymm1, [eax]
     vmovdqu    ymm2, [eax + edx]
     lea        eax,  [eax + 32]
-    sub        ecx, 32
     vpsubusb   ymm3, ymm1, ymm2  // abs difference trick
     vpsubusb   ymm2, ymm2, ymm1
@@ -93 +105 @@
     vpaddd     ymm0, ymm0, ymm1
     vpaddd     ymm0, ymm0, ymm2
+    sub        ecx, 32
     jg         wloop
 
@@ -108 +121 @@
 #endif  // _MSC_VER >= 1700
 
-#define HAS_HASHDJB2_SSE41
-static uvec32 kHash16x33 = { 0x92d9e201, 0, 0, 0 };  // 33 ^ 16
-static uvec32 kHashMul0 = {
-  0x0c3525e1,  // 33 ^ 15
-  0xa3476dc1,  // 33 ^ 14
-  0x3b4039a1,  // 33 ^ 13
-  0x4f5f0981,  // 33 ^ 12
-};
-static uvec32 kHashMul1 = {
-  0x30f35d61,  // 33 ^ 11
-  0x855cb541,  // 33 ^ 10
-  0x040a9121,  // 33 ^ 9
-  0x747c7101,  // 33 ^ 8
-};
-static uvec32 kHashMul2 = {
-  0xec41d4e1,  // 33 ^ 7
-  0x4cfa3cc1,  // 33 ^ 6
-  0x025528a1,  // 33 ^ 5
-  0x00121881,  // 33 ^ 4
-};
-static uvec32 kHashMul3 = {
-  0x00008c61,  // 33 ^ 3
-  0x00000441,  // 33 ^ 2
-  0x00000021,  // 33 ^ 1
-  0x00000001,  // 33 ^ 0
-};
-
-// 27: 66 0F 38 40 C6     pmulld      xmm0,xmm6
-// 44: 66 0F 38 40 DD     pmulld      xmm3,xmm5
-// 59: 66 0F 38 40 E5     pmulld      xmm4,xmm5
-// 72: 66 0F 38 40 D5     pmulld      xmm2,xmm5
-// 83: 66 0F 38 40 CD     pmulld      xmm1,xmm5
-#define pmulld(reg) _asm _emit 0x66 _asm _emit 0x0F _asm _emit 0x38 \
-    _asm _emit 0x40 _asm _emit reg
-
-#if (_MSC_VER >= 1900)
-__declspec(naked)
-#else
-__declspec(naked) __declspec(align(16))
-#endif
-uint32 HashDjb2_SSE41(const uint8* src, int count, uint32 seed) {
-  __asm {
-    mov        eax, [esp + 4]    // src
-    mov        ecx, [esp + 8]    // count
+uvec32 kHash16x33 = {0x92d9e201, 0, 0, 0};  // 33 ^ 16
+uvec32 kHashMul0 = {
+    0x0c3525e1,  // 33 ^ 15
+    0xa3476dc1,  // 33 ^ 14
+    0x3b4039a1,  // 33 ^ 13
+    0x4f5f0981,  // 33 ^ 12
+};
+uvec32 kHashMul1 = {
+    0x30f35d61,  // 33 ^ 11
+    0x855cb541,  // 33 ^ 10
+    0x040a9121,  // 33 ^ 9
+    0x747c7101,  // 33 ^ 8
+};
+uvec32 kHashMul2 = {
+    0xec41d4e1,  // 33 ^ 7
+    0x4cfa3cc1,  // 33 ^ 6
+    0x025528a1,  // 33 ^ 5
+    0x00121881,  // 33 ^ 4
+};
+uvec32 kHashMul3 = {
+    0x00008c61,  // 33 ^ 3
+    0x00000441,  // 33 ^ 2
+    0x00000021,  // 33 ^ 1
+    0x00000001,  // 33 ^ 0
+};
+
+__declspec(naked) uint32
+    HashDjb2_SSE41(const uint8* src, int count, uint32 seed) {
+  __asm {
+    mov        eax, [esp + 4]  // src
+    mov        ecx, [esp + 8]  // count
     movd       xmm0, [esp + 12]  // seed
 
-    pxor       xmm7, xmm7        // constant 0 for unpck
-    movdqa     xmm6, kHash16x33
-
-    align      4
-  wloop:
-    movdqu     xmm1, [eax]       // src[0-15]
+    pxor       xmm7, xmm7  // constant 0 for unpck
+    movdqa     xmm6, xmmword ptr kHash16x33
+
+  wloop:
+    movdqu     xmm1, [eax]  // src[0-15]
     lea        eax, [eax + 16]
-    pmulld(0xc6)                 // pmulld      xmm0,xmm6  hash *= 33 ^ 16
-    movdqa     xmm5, kHashMul0
+    pmulld     xmm0, xmm6  // hash *= 33 ^ 16
+    movdqa     xmm5, xmmword ptr kHashMul0
     movdqa     xmm2, xmm1
-    punpcklbw  xmm2, xmm7        // src[0-7]
+    punpcklbw  xmm2, xmm7  // src[0-7]
     movdqa     xmm3, xmm2
-    punpcklwd  xmm3, xmm7        // src[0-3]
-    pmulld(0xdd)                 // pmulld     xmm3, xmm5
-    movdqa     xmm5, kHashMul1
+    punpcklwd  xmm3, xmm7  // src[0-3]
+    pmulld     xmm3, xmm5
+    movdqa     xmm5, xmmword ptr kHashMul1
     movdqa     xmm4, xmm2
-    punpckhwd  xmm4, xmm7        // src[4-7]
-    pmulld(0xe5)                 // pmulld     xmm4, xmm5
-    movdqa     xmm5, kHashMul2
-    punpckhbw  xmm1, xmm7        // src[8-15]
+    punpckhwd  xmm4, xmm7  // src[4-7]
+    pmulld     xmm4, xmm5
+    movdqa     xmm5, xmmword ptr kHashMul2
+    punpckhbw  xmm1, xmm7  // src[8-15]
     movdqa     xmm2, xmm1
-    punpcklwd  xmm2, xmm7        // src[8-11]
-    pmulld(0xd5)                 // pmulld     xmm2, xmm5
-    movdqa     xmm5, kHashMul3
-    punpckhwd  xmm1, xmm7        // src[12-15]
-    pmulld(0xcd)                 // pmulld     xmm1, xmm5
-    paddd      xmm3, xmm4        // add 16 results
+    punpcklwd  xmm2, xmm7  // src[8-11]
+    pmulld     xmm2, xmm5
+    movdqa     xmm5, xmmword ptr kHashMul3
+    punpckhwd  xmm1, xmm7  // src[12-15]
+    pmulld     xmm1, xmm5
+    paddd      xmm3, xmm4  // add 16 results
     paddd      xmm1, xmm2
-    sub        ecx, 16
     paddd      xmm1, xmm3
 
@@ -190 +188 @@
     paddd      xmm1, xmm2
     paddd      xmm0, xmm1
-    jg         wloop
-
-    movd       eax, xmm0         // return hash
+    sub        ecx, 16
+    jg         wloop
+
+    movd       eax, xmm0  // return hash
     ret
   }
@@ -199 +198 @@
 // Visual C 2012 required for AVX2.
 #if _MSC_VER >= 1700
-#if (_MSC_VER >= 1900)
-__declspec(naked)
-#else
-__declspec(naked) __declspec(align(16))
-#endif
-uint32 HashDjb2_AVX2(const uint8* src, int count, uint32 seed) {
-  __asm {
-    mov        eax, [esp + 4]    // src
-    mov        ecx, [esp + 8]    // count
-    movd       xmm0, [esp + 12]  // seed
-    movdqa     xmm6, kHash16x33
-
-    align      4
-  wloop:
-    vpmovzxbd  xmm3, dword ptr [eax]  // src[0-3]
-    pmulld     xmm0, xmm6  // hash *= 33 ^ 16
-    vpmovzxbd  xmm4, dword ptr [eax + 4]  // src[4-7]
-    pmulld     xmm3, kHashMul0
-    vpmovzxbd  xmm2, dword ptr [eax + 8]  // src[8-11]
-    pmulld     xmm4, kHashMul1
-    vpmovzxbd  xmm1, dword ptr [eax + 12]  // src[12-15]
-    pmulld     xmm2, kHashMul2
+__declspec(naked) uint32
+    HashDjb2_AVX2(const uint8* src, int count, uint32 seed) {
+  __asm {
+    mov        eax, [esp + 4]  // src
+    mov        ecx, [esp + 8]  // count
+    vmovd      xmm0, [esp + 12]  // seed
+
+  wloop:
+    vpmovzxbd  xmm3, [eax]  // src[0-3]
+    vpmulld    xmm0, xmm0, xmmword ptr kHash16x33  // hash *= 33 ^ 16
+    vpmovzxbd  xmm4, [eax + 4]  // src[4-7]
+    vpmulld    xmm3, xmm3, xmmword ptr kHashMul0
+    vpmovzxbd  xmm2, [eax + 8]  // src[8-11]
+    vpmulld    xmm4, xmm4, xmmword ptr kHashMul1
+    vpmovzxbd  xmm1, [eax + 12]  // src[12-15]
+    vpmulld    xmm2, xmm2, xmmword ptr kHashMul2
     lea        eax, [eax + 16]
-    pmulld     xmm1, kHashMul3
-    paddd      xmm3, xmm4        // add 16 results
-    paddd      xmm1, xmm2
+    vpmulld    xmm1, xmm1, xmmword ptr kHashMul3
+    vpaddd     xmm3, xmm3, xmm4  // add 16 results
+    vpaddd     xmm1, xmm1, xmm2
+    vpaddd     xmm1, xmm1, xmm3
+    vpshufd    xmm2, xmm1, 0x0e  // upper 2 dwords
+    vpaddd     xmm1, xmm1,xmm2
+    vpshufd    xmm2, xmm1, 0x01
+    vpaddd     xmm1, xmm1, xmm2
+    vpaddd     xmm0, xmm0, xmm1
     sub        ecx, 16
-    paddd      xmm1, xmm3
-    pshufd     xmm2, xmm1, 0x0e  // upper 2 dwords
-    paddd      xmm1, xmm2
-    pshufd     xmm2, xmm1, 0x01
-    paddd      xmm1, xmm2
-    paddd      xmm0, xmm1
-    jg         wloop
-
-    movd       eax, xmm0         // return hash
+    jg         wloop
+
+    vmovd      eax, xmm0  // return hash
+    vzeroupper
     ret
   }
@@ -240 +234 @@
 #endif  // _MSC_VER >= 1700
 
-#endif  // !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86) && defined(_MSC_VER)
+#endif  // !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86)
 
 #ifdef __cplusplus

pjproject/trunk/third_party/yuv/source/convert_from.cc

@@ -658 +658 @@
 }
 
+// Convert H420 to RGB24.
+LIBYUV_API
+int H420ToRGB24(const uint8* src_y,
+                int src_stride_y,
+                const uint8* src_u,
+                int src_stride_u,
+                const uint8* src_v,
+                int src_stride_v,
+                uint8* dst_rgb24,
+                int dst_stride_rgb24,
+                int width,
+                int height) {
+  return I420ToRGB24Matrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+                           src_stride_v, dst_rgb24, dst_stride_rgb24,
+                           &kYuvH709Constants, width, height);
+}
+
+// Convert H420 to RAW.
+LIBYUV_API
+int H420ToRAW(const uint8* src_y,
+              int src_stride_y,
+              const uint8* src_u,
+              int src_stride_u,
+              const uint8* src_v,
+              int src_stride_v,
+              uint8* dst_raw,
+              int dst_stride_raw,
+              int width,
+              int height) {
+  return I420ToRGB24Matrix(src_y, src_stride_y, src_v,
+                           src_stride_v,  // Swap U and V
+                           src_u, src_stride_u, dst_raw, dst_stride_raw,
+                           &kYvuH709Constants,  // Use Yvu matrix
+                           width, height);
+}
+
 // Convert I420 to ARGB1555.
 LIBYUV_API
@@ -1076 +1112 @@
       I422ToARGBRow(src_y, src_u, src_v, row_argb, &kYuvI601Constants, width);
       ARGBToRGB565DitherRow(row_argb, dst_rgb565,
-                            *(uint32*)(dither4x4 + ((y & 3) << 2)),
-                            width);  // NOLINT
+                            *(uint32*)(dither4x4 + ((y & 3) << 2)),  // NOLINT
+                            width);                                  // NOLINT
       dst_rgb565 += dst_stride_rgb565;
       src_y += src_stride_y;

pjproject/trunk/third_party/yuv/source/cpu_id.cc

@@ -125 +125 @@
   int xcr0 = 0;
 #if defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 160040219)
-  xcr0 = _xgetbv(0);  // VS2010 SP1 required.
+  xcr0 = (int)_xgetbv(0);  // VS2010 SP1 required.  NOLINT
 #elif defined(__i386__) || defined(__x86_64__)
   asm(".byte 0x0f, 0x01, 0xd0" : "=a"(xcr0) : "c"(0) : "%edx");
@@ -243 +243 @@
     // Detect AVX512bw
     if ((GetXCR0() & 0xe0) == 0xe0) {
-      cpu_info |= (cpu_info7[1] & 0x40000000) ? kCpuHasAVX3 : 0;
-    }
-  }
-
+      cpu_info |= (cpu_info7[1] & 0x40000000) ? kCpuHasAVX512BW : 0;
+      cpu_info |= (cpu_info7[1] & 0x80000000) ? kCpuHasAVX512VL : 0;
+      cpu_info |= (cpu_info7[2] & 0x00000002) ? kCpuHasAVX512VBMI : 0;
+      cpu_info |= (cpu_info7[2] & 0x00000040) ? kCpuHasAVX512VBMI2 : 0;
+      cpu_info |= (cpu_info7[2] & 0x00001000) ? kCpuHasAVX512VBITALG : 0;
+      cpu_info |= (cpu_info7[2] & 0x00004000) ? kCpuHasAVX512VPOPCNTDQ : 0;
+      cpu_info |= (cpu_info7[2] & 0x00000100) ? kCpuHasGFNI : 0;
+    }
+  }
+
+  // TODO(fbarchard): Consider moving these to gtest
   // Environment variable overrides for testing.
   if (TestEnv("LIBYUV_DISABLE_X86")) {
@@ -275 +282 @@
     cpu_info &= ~kCpuHasFMA3;
   }
-  if (TestEnv("LIBYUV_DISABLE_AVX3")) {
-    cpu_info &= ~kCpuHasAVX3;
-  }
   if (TestEnv("LIBYUV_DISABLE_F16C")) {
     cpu_info &= ~kCpuHasF16C;
+  }
+  if (TestEnv("LIBYUV_DISABLE_AVX512BW")) {
+    cpu_info &= ~kCpuHasAVX512BW;
   }
 

pjproject/trunk/third_party/yuv/source/mjpeg_decoder.cc

@@ -13 +13 @@
 #ifdef HAVE_JPEG
 #include <assert.h>
-
-#ifdef __cplusplus
-#include <new>
-#endif
 
 #if !defined(__pnacl__) && !defined(__CLR_VER) && \

pjproject/trunk/third_party/yuv/source/mjpeg_validate.cc

@@ -25 +25 @@
     while (it < end) {
       // TODO(fbarchard): scan for 0xd9 instead.
-      it = static_cast<const uint8*>(memchr(it, 0xff, end - it));
+      it = (const uint8*)(memchr(it, 0xff, end - it));
       if (it == NULL) {
         break;

pjproject/trunk/third_party/yuv/source/planar_functions.cc

@@ -322 +322 @@
   }
 #endif
+#if defined(HAS_SPLITUVROW_MSA)
+  if (TestCpuFlag(kCpuHasMSA)) {
+    SplitUVRow = SplitUVRow_Any_MSA;
+    if (IS_ALIGNED(width, 32)) {
+      SplitUVRow = SplitUVRow_MSA;
+    }
+  }
+#endif
 
   for (y = 0; y < height; ++y) {
@@ -397 +405 @@
     src_v += src_stride_v;
     dst_uv += dst_stride_uv;
+  }
+}
+
+// Support function for NV12 etc RGB channels.
+// Width and height are plane sizes (typically half pixel width).
+LIBYUV_API
+void SplitRGBPlane(const uint8* src_rgb,
+                   int src_stride_rgb,
+                   uint8* dst_r,
+                   int dst_stride_r,
+                   uint8* dst_g,
+                   int dst_stride_g,
+                   uint8* dst_b,
+                   int dst_stride_b,
+                   int width,
+                   int height) {
+  int y;
+  void (*SplitRGBRow)(const uint8* src_rgb, uint8* dst_r, uint8* dst_g,
+                      uint8* dst_b, int width) = SplitRGBRow_C;
+  // Negative height means invert the image.
+  if (height < 0) {
+    height = -height;
+    dst_r = dst_r + (height - 1) * dst_stride_r;
+    dst_g = dst_g + (height - 1) * dst_stride_g;
+    dst_b = dst_b + (height - 1) * dst_stride_b;
+    dst_stride_r = -dst_stride_r;
+    dst_stride_g = -dst_stride_g;
+    dst_stride_b = -dst_stride_b;
+  }
+  // Coalesce rows.
+  if (src_stride_rgb == width * 3 && dst_stride_r == width &&
+      dst_stride_g == width && dst_stride_b == width) {
+    width *= height;
+    height = 1;
+    src_stride_rgb = dst_stride_r = dst_stride_g = dst_stride_b = 0;
+  }
+#if defined(HAS_SPLITRGBROW_SSSE3)
+  if (TestCpuFlag(kCpuHasSSSE3)) {
+    SplitRGBRow = SplitRGBRow_Any_SSSE3;
+    if (IS_ALIGNED(width, 16)) {
+      SplitRGBRow = SplitRGBRow_SSSE3;
+    }
+  }
+#endif
+#if defined(HAS_SPLITRGBROW_NEON)
+  if (TestCpuFlag(kCpuHasNEON)) {
+    SplitRGBRow = SplitRGBRow_Any_NEON;
+    if (IS_ALIGNED(width, 16)) {
+      SplitRGBRow = SplitRGBRow_NEON;
+    }
+  }
+#endif
+
+  for (y = 0; y < height; ++y) {
+    // Copy a row of RGB.
+    SplitRGBRow(src_rgb, dst_r, dst_g, dst_b, width);
+    dst_r += dst_stride_r;
+    dst_g += dst_stride_g;
+    dst_b += dst_stride_b;
+    src_rgb += src_stride_rgb;
+  }
+}
+
+LIBYUV_API
+void MergeRGBPlane(const uint8* src_r,
+                   int src_stride_r,
+                   const uint8* src_g,
+                   int src_stride_g,
+                   const uint8* src_b,
+                   int src_stride_b,
+                   uint8* dst_rgb,
+                   int dst_stride_rgb,
+                   int width,
+                   int height) {
+  int y;
+  void (*MergeRGBRow)(const uint8* src_r, const uint8* src_g,
+                      const uint8* src_b, uint8* dst_rgb, int width) =
+      MergeRGBRow_C;
+  // Coalesce rows.
+  // Negative height means invert the image.
+  if (height < 0) {
+    height = -height;
+    dst_rgb = dst_rgb + (height - 1) * dst_stride_rgb;
+    dst_stride_rgb = -dst_stride_rgb;
+  }
+  // Coalesce rows.
+  if (src_stride_r == width && src_stride_g == width && src_stride_b == width &&
+      dst_stride_rgb == width * 3) {
+    width *= height;
+    height = 1;
+    src_stride_r = src_stride_g = src_stride_b = dst_stride_rgb = 0;
+  }
+#if defined(HAS_MERGERGBROW_SSSE3)
+  if (TestCpuFlag(kCpuHasSSSE3)) {
+    MergeRGBRow = MergeRGBRow_Any_SSSE3;
+    if (IS_ALIGNED(width, 16)) {
+      MergeRGBRow = MergeRGBRow_SSSE3;
+    }
+  }
+#endif
+#if defined(HAS_MERGERGBROW_NEON)
+  if (TestCpuFlag(kCpuHasNEON)) {
+    MergeRGBRow = MergeRGBRow_Any_NEON;
+    if (IS_ALIGNED(width, 16)) {
+      MergeRGBRow = MergeRGBRow_NEON;
+    }
+  }
+#endif
+
+  for (y = 0; y < height; ++y) {
+    // Merge a row of U and V into a row of RGB.
+    MergeRGBRow(src_r, src_g, src_b, dst_rgb, width);
+    src_r += src_stride_r;
+    src_g += src_stride_g;
+    src_b += src_stride_b;
+    dst_rgb += dst_stride_rgb;
   }
 }
@@ -845 +969 @@
   if (TestCpuFlag(kCpuHasNEON)) {
     ARGBBlendRow = ARGBBlendRow_NEON;
+  }
+#endif
+#if defined(HAS_ARGBBLENDROW_MSA)
+  if (TestCpuFlag(kCpuHasMSA)) {
+    ARGBBlendRow = ARGBBlendRow_MSA;
   }
 #endif
@@ -1575 +1704 @@
   }
 #endif
+#if defined(HAS_SETROW_MSA)
+  if (TestCpuFlag(kCpuHasMSA) && IS_ALIGNED(width, 16)) {
+    SetRow = SetRow_MSA;
+  }
+#endif
 
   // Set plane
@@ -1975 +2109 @@
   }
 #endif
+#if defined(HAS_ARGBCOLORMATRIXROW_MSA)
+  if (TestCpuFlag(kCpuHasMSA) && IS_ALIGNED(width, 8)) {
+    ARGBColorMatrixRow = ARGBColorMatrixRow_MSA;
+  }
+#endif
   for (y = 0; y < height; ++y) {
     ARGBColorMatrixRow(src_argb, dst_argb, matrix_argb, width);
@@ -2133 +2272 @@
   if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 8)) {
     ARGBQuantizeRow = ARGBQuantizeRow_NEON;
+  }
+#endif
+#if defined(HAS_ARGBQUANTIZEROW_MSA)
+  if (TestCpuFlag(kCpuHasMSA) && IS_ALIGNED(width, 8)) {
+    ARGBQuantizeRow = ARGBQuantizeRow_MSA;
   }
 #endif
@@ -2620 +2764 @@
   }
 #endif
+#if defined(HAS_SOBELYROW_MSA)
+  if (TestCpuFlag(kCpuHasMSA)) {
+    SobelYRow = SobelYRow_MSA;
+  }
+#endif
 #if defined(HAS_SOBELXROW_SSE2)
   if (TestCpuFlag(kCpuHasSSE2)) {
@@ -2628 +2777 @@
   if (TestCpuFlag(kCpuHasNEON)) {
     SobelXRow = SobelXRow_NEON;
+  }
+#endif
+#if defined(HAS_SOBELXROW_MSA)
+  if (TestCpuFlag(kCpuHasMSA)) {
+    SobelXRow = SobelXRow_MSA;
   }
 #endif
@@ -2904 +3058 @@
   }
 #endif
+#if defined(HAS_HALFFLOATROW_MSA)
+  if (TestCpuFlag(kCpuHasMSA)) {
+    HalfFloatRow = HalfFloatRow_Any_MSA;
+    if (IS_ALIGNED(width, 32)) {
+      HalfFloatRow = HalfFloatRow_MSA;
+    }
+  }
+#endif
 
   for (y = 0; y < height; ++y) {
@@ -3047 +3209 @@
     ARGBExtractAlphaRow = IS_ALIGNED(width, 16) ? ARGBExtractAlphaRow_NEON
                                                 : ARGBExtractAlphaRow_Any_NEON;
+  }
+#endif
+#if defined(HAS_ARGBEXTRACTALPHAROW_MSA)
+  if (TestCpuFlag(kCpuHasMSA)) {
+    ARGBExtractAlphaRow = IS_ALIGNED(width, 16) ? ARGBExtractAlphaRow_MSA
+                                                : ARGBExtractAlphaRow_Any_MSA;
   }
 #endif
@@ -3158 +3326 @@
     if (IS_ALIGNED(width, 16)) {
       SplitUVRow = SplitUVRow_NEON;
+    }
+  }
+#endif
+#if defined(HAS_SPLITUVROW_MSA)
+  if (TestCpuFlag(kCpuHasMSA)) {
+    SplitUVRow = SplitUVRow_Any_MSA;
+    if (IS_ALIGNED(width, 32)) {
+      SplitUVRow = SplitUVRow_MSA;
     }
   }
@@ -3269 +3445 @@
   }
 #endif
+#if defined(HAS_SPLITUVROW_MSA)
+  if (TestCpuFlag(kCpuHasMSA)) {
+    SplitUVRow = SplitUVRow_Any_MSA;
+    if (IS_ALIGNED(width, 32)) {
+      SplitUVRow = SplitUVRow_MSA;
+    }
+  }
+#endif
 #if defined(HAS_INTERPOLATEROW_SSSE3)
   if (TestCpuFlag(kCpuHasSSSE3)) {

pjproject/trunk/third_party/yuv/source/rotate.cc

@@ -360 +360 @@
       IS_ALIGNED(src_stride, 4)) {
     MirrorUVRow = MirrorUVRow_DSPR2;
+  }
+#endif
+#if defined(HAS_MIRRORUVROW_MSA)
+  if (TestCpuFlag(kCpuHasMSA) && IS_ALIGNED(width, 32)) {
+    MirrorUVRow = MirrorUVRow_MSA;
   }
 #endif

pjproject/trunk/third_party/yuv/source/rotate_neon64.cc

@@ -31 +31 @@
                        int width) {
   const uint8* src_temp;
-  asm volatile (
-    // loops are on blocks of 8. loop will stop when
-    // counter gets to or below 0. starting the counter
-    // at w-8 allow for this
-    "sub         %w3, %w3, #8                     \n"
-
-    // handle 8x8 blocks. this should be the majority of the plane
-    "1:                                          \n"
+  asm volatile(
+      // loops are on blocks of 8. loop will stop when
+      // counter gets to or below 0. starting the counter
+      // at w-8 allow for this
+      "sub         %w3, %w3, #8                     \n"
+
+      // handle 8x8 blocks. this should be the majority of the plane
+      "1:                                          \n"
       "mov         %0, %1                        \n"
 
@@ -93 +93 @@
       "b.ge        1b                            \n"
 
-    // add 8 back to counter. if the result is 0 there are
-    // no residuals.
-    "adds        %w3, %w3, #8                    \n"
-    "b.eq        4f                              \n"
-
-    // some residual, so between 1 and 7 lines left to transpose
-    "cmp         %w3, #2                          \n"
-    "b.lt        3f                              \n"
-
-    "cmp         %w3, #4                          \n"
-    "b.lt        2f                              \n"
-
-    // 4x8 block
-    "mov         %0, %1                          \n"
-    "ld1     {v0.s}[0], [%0], %5                 \n"
-    "ld1     {v0.s}[1], [%0], %5                 \n"
-    "ld1     {v0.s}[2], [%0], %5                 \n"
-    "ld1     {v0.s}[3], [%0], %5                 \n"
-    "ld1     {v1.s}[0], [%0], %5                 \n"
-    "ld1     {v1.s}[1], [%0], %5                 \n"
-    "ld1     {v1.s}[2], [%0], %5                 \n"
-    "ld1     {v1.s}[3], [%0]                     \n"
-
-    "mov         %0, %2                          \n"
-
-    "ld1      {v2.16b}, [%4]                     \n"
-
-    "tbl      v3.16b, {v0.16b}, v2.16b           \n"
-    "tbl      v0.16b, {v1.16b}, v2.16b           \n"
-
-    // TODO(frkoenig): Rework shuffle above to
-    // write out with 4 instead of 8 writes.
-    "st1 {v3.s}[0], [%0], %6                     \n"
-    "st1 {v3.s}[1], [%0], %6                     \n"
-    "st1 {v3.s}[2], [%0], %6                     \n"
-    "st1 {v3.s}[3], [%0]                         \n"
-
-    "add         %0, %2, #4                      \n"
-    "st1 {v0.s}[0], [%0], %6                     \n"
-    "st1 {v0.s}[1], [%0], %6                     \n"
-    "st1 {v0.s}[2], [%0], %6                     \n"
-    "st1 {v0.s}[3], [%0]                         \n"
-
-    "add         %1, %1, #4                      \n"  // src += 4
-    "add         %2, %2, %6, lsl #2              \n"  // dst += 4 * dst_stride
-    "subs        %w3, %w3, #4                    \n"  // w   -= 4
-    "b.eq        4f                              \n"
-
-    // some residual, check to see if it includes a 2x8 block,
-    // or less
-    "cmp         %w3, #2                         \n"
-    "b.lt        3f                              \n"
-
-    // 2x8 block
-    "2:                                          \n"
-    "mov         %0, %1                          \n"
-    "ld1     {v0.h}[0], [%0], %5                 \n"
-    "ld1     {v1.h}[0], [%0], %5                 \n"
-    "ld1     {v0.h}[1], [%0], %5                 \n"
-    "ld1     {v1.h}[1], [%0], %5                 \n"
-    "ld1     {v0.h}[2], [%0], %5                 \n"
-    "ld1     {v1.h}[2], [%0], %5                 \n"
-    "ld1     {v0.h}[3], [%0], %5                 \n"
-    "ld1     {v1.h}[3], [%0]                     \n"
-
-    "trn2    v2.8b, v0.8b, v1.8b                 \n"
-    "trn1    v3.8b, v0.8b, v1.8b                 \n"
-
-    "mov         %0, %2                          \n"
-
-    "st1     {v3.8b}, [%0], %6                   \n"
-    "st1     {v2.8b}, [%0]                       \n"
-
-    "add         %1, %1, #2                      \n"  // src += 2
-    "add         %2, %2, %6, lsl #1              \n"  // dst += 2 * dst_stride
-    "subs        %w3, %w3,  #2                   \n"  // w   -= 2
-    "b.eq        4f                              \n"
-
-    // 1x8 block
-    "3:                                          \n"
-    "ld1         {v0.b}[0], [%1], %5             \n"
-    "ld1         {v0.b}[1], [%1], %5             \n"
-    "ld1         {v0.b}[2], [%1], %5             \n"
-    "ld1         {v0.b}[3], [%1], %5             \n"
-    "ld1         {v0.b}[4], [%1], %5             \n"
-    "ld1         {v0.b}[5], [%1], %5             \n"
-    "ld1         {v0.b}[6], [%1], %5             \n"
-    "ld1         {v0.b}[7], [%1]                 \n"
-
-    "st1         {v0.8b}, [%2]                   \n"
-
-    "4:                                          \n"
-
-    : "=&r"(src_temp),                            // %0
-      "+r"(src),                                  // %1
-      "+r"(dst),                                  // %2
-      "+r"(width)                                 // %3
-    : "r"(&kVTbl4x4Transpose),                    // %4
-      "r"(static_cast<ptrdiff_t>(src_stride)),    // %5
-      "r"(static_cast<ptrdiff_t>(dst_stride))     // %6
-    : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
-      "v17", "v18", "v19", "v20", "v21", "v22", "v23"
-  );
+      // add 8 back to counter. if the result is 0 there are
+      // no residuals.
+      "adds        %w3, %w3, #8                    \n"
+      "b.eq        4f                              \n"
+
+      // some residual, so between 1 and 7 lines left to transpose
+      "cmp         %w3, #2                          \n"
+      "b.lt        3f                              \n"
+
+      "cmp         %w3, #4                          \n"
+      "b.lt        2f                              \n"
+
+      // 4x8 block
+      "mov         %0, %1                          \n"
+      "ld1     {v0.s}[0], [%0], %5                 \n"
+      "ld1     {v0.s}[1], [%0], %5                 \n"
+      "ld1     {v0.s}[2], [%0], %5                 \n"
+      "ld1     {v0.s}[3], [%0], %5                 \n"
+      "ld1     {v1.s}[0], [%0], %5                 \n"
+      "ld1     {v1.s}[1], [%0], %5                 \n"
+      "ld1     {v1.s}[2], [%0], %5                 \n"
+      "ld1     {v1.s}[3], [%0]                     \n"
+
+      "mov         %0, %2                          \n"
+
+      "ld1      {v2.16b}, [%4]                     \n"
+
+      "tbl      v3.16b, {v0.16b}, v2.16b           \n"
+      "tbl      v0.16b, {v1.16b}, v2.16b           \n"
+
+      // TODO(frkoenig): Rework shuffle above to
+      // write out with 4 instead of 8 writes.
+      "st1 {v3.s}[0], [%0], %6                     \n"
+      "st1 {v3.s}[1], [%0], %6                     \n"
+      "st1 {v3.s}[2], [%0], %6                     \n"
+      "st1 {v3.s}[3], [%0]                         \n"
+
+      "add         %0, %2, #4                      \n"
+      "st1 {v0.s}[0], [%0], %6                     \n"
+      "st1 {v0.s}[1], [%0], %6                     \n"
+      "st1 {v0.s}[2], [%0], %6                     \n"
+      "st1 {v0.s}[3], [%0]                         \n"
+
+      "add         %1, %1, #4                      \n"  // src += 4
+      "add         %2, %2, %6, lsl #2              \n"  // dst += 4 * dst_stride
+      "subs        %w3, %w3, #4                    \n"  // w   -= 4
+      "b.eq        4f                              \n"
+
+      // some residual, check to see if it includes a 2x8 block,
+      // or less
+      "cmp         %w3, #2                         \n"
+      "b.lt        3f                              \n"
+
+      // 2x8 block
+      "2:                                          \n"
+      "mov         %0, %1                          \n"
+      "ld1     {v0.h}[0], [%0], %5                 \n"
+      "ld1     {v1.h}[0], [%0], %5                 \n"
+      "ld1     {v0.h}[1], [%0], %5                 \n"
+      "ld1     {v1.h}[1], [%0], %5                 \n"
+      "ld1     {v0.h}[2], [%0], %5                 \n"
+      "ld1     {v1.h}[2], [%0], %5                 \n"
+      "ld1     {v0.h}[3], [%0], %5                 \n"
+      "ld1     {v1.h}[3], [%0]                     \n"
+
+      "trn2    v2.8b, v0.8b, v1.8b                 \n"
+      "trn1    v3.8b, v0.8b, v1.8b                 \n"
+
+      "mov         %0, %2                          \n"
+
+      "st1     {v3.8b}, [%0], %6                   \n"
+      "st1     {v2.8b}, [%0]                       \n"
+
+      "add         %1, %1, #2                      \n"  // src += 2
+      "add         %2, %2, %6, lsl #1              \n"  // dst += 2 * dst_stride
+      "subs        %w3, %w3,  #2                   \n"  // w   -= 2
+      "b.eq        4f                              \n"
+
+      // 1x8 block
+      "3:                                          \n"
+      "ld1         {v0.b}[0], [%1], %5             \n"
+      "ld1         {v0.b}[1], [%1], %5             \n"
+      "ld1         {v0.b}[2], [%1], %5             \n"
+      "ld1         {v0.b}[3], [%1], %5             \n"
+      "ld1         {v0.b}[4], [%1], %5             \n"
+      "ld1         {v0.b}[5], [%1], %5             \n"
+      "ld1         {v0.b}[6], [%1], %5             \n"
+      "ld1         {v0.b}[7], [%1]                 \n"
+
+      "st1         {v0.8b}, [%2]                   \n"
+
+      "4:                                          \n"
+
+      : "=&r"(src_temp),                          // %0
+        "+r"(src),                                // %1
+        "+r"(dst),                                // %2
+        "+r"(width)                               // %3
+      : "r"(&kVTbl4x4Transpose),                  // %4
+        "r"(static_cast<ptrdiff_t>(src_stride)),  // %5
+        "r"(static_cast<ptrdiff_t>(dst_stride))   // %6
+      : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
+        "v17", "v18", "v19", "v20", "v21", "v22", "v23");
 }
 
@@ -210 +209 @@
                          int width) {
   const uint8* src_temp;
-  asm volatile (
-    // loops are on blocks of 8. loop will stop when
-    // counter gets to or below 0. starting the counter
-    // at w-8 allow for this
-    "sub       %w4, %w4, #8                    \n"
-
-    // handle 8x8 blocks. this should be the majority of the plane
-    "1:                                        \n"
-    "mov       %0, %1                          \n"
-
-    "ld1       {v0.16b}, [%0], %5              \n"
-    "ld1       {v1.16b}, [%0], %5              \n"
-    "ld1       {v2.16b}, [%0], %5              \n"
-    "ld1       {v3.16b}, [%0], %5              \n"
-    "ld1       {v4.16b}, [%0], %5              \n"
-    "ld1       {v5.16b}, [%0], %5              \n"
-    "ld1       {v6.16b}, [%0], %5              \n"
-    "ld1       {v7.16b}, [%0]                  \n"
-
-    "trn1      v16.16b, v0.16b, v1.16b         \n"
-    "trn2      v17.16b, v0.16b, v1.16b         \n"
-    "trn1      v18.16b, v2.16b, v3.16b         \n"
-    "trn2      v19.16b, v2.16b, v3.16b         \n"
-    "trn1      v20.16b, v4.16b, v5.16b         \n"
-    "trn2      v21.16b, v4.16b, v5.16b         \n"
-    "trn1      v22.16b, v6.16b, v7.16b         \n"
-    "trn2      v23.16b, v6.16b, v7.16b         \n"
-
-    "trn1      v0.8h, v16.8h, v18.8h           \n"
-    "trn2      v1.8h, v16.8h, v18.8h           \n"
-    "trn1      v2.8h, v20.8h, v22.8h           \n"
-    "trn2      v3.8h, v20.8h, v22.8h           \n"
-    "trn1      v4.8h, v17.8h, v19.8h           \n"
-    "trn2      v5.8h, v17.8h, v19.8h           \n"
-    "trn1      v6.8h, v21.8h, v23.8h           \n"
-    "trn2      v7.8h, v21.8h, v23.8h           \n"
-
-    "trn1      v16.4s, v0.4s, v2.4s            \n"
-    "trn2      v17.4s, v0.4s, v2.4s            \n"
-    "trn1      v18.4s, v1.4s, v3.4s            \n"
-    "trn2      v19.4s, v1.4s, v3.4s            \n"
-    "trn1      v20.4s, v4.4s, v6.4s            \n"
-    "trn2      v21.4s, v4.4s, v6.4s            \n"
-    "trn1      v22.4s, v5.4s, v7.4s            \n"
-    "trn2      v23.4s, v5.4s, v7.4s            \n"
-
-    "mov       %0, %2                          \n"
-
-    "st1       {v16.d}[0], [%0], %6            \n"
-    "st1       {v18.d}[0], [%0], %6            \n"
-    "st1       {v17.d}[0], [%0], %6            \n"
-    "st1       {v19.d}[0], [%0], %6            \n"
-    "st1       {v16.d}[1], [%0], %6            \n"
-    "st1       {v18.d}[1], [%0], %6            \n"
-    "st1       {v17.d}[1], [%0], %6            \n"
-    "st1       {v19.d}[1], [%0]                \n"
-
-    "mov       %0, %3                          \n"
-
-    "st1       {v20.d}[0], [%0], %7            \n"
-    "st1       {v22.d}[0], [%0], %7            \n"
-    "st1       {v21.d}[0], [%0], %7            \n"
-    "st1       {v23.d}[0], [%0], %7            \n"
-    "st1       {v20.d}[1], [%0], %7            \n"
-    "st1       {v22.d}[1], [%0], %7            \n"
-    "st1       {v21.d}[1], [%0], %7            \n"
-    "st1       {v23.d}[1], [%0]                \n"
-
-    "add       %1, %1, #16                     \n"  // src   += 8*2
-    "add       %2, %2, %6, lsl #3              \n"  // dst_a += 8 * dst_stride_a
-    "add       %3, %3, %7, lsl #3              \n"  // dst_b += 8 * dst_stride_b
-    "subs      %w4, %w4,  #8                   \n"  // w     -= 8
-    "b.ge      1b                              \n"
-
-    // add 8 back to counter. if the result is 0 there are
-    // no residuals.
-    "adds      %w4, %w4, #8                    \n"
-    "b.eq      4f                              \n"
-
-    // some residual, so between 1 and 7 lines left to transpose
-    "cmp       %w4, #2                         \n"
-    "b.lt      3f                              \n"
-
-    "cmp       %w4, #4                         \n"
-    "b.lt      2f                              \n"
-
-    // TODO(frkoenig): Clean this up
-    // 4x8 block
-    "mov       %0, %1                          \n"
-    "ld1       {v0.8b}, [%0], %5               \n"
-    "ld1       {v1.8b}, [%0], %5               \n"
-    "ld1       {v2.8b}, [%0], %5               \n"
-    "ld1       {v3.8b}, [%0], %5               \n"
-    "ld1       {v4.8b}, [%0], %5               \n"
-    "ld1       {v5.8b}, [%0], %5               \n"
-    "ld1       {v6.8b}, [%0], %5               \n"
-    "ld1       {v7.8b}, [%0]                   \n"
-
-    "ld1       {v30.16b}, [%8], #16            \n"
-    "ld1       {v31.16b}, [%8]                 \n"
-
-    "tbl       v16.16b, {v0.16b, v1.16b, v2.16b, v3.16b}, v30.16b  \n"
-    "tbl       v17.16b, {v0.16b, v1.16b, v2.16b, v3.16b}, v31.16b  \n"
-    "tbl       v18.16b, {v4.16b, v5.16b, v6.16b, v7.16b}, v30.16b  \n"
-    "tbl       v19.16b, {v4.16b, v5.16b, v6.16b, v7.16b}, v31.16b  \n"
-
-    "mov       %0, %2                          \n"
-
-    "st1       {v16.s}[0],  [%0], %6           \n"
-    "st1       {v16.s}[1],  [%0], %6           \n"
-    "st1       {v16.s}[2],  [%0], %6           \n"
-    "st1       {v16.s}[3],  [%0], %6           \n"
-
-    "add       %0, %2, #4                      \n"
-    "st1       {v18.s}[0], [%0], %6            \n"
-    "st1       {v18.s}[1], [%0], %6            \n"
-    "st1       {v18.s}[2], [%0], %6            \n"
-    "st1       {v18.s}[3], [%0]                \n"
-
-    "mov       %0, %3                          \n"
-
-    "st1       {v17.s}[0], [%0], %7            \n"
-    "st1       {v17.s}[1], [%0], %7            \n"
-    "st1       {v17.s}[2], [%0], %7            \n"
-    "st1       {v17.s}[3], [%0], %7            \n"
-
-    "add       %0, %3, #4                      \n"
-    "st1       {v19.s}[0],  [%0], %7           \n"
-    "st1       {v19.s}[1],  [%0], %7           \n"
-    "st1       {v19.s}[2],  [%0], %7           \n"
-    "st1       {v19.s}[3],  [%0]               \n"
-
-    "add       %1, %1, #8                      \n"  // src   += 4 * 2
-    "add       %2, %2, %6, lsl #2              \n"  // dst_a += 4 * dst_stride_a
-    "add       %3, %3, %7, lsl #2              \n"  // dst_b += 4 * dst_stride_b
-    "subs      %w4,  %w4,  #4                  \n"  // w     -= 4
-    "b.eq      4f                              \n"
-
-    // some residual, check to see if it includes a 2x8 block,
-    // or less
-    "cmp       %w4, #2                         \n"
-    "b.lt      3f                              \n"
-
-    // 2x8 block
-    "2:                                        \n"
-    "mov       %0, %1                          \n"
-    "ld2       {v0.h, v1.h}[0], [%0], %5       \n"
-    "ld2       {v2.h, v3.h}[0], [%0], %5       \n"
-    "ld2       {v0.h, v1.h}[1], [%0], %5       \n"
-    "ld2       {v2.h, v3.h}[1], [%0], %5       \n"
-    "ld2       {v0.h, v1.h}[2], [%0], %5       \n"
-    "ld2       {v2.h, v3.h}[2], [%0], %5       \n"
-    "ld2       {v0.h, v1.h}[3], [%0], %5       \n"
-    "ld2       {v2.h, v3.h}[3], [%0]           \n"
-
-    "trn1      v4.8b, v0.8b, v2.8b             \n"
-    "trn2      v5.8b, v0.8b, v2.8b             \n"
-    "trn1      v6.8b, v1.8b, v3.8b             \n"
-    "trn2      v7.8b, v1.8b, v3.8b             \n"
-
-    "mov       %0, %2                          \n"
-
-    "st1       {v4.d}[0], [%0], %6             \n"
-    "st1       {v6.d}[0], [%0]                 \n"
-
-    "mov       %0, %3                          \n"
-
-    "st1       {v5.d}[0], [%0], %7             \n"
-    "st1       {v7.d}[0], [%0]                 \n"
-
-    "add       %1, %1, #4                      \n"  // src   += 2 * 2
-    "add       %2, %2, %6, lsl #1              \n"  // dst_a += 2 * dst_stride_a
-    "add       %3, %3, %7, lsl #1              \n"  // dst_b += 2 * dst_stride_b
-    "subs      %w4,  %w4,  #2                  \n"  // w     -= 2
-    "b.eq      4f                              \n"
-
-    // 1x8 block
-    "3:                                        \n"
-    "ld2       {v0.b, v1.b}[0], [%1], %5       \n"
-    "ld2       {v0.b, v1.b}[1], [%1], %5       \n"
-    "ld2       {v0.b, v1.b}[2], [%1], %5       \n"
-    "ld2       {v0.b, v1.b}[3], [%1], %5       \n"
-    "ld2       {v0.b, v1.b}[4], [%1], %5       \n"
-    "ld2       {v0.b, v1.b}[5], [%1], %5       \n"
-    "ld2       {v0.b, v1.b}[6], [%1], %5       \n"
-    "ld2       {v0.b, v1.b}[7], [%1]           \n"
-
-    "st1       {v0.d}[0], [%2]                 \n"
-    "st1       {v1.d}[0], [%3]                 \n"
-
-    "4:                                        \n"
-
-    : "=&r"(src_temp),                            // %0
-      "+r"(src),                                  // %1
-      "+r"(dst_a),                                // %2
-      "+r"(dst_b),                                // %3
-      "+r"(width)                                 // %4
-    : "r"(static_cast<ptrdiff_t>(src_stride)),    // %5
-      "r"(static_cast<ptrdiff_t>(dst_stride_a)),  // %6
-      "r"(static_cast<ptrdiff_t>(dst_stride_b)),  // %7
-      "r"(&kVTbl4x4TransposeDi)                   // %8
-    : "memory", "cc",
-      "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
-      "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
-      "v30", "v31"
-  );
+  asm volatile(
+      // loops are on blocks of 8. loop will stop when
+      // counter gets to or below 0. starting the counter
+      // at w-8 allow for this
+      "sub       %w4, %w4, #8                    \n"
+
+      // handle 8x8 blocks. this should be the majority of the plane
+      "1:                                        \n"
+      "mov       %0, %1                          \n"
+
+      "ld1       {v0.16b}, [%0], %5              \n"
+      "ld1       {v1.16b}, [%0], %5              \n"
+      "ld1       {v2.16b}, [%0], %5              \n"
+      "ld1       {v3.16b}, [%0], %5              \n"
+      "ld1       {v4.16b}, [%0], %5              \n"
+      "ld1       {v5.16b}, [%0], %5              \n"
+      "ld1       {v6.16b}, [%0], %5              \n"
+      "ld1       {v7.16b}, [%0]                  \n"
+
+      "trn1      v16.16b, v0.16b, v1.16b         \n"
+      "trn2      v17.16b, v0.16b, v1.16b         \n"
+      "trn1      v18.16b, v2.16b, v3.16b         \n"
+      "trn2      v19.16b, v2.16b, v3.16b         \n"
+      "trn1      v20.16b, v4.16b, v5.16b         \n"
+      "trn2      v21.16b, v4.16b, v5.16b         \n"
+      "trn1      v22.16b, v6.16b, v7.16b         \n"
+      "trn2      v23.16b, v6.16b, v7.16b         \n"
+
+      "trn1      v0.8h, v16.8h, v18.8h           \n"
+      "trn2      v1.8h, v16.8h, v18.8h           \n"
+      "trn1      v2.8h, v20.8h, v22.8h           \n"
+      "trn2      v3.8h, v20.8h, v22.8h           \n"
+      "trn1      v4.8h, v17.8h, v19.8h           \n"
+      "trn2      v5.8h, v17.8h, v19.8h           \n"
+      "trn1      v6.8h, v21.8h, v23.8h           \n"
+      "trn2      v7.8h, v21.8h, v23.8h           \n"
+
+      "trn1      v16.4s, v0.4s, v2.4s            \n"
+      "trn2      v17.4s, v0.4s, v2.4s            \n"
+      "trn1      v18.4s, v1.4s, v3.4s            \n"
+      "trn2      v19.4s, v1.4s, v3.4s            \n"
+      "trn1      v20.4s, v4.4s, v6.4s            \n"
+      "trn2      v21.4s, v4.4s, v6.4s            \n"
+      "trn1      v22.4s, v5.4s, v7.4s            \n"
+      "trn2      v23.4s, v5.4s, v7.4s            \n"
+
+      "mov       %0, %2                          \n"
+
+      "st1       {v16.d}[0], [%0], %6            \n"
+      "st1       {v18.d}[0], [%0], %6            \n"
+      "st1       {v17.d}[0], [%0], %6            \n"
+      "st1       {v19.d}[0], [%0], %6            \n"
+      "st1       {v16.d}[1], [%0], %6            \n"
+      "st1       {v18.d}[1], [%0], %6            \n"
+      "st1       {v17.d}[1], [%0], %6            \n"
+      "st1       {v19.d}[1], [%0]                \n"
+
+      "mov       %0, %3                          \n"
+
+      "st1       {v20.d}[0], [%0], %7            \n"
+      "st1       {v22.d}[0], [%0], %7            \n"
+      "st1       {v21.d}[0], [%0], %7            \n"
+      "st1       {v23.d}[0], [%0], %7            \n"
+      "st1       {v20.d}[1], [%0], %7            \n"
+      "st1       {v22.d}[1], [%0], %7            \n"
+      "st1       {v21.d}[1], [%0], %7            \n"
+      "st1       {v23.d}[1], [%0]                \n"
+
+      "add       %1, %1, #16                     \n"  // src   += 8*2
+      "add       %2, %2, %6, lsl #3              \n"  // dst_a += 8 *
+                                                      // dst_stride_a
+      "add       %3, %3, %7, lsl #3              \n"  // dst_b += 8 *
+                                                      // dst_stride_b
+      "subs      %w4, %w4,  #8                   \n"  // w     -= 8
+      "b.ge      1b                              \n"
+
+      // add 8 back to counter. if the result is 0 there are
+      // no residuals.
+      "adds      %w4, %w4, #8                    \n"
+      "b.eq      4f                              \n"
+
+      // some residual, so between 1 and 7 lines left to transpose
+      "cmp       %w4, #2                         \n"
+      "b.lt      3f                              \n"
+
+      "cmp       %w4, #4                         \n"
+      "b.lt      2f                              \n"
+
+      // TODO(frkoenig): Clean this up
+      // 4x8 block
+      "mov       %0, %1                          \n"
+      "ld1       {v0.8b}, [%0], %5               \n"
+      "ld1       {v1.8b}, [%0], %5               \n"
+      "ld1       {v2.8b}, [%0], %5               \n"
+      "ld1       {v3.8b}, [%0], %5               \n"
+      "ld1       {v4.8b}, [%0], %5               \n"
+      "ld1       {v5.8b}, [%0], %5               \n"
+      "ld1       {v6.8b}, [%0], %5               \n"
+      "ld1       {v7.8b}, [%0]                   \n"
+
+      "ld1       {v30.16b}, [%8], #16            \n"
+      "ld1       {v31.16b}, [%8]                 \n"
+
+      "tbl       v16.16b, {v0.16b, v1.16b, v2.16b, v3.16b}, v30.16b  \n"
+      "tbl       v17.16b, {v0.16b, v1.16b, v2.16b, v3.16b}, v31.16b  \n"
+      "tbl       v18.16b, {v4.16b, v5.16b, v6.16b, v7.16b}, v30.16b  \n"
+      "tbl       v19.16b, {v4.16b, v5.16b, v6.16b, v7.16b}, v31.16b  \n"
+
+      "mov       %0, %2                          \n"
+
+      "st1       {v16.s}[0],  [%0], %6           \n"
+      "st1       {v16.s}[1],  [%0], %6           \n"
+      "st1       {v16.s}[2],  [%0], %6           \n"
+      "st1       {v16.s}[3],  [%0], %6           \n"
+
+      "add       %0, %2, #4                      \n"
+      "st1       {v18.s}[0], [%0], %6            \n"
+      "st1       {v18.s}[1], [%0], %6            \n"
+      "st1       {v18.s}[2], [%0], %6            \n"
+      "st1       {v18.s}[3], [%0]                \n"
+
+      "mov       %0, %3                          \n"
+
+      "st1       {v17.s}[0], [%0], %7            \n"
+      "st1       {v17.s}[1], [%0], %7            \n"
+      "st1       {v17.s}[2], [%0], %7            \n"
+      "st1       {v17.s}[3], [%0], %7            \n"
+
+      "add       %0, %3, #4                      \n"
+      "st1       {v19.s}[0],  [%0], %7           \n"
+      "st1       {v19.s}[1],  [%0], %7           \n"
+      "st1       {v19.s}[2],  [%0], %7           \n"
+      "st1       {v19.s}[3],  [%0]               \n"
+
+      "add       %1, %1, #8                      \n"  // src   += 4 * 2
+      "add       %2, %2, %6, lsl #2              \n"  // dst_a += 4 *
+                                                      // dst_stride_a
+      "add       %3, %3, %7, lsl #2              \n"  // dst_b += 4 *
+                                                      // dst_stride_b
+      "subs      %w4,  %w4,  #4                  \n"  // w     -= 4
+      "b.eq      4f                              \n"
+
+      // some residual, check to see if it includes a 2x8 block,
+      // or less
+      "cmp       %w4, #2                         \n"
+      "b.lt      3f                              \n"
+
+      // 2x8 block
+      "2:                                        \n"
+      "mov       %0, %1                          \n"
+      "ld2       {v0.h, v1.h}[0], [%0], %5       \n"
+      "ld2       {v2.h, v3.h}[0], [%0], %5       \n"
+      "ld2       {v0.h, v1.h}[1], [%0], %5       \n"
+      "ld2       {v2.h, v3.h}[1], [%0], %5       \n"
+      "ld2       {v0.h, v1.h}[2], [%0], %5       \n"
+      "ld2       {v2.h, v3.h}[2], [%0], %5       \n"
+      "ld2       {v0.h, v1.h}[3], [%0], %5       \n"
+      "ld2       {v2.h, v3.h}[3], [%0]           \n"
+
+      "trn1      v4.8b, v0.8b, v2.8b             \n"
+      "trn2      v5.8b, v0.8b, v2.8b             \n"
+      "trn1      v6.8b, v1.8b, v3.8b             \n"
+      "trn2      v7.8b, v1.8b, v3.8b             \n"
+
+      "mov       %0, %2                          \n"
+
+      "st1       {v4.d}[0], [%0], %6             \n"
+      "st1       {v6.d}[0], [%0]                 \n"
+
+      "mov       %0, %3                          \n"
+
+      "st1       {v5.d}[0], [%0], %7             \n"
+      "st1       {v7.d}[0], [%0]                 \n"
+
+      "add       %1, %1, #4                      \n"  // src   += 2 * 2
+      "add       %2, %2, %6, lsl #1              \n"  // dst_a += 2 *
+                                                      // dst_stride_a
+      "add       %3, %3, %7, lsl #1              \n"  // dst_b += 2 *
+                                                      // dst_stride_b
+      "subs      %w4,  %w4,  #2                  \n"  // w     -= 2
+      "b.eq      4f                              \n"
+
+      // 1x8 block
+      "3:                                        \n"
+      "ld2       {v0.b, v1.b}[0], [%1], %5       \n"
+      "ld2       {v0.b, v1.b}[1], [%1], %5       \n"
+      "ld2       {v0.b, v1.b}[2], [%1], %5       \n"
+      "ld2       {v0.b, v1.b}[3], [%1], %5       \n"
+      "ld2       {v0.b, v1.b}[4], [%1], %5       \n"
+      "ld2       {v0.b, v1.b}[5], [%1], %5       \n"
+      "ld2       {v0.b, v1.b}[6], [%1], %5       \n"
+      "ld2       {v0.b, v1.b}[7], [%1]           \n"
+
+      "st1       {v0.d}[0], [%2]                 \n"
+      "st1       {v1.d}[0], [%3]                 \n"
+
+      "4:                                        \n"
+
+      : "=&r"(src_temp),                            // %0
+        "+r"(src),                                  // %1
+        "+r"(dst_a),                                // %2
+        "+r"(dst_b),                                // %3
+        "+r"(width)                                 // %4
+      : "r"(static_cast<ptrdiff_t>(src_stride)),    // %5
+        "r"(static_cast<ptrdiff_t>(dst_stride_a)),  // %6
+        "r"(static_cast<ptrdiff_t>(dst_stride_b)),  // %7
+        "r"(&kVTbl4x4TransposeDi)                   // %8
+      : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
+        "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v30", "v31");
 }
 #endif  // !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)

pjproject/trunk/third_party/yuv/source/rotate_win.cc

@@ -18 +18 @@
 
 // This module is for 32 bit Visual C x86 and clangcl
-#if !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86)
+#if !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86) && defined(_MSC_VER)
 
 __declspec(naked) void TransposeWx8_SSSE3(const uint8* src,
@@ -173 +173 @@
     lea       eax, [eax + 8 * edi + 16]
     neg       edi
-    // Second round of bit swap.
+        // Second round of bit swap.
     movdqa    xmm5, xmm0
     punpcklwd xmm0, xmm2
@@ -193 +193 @@
     movdqa    xmm7, xmm6
 
-    // Third round of bit swap.
-    // Write to the destination pointer.
+        // Third round of bit swap.
+        // Write to the destination pointer.
     movdqa    xmm6, xmm0
     punpckldq xmm0, xmm4

pjproject/trunk/third_party/yuv/source/row_any.cc

@@ -85 +85 @@
            SS(r, DUVSHIFT) * BPP);                                         \
   }
+
+// Merge functions.
+#ifdef HAS_MERGERGBROW_SSSE3
+ANY31(MergeRGBRow_Any_SSSE3, MergeRGBRow_SSSE3, 0, 0, 3, 15)
+#endif
+#ifdef HAS_MERGERGBROW_NEON
+ANY31(MergeRGBRow_Any_NEON, MergeRGBRow_NEON, 0, 0, 3, 15)
+#endif
 #ifdef HAS_I422TOYUY2ROW_SSE2
 ANY31(I422ToYUY2Row_Any_SSE2, I422ToYUY2Row_SSE2, 1, 1, 4, 15)
@@ -622 +630 @@
 ANY11(ARGBExtractAlphaRow_Any_NEON, ARGBExtractAlphaRow_NEON, 0, 4, 1, 15)
 #endif
+#ifdef HAS_ARGBEXTRACTALPHAROW_MSA
+ANY11(ARGBExtractAlphaRow_Any_MSA, ARGBExtractAlphaRow_MSA, 0, 4, 1, 15)
+#endif
 #undef ANY11
 
@@ -746 +757 @@
 ANY11P16(HalfFloatRow_Any_NEON, HalfFloatRow_NEON, float, 2, 2, 7)
 ANY11P16(HalfFloat1Row_Any_NEON, HalfFloat1Row_NEON, float, 2, 2, 7)
+#endif
+#ifdef HAS_HALFFLOATROW_MSA
+ANY11P16(HalfFloatRow_Any_MSA, HalfFloatRow_MSA, float, 2, 2, 31)
 #endif
 #undef ANY11P16
@@ -912 +926 @@
 ANY12(SplitUVRow_Any_DSPR2, SplitUVRow_DSPR2, 0, 2, 0, 15)
 #endif
+#ifdef HAS_SPLITUVROW_MSA
+ANY12(SplitUVRow_Any_MSA, SplitUVRow_MSA, 0, 2, 0, 31)
+#endif
 #ifdef HAS_ARGBTOUV444ROW_SSSE3
 ANY12(ARGBToUV444Row_Any_SSSE3, ARGBToUV444Row_SSSE3, 0, 4, 0, 15)
@@ -934 +951 @@
 #endif
 #undef ANY12
+
+// Any 1 to 3.  Outputs RGB planes.
+#define ANY13(NAMEANY, ANY_SIMD, BPP, MASK)                                    \
+  void NAMEANY(const uint8* src_ptr, uint8* dst_r, uint8* dst_g, uint8* dst_b, \
+               int width) {                                                    \
+    SIMD_ALIGNED(uint8 temp[16 * 6]);                                          \
+    memset(temp, 0, 16 * 3); /* for msan */                                    \
+    int r = width & MASK;                                                      \
+    int n = width & ~MASK;                                                     \
+    if (n > 0) {                                                               \
+      ANY_SIMD(src_ptr, dst_r, dst_g, dst_b, n);                               \
+    }                                                                          \
+    memcpy(temp, src_ptr + n * BPP, r * BPP);                                  \
+    ANY_SIMD(temp, temp + 16 * 3, temp + 16 * 4, temp + 16 * 5, MASK + 1);     \
+    memcpy(dst_r + n, temp + 16 * 3, r);                                       \
+    memcpy(dst_g + n, temp + 16 * 4, r);                                       \
+    memcpy(dst_b + n, temp + 16 * 5, r);                                       \
+  }
+
+#ifdef HAS_SPLITRGBROW_SSSE3
+ANY13(SplitRGBRow_Any_SSSE3, SplitRGBRow_SSSE3, 3, 15)
+#endif
+#ifdef HAS_SPLITRGBROW_NEON
+ANY13(SplitRGBRow_Any_NEON, SplitRGBRow_NEON, 3, 15)
+#endif
 
 // Any 1 to 2 with source stride (2 rows of source).  Outputs UV planes.

pjproject/trunk/third_party/yuv/source/row_common.cc

@@ -1771 +1771 @@
 }
 
+void SplitRGBRow_C(const uint8* src_rgb,
+                   uint8* dst_r,
+                   uint8* dst_g,
+                   uint8* dst_b,
+                   int width) {
+  int x;
+  for (x = 0; x < width; ++x) {
+    dst_r[x] = src_rgb[0];
+    dst_g[x] = src_rgb[1];
+    dst_b[x] = src_rgb[2];
+    src_rgb += 3;
+  }
+}
+
+void MergeRGBRow_C(const uint8* src_r,
+                   const uint8* src_g,
+                   const uint8* src_b,
+                   uint8* dst_rgb,
+                   int width) {
+  int x;
+  for (x = 0; x < width; ++x) {
+    dst_rgb[0] = src_r[x];
+    dst_rgb[1] = src_g[x];
+    dst_rgb[2] = src_b[x];
+    dst_rgb += 3;
+  }
+}
+
+void MergeUVRow_16_C(const uint16* src_u,
+                     const uint16* src_v,
+                     uint16* dst_uv,
+                     int scale,
+                     int width) {
+  int x;
+  for (x = 0; x < width - 1; x += 2) {
+    dst_uv[0] = src_u[x] * scale;
+    dst_uv[1] = src_v[x] * scale;
+    dst_uv[2] = src_u[x + 1] * scale;
+    dst_uv[3] = src_v[x + 1] * scale;
+    dst_uv += 4;
+  }
+  if (width & 1) {
+    dst_uv[0] = src_u[width - 1] * scale;
+    dst_uv[1] = src_v[width - 1] * scale;
+  }
+}
+
+void MultiplyRow_16_C(const uint16* src_y,
+                      uint16* dst_y,
+                      int scale,
+                      int width) {
+  int x;
+  for (x = 0; x < width; ++x) {
+    dst_y[x] = src_y[x] * scale;
+  }
+}
+
 void CopyRow_C(const uint8* src, uint8* dst, int count) {
   memcpy(dst, src, count);
@@ -2640 +2697 @@
 #endif
 
+float ScaleSumSamples_C(const float* src, float* dst, float scale, int width) {
+  float fsum = 0.f;
+  int i;
+#if defined(__clang__)
+#pragma clang loop vectorize_width(4)
+#endif
+  for (i = 0; i < width; ++i) {
+    float v = *src++;
+    fsum += v * v;
+    *dst++ = v * scale;
+  }
+  return fsum;
+}
+
+float ScaleMaxSamples_C(const float* src, float* dst, float scale, int width) {
+  float fmax = 0.f;
+  int i;
+  for (i = 0; i < width; ++i) {
+    float v = *src++;
+    float vs = v * scale;
+    fmax = (v > fmax) ? v : fmax;
+    *dst++ = vs;
+  }
+  return fmax;
+}
+
+void ScaleSamples_C(const float* src, float* dst, float scale, int width) {
+  int i;
+  for (i = 0; i < width; ++i) {
+    *dst++ = *src++ * scale;
+  }
+}
+
+void GaussRow_C(const uint32* src, uint16* dst, int width) {
+  int i;
+  for (i = 0; i < width; ++i) {
+    *dst++ =
+        (src[0] + src[1] * 4 + src[2] * 6 + src[3] * 4 + src[4] + 128) >> 8;
+    ++src;
+  }
+}
+
+// filter 5 rows with 1, 4, 6, 4, 1 coefficients to produce 1 row.
+void GaussCol_C(const uint16* src0,
+                const uint16* src1,
+                const uint16* src2,
+                const uint16* src3,
+                const uint16* src4,
+                uint32* dst,
+                int width) {
+  int i;
+  for (i = 0; i < width; ++i) {
+    *dst++ = *src0++ + *src1++ * 4 + *src2++ * 6 + *src3++ * 4 + *src4++;
+  }
+}
+
 #ifdef __cplusplus
 }  // extern "C"

pjproject/trunk/third_party/yuv/source/row_gcc.cc

@@ -39 +39 @@
                          127, -84, -43, 0, 127, -84, -43, 0};
 
-static vec8 kARGBToV = {
-    -18, -94, 112, 0, -18, -94, 112, 0, -18, -94, 112, 0, -18, -94, 112, 0,
-};
+static vec8 kARGBToV = {-18, -94, 112, 0, -18, -94, 112, 0,
+                        -18, -94, 112, 0, -18, -94, 112, 0};
 
 static vec8 kARGBToVJ = {-20, -107, 127, 0, -20, -107, 127, 0,
@@ -2754 +2753 @@
 }
 #endif  // HAS_MERGEUVROW_SSE2
+
+// Use scale to convert lsb formats to msb, depending how many bits there are:
+// 128 = 9 bits
+// 64 = 10 bits
+// 16 = 12 bits
+// 1 = 16 bits
+#ifdef HAS_MERGEUVROW_16_AVX2
+void MergeUVRow_16_AVX2(const uint16* src_u,
+                        const uint16* src_v,
+                        uint16* dst_uv,
+                        int scale,
+                        int width) {
+  // clang-format off
+  asm volatile (
+    "vmovd      %4,%%xmm3                      \n"
+    "vpunpcklwd %%xmm3,%%xmm3,%%xmm3           \n"
+    "vbroadcastss %%xmm3,%%ymm3                \n"
+    "sub       %0,%1                           \n"
+
+    // 16 pixels per loop.
+    LABELALIGN
+    "1:                                        \n"
+    "vmovdqu   (%0),%%ymm0                     \n"
+    "vmovdqu   (%0,%1,1),%%ymm1                \n"
+    "add        $0x20,%0                       \n"
+
+    "vpmullw   %%ymm3,%%ymm0,%%ymm0            \n"
+    "vpmullw   %%ymm3,%%ymm1,%%ymm1            \n"
+    "vpunpcklwd %%ymm1,%%ymm0,%%ymm2           \n"  // mutates
+    "vpunpckhwd %%ymm1,%%ymm0,%%ymm0           \n"
+    "vextractf128 $0x0,%%ymm2,(%2)             \n"
+    "vextractf128 $0x0,%%ymm0,0x10(%2)         \n"
+    "vextractf128 $0x1,%%ymm2,0x20(%2)         \n"
+    "vextractf128 $0x1,%%ymm0,0x30(%2)         \n"
+    "add       $0x40,%2                        \n"
+    "sub       $0x10,%3                        \n"
+    "jg        1b                              \n"
+    "vzeroupper                                \n"
+  : "+r"(src_u),   // %0
+    "+r"(src_v),   // %1
+    "+r"(dst_uv),  // %2
+    "+r"(width)    // %3
+  : "r"(scale)     // %4
+  : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3");
+  // clang-format on
+}
+#endif  // HAS_MERGEUVROW_AVX2
+
+#ifdef HAS_MULTIPLYROW_16_AVX2
+void MultiplyRow_16_AVX2(const uint16* src_y,
+                         uint16* dst_y,
+                         int scale,
+                         int width) {
+  // clang-format off
+  asm volatile (
+    "vmovd      %3,%%xmm3                      \n"
+    "vpunpcklwd %%xmm3,%%xmm3,%%xmm3           \n"
+    "vbroadcastss %%xmm3,%%ymm3                \n"
+    "sub       %0,%1                           \n"
+
+    // 16 pixels per loop.
+    LABELALIGN
+    "1:                                        \n"
+    "vmovdqu   (%0),%%ymm0                     \n"
+    "vmovdqu   0x20(%0),%%ymm1                 \n"
+    "vpmullw   %%ymm3,%%ymm0,%%ymm0            \n"
+    "vpmullw   %%ymm3,%%ymm1,%%ymm1            \n"
+    "vmovdqu   %%ymm0,(%0,%1)                  \n"
+    "vmovdqu   %%ymm1,0x20(%0,%1)              \n"
+    "add        $0x40,%0                       \n"
+    "sub       $0x20,%2                        \n"
+    "jg        1b                              \n"
+    "vzeroupper                                \n"
+  : "+r"(src_y),   // %0
+    "+r"(dst_y),   // %1
+    "+r"(width)    // %2
+  : "r"(scale)     // %3
+  : "memory", "cc", "xmm0", "xmm1", "xmm3");
+  // clang-format on
+}
+#endif  // HAS_MULTIPLYROW_16_AVX2
+
+#ifdef HAS_SPLITRGBROW_SSSE3
+
+// Shuffle table for converting RGB to Planar.
+static uvec8 kShuffleMaskRGBToR0 = {0u,   3u,   6u,   9u,   12u,  15u,
+                                    128u, 128u, 128u, 128u, 128u, 128u,
+                                    128u, 128u, 128u, 128u};
+static uvec8 kShuffleMaskRGBToR1 = {128u, 128u, 128u, 128u, 128u, 128u,
+                                    2u,   5u,   8u,   11u,  14u,  128u,
+                                    128u, 128u, 128u, 128u};
+static uvec8 kShuffleMaskRGBToR2 = {128u, 128u, 128u, 128u, 128u, 128u,
+                                    128u, 128u, 128u, 128u, 128u, 1u,
+                                    4u,   7u,   10u,  13u};
+
+static uvec8 kShuffleMaskRGBToG0 = {1u,   4u,   7u,   10u,  13u,  128u,
+                                    128u, 128u, 128u, 128u, 128u, 128u,
+                                    128u, 128u, 128u, 128u};
+static uvec8 kShuffleMaskRGBToG1 = {128u, 128u, 128u, 128u, 128u, 0u,
+                                    3u,   6u,   9u,   12u,  15u,  128u,
+                                    128u, 128u, 128u, 128u};
+static uvec8 kShuffleMaskRGBToG2 = {128u, 128u, 128u, 128u, 128u, 128u,
+                                    128u, 128u, 128u, 128u, 128u, 2u,
+                                    5u,   8u,   11u,  14u};
+
+static uvec8 kShuffleMaskRGBToB0 = {2u,   5u,   8u,   11u,  14u,  128u,
+                                    128u, 128u, 128u, 128u, 128u, 128u,
+                                    128u, 128u, 128u, 128u};
+static uvec8 kShuffleMaskRGBToB1 = {128u, 128u, 128u, 128u, 128u, 1u,
+                                    4u,   7u,   10u,  13u,  128u, 128u,
+                                    128u, 128u, 128u, 128u};
+static uvec8 kShuffleMaskRGBToB2 = {128u, 128u, 128u, 128u, 128u, 128u,
+                                    128u, 128u, 128u, 128u, 0u,   3u,
+                                    6u,   9u,   12u,  15u};
+
+void SplitRGBRow_SSSE3(const uint8* src_rgb,
+                       uint8* dst_r,
+                       uint8* dst_g,
+                       uint8* dst_b,
+                       int width) {
+  asm volatile (
+    LABELALIGN
+    "1:                                        \n"
+    "movdqu     " MEMACCESS(0) ",%%xmm0        \n"
+    "movdqu     " MEMACCESS2(0x10,0) ",%%xmm1  \n"
+    "movdqu     " MEMACCESS2(0x20,0) ",%%xmm2  \n"
+    "pshufb     %5, %%xmm0                     \n"
+    "pshufb     %6, %%xmm1                     \n"
+    "pshufb     %7, %%xmm2                     \n"
+    "por        %%xmm1,%%xmm0                  \n"
+    "por        %%xmm2,%%xmm0                  \n"
+    "movdqu     %%xmm0," MEMACCESS(1) "        \n"
+    "lea        " MEMLEA(0x10,1) ",%1          \n"
+
+    "movdqu     " MEMACCESS(0) ",%%xmm0        \n"
+    "movdqu     " MEMACCESS2(0x10,0) ",%%xmm1  \n"
+    "movdqu     " MEMACCESS2(0x20,0) ",%%xmm2  \n"
+    "pshufb     %8, %%xmm0                     \n"
+    "pshufb     %9, %%xmm1                     \n"
+    "pshufb     %10, %%xmm2                    \n"
+    "por        %%xmm1,%%xmm0                  \n"
+    "por        %%xmm2,%%xmm0                  \n"
+    "movdqu     %%xmm0," MEMACCESS(2) "        \n"
+    "lea        " MEMLEA(0x10,2) ",%2          \n"
+
+    "movdqu     " MEMACCESS(0) ",%%xmm0        \n"
+    "movdqu     " MEMACCESS2(0x10,0) ",%%xmm1  \n"
+    "movdqu     " MEMACCESS2(0x20,0) ",%%xmm2  \n"
+    "pshufb     %11, %%xmm0                    \n"
+    "pshufb     %12, %%xmm1                    \n"
+    "pshufb     %13, %%xmm2                    \n"
+    "por        %%xmm1,%%xmm0                  \n"
+    "por        %%xmm2,%%xmm0                  \n"
+    "movdqu     %%xmm0," MEMACCESS(3) "        \n"
+    "lea        " MEMLEA(0x10,3) ",%3          \n"
+    "lea        " MEMLEA(0x30,0) ",%0          \n"
+    "sub        $0x10,%4                       \n"
+    "jg         1b                             \n"
+  : "+r"(src_rgb),              // %0
+    "+r"(dst_r),                // %1
+    "+r"(dst_g),                // %2
+    "+r"(dst_b),                // %3
+    "+r"(width)                 // %4
+  : "m"(kShuffleMaskRGBToR0),   // %5
+    "m"(kShuffleMaskRGBToR1),   // %6
+    "m"(kShuffleMaskRGBToR2),   // %7
+    "m"(kShuffleMaskRGBToG0),   // %8
+    "m"(kShuffleMaskRGBToG1),   // %9
+    "m"(kShuffleMaskRGBToG2),   // %10
+    "m"(kShuffleMaskRGBToB0),   // %11
+    "m"(kShuffleMaskRGBToB1),   // %12
+    "m"(kShuffleMaskRGBToB2)    // %13
+  : "memory", "cc", NACL_R14
+    "xmm0", "xmm1", "xmm2"
+  );
+}
+#endif  // HAS_SPLITRGBROW_SSSE3
+
+#ifdef HAS_MERGERGBROW_SSSE3
+
+// Shuffle table for converting RGB to Planar.
+static uvec8 kShuffleMaskRToRGB0 = {0u, 128u, 128u, 1u, 128u, 128u,
+                                    2u, 128u, 128u, 3u, 128u, 128u,
+                                    4u, 128u, 128u, 5u};
+static uvec8 kShuffleMaskGToRGB0 = {128u, 0u, 128u, 128u, 1u, 128u,
+                                    128u, 2u, 128u, 128u, 3u, 128u,
+                                    128u, 4u, 128u, 128u};
+static uvec8 kShuffleMaskBToRGB0 = {128u, 128u, 0u, 128u, 128u, 1u,
+                                    128u, 128u, 2u, 128u, 128u, 3u,
+                                    128u, 128u, 4u, 128u};
+
+static uvec8 kShuffleMaskGToRGB1 = {5u, 128u, 128u, 6u, 128u, 128u,
+                                    7u, 128u, 128u, 8u, 128u, 128u,
+                                    9u, 128u, 128u, 10u};
+static uvec8 kShuffleMaskBToRGB1 = {128u, 5u, 128u, 128u, 6u, 128u,
+                                    128u, 7u, 128u, 128u, 8u, 128u,
+                                    128u, 9u, 128u, 128u};
+static uvec8 kShuffleMaskRToRGB1 = {128u, 128u, 6u,  128u, 128u, 7u,
+                                    128u, 128u, 8u,  128u, 128u, 9u,
+                                    128u, 128u, 10u, 128u};
+
+static uvec8 kShuffleMaskBToRGB2 = {10u, 128u, 128u, 11u, 128u, 128u,
+                                    12u, 128u, 128u, 13u, 128u, 128u,
+                                    14u, 128u, 128u, 15u};
+static uvec8 kShuffleMaskRToRGB2 = {128u, 11u, 128u, 128u, 12u, 128u,
+                                    128u, 13u, 128u, 128u, 14u, 128u,
+                                    128u, 15u, 128u, 128u};
+static uvec8 kShuffleMaskGToRGB2 = {128u, 128u, 11u, 128u, 128u, 12u,
+                                    128u, 128u, 13u, 128u, 128u, 14u,
+                                    128u, 128u, 15u, 128u};
+
+void MergeRGBRow_SSSE3(const uint8* src_r,
+                       const uint8* src_g,
+                       const uint8* src_b,
+                       uint8* dst_rgb,
+                       int width) {
+  asm volatile (
+    LABELALIGN
+    "1:                                        \n"
+    "movdqu     " MEMACCESS(0) ",%%xmm0        \n"
+    "movdqu     " MEMACCESS(1) ",%%xmm1        \n"
+    "movdqu     " MEMACCESS(2) ",%%xmm2        \n"
+    "pshufb     %5, %%xmm0                     \n"
+    "pshufb     %6, %%xmm1                     \n"
+    "pshufb     %7, %%xmm2                     \n"
+    "por        %%xmm1,%%xmm0                  \n"
+    "por        %%xmm2,%%xmm0                  \n"
+    "movdqu     %%xmm0," MEMACCESS(3) "        \n"
+
+    "movdqu     " MEMACCESS(0) ",%%xmm0        \n"
+    "movdqu     " MEMACCESS(1) ",%%xmm1        \n"
+    "movdqu     " MEMACCESS(2) ",%%xmm2        \n"
+    "pshufb     %8, %%xmm0                     \n"
+    "pshufb     %9, %%xmm1                     \n"
+    "pshufb     %10, %%xmm2                    \n"
+    "por        %%xmm1,%%xmm0                  \n"
+    "por        %%xmm2,%%xmm0                  \n"
+    "movdqu     %%xmm0," MEMACCESS2(16, 3) "   \n"
+
+    "movdqu     " MEMACCESS(0) ",%%xmm0        \n"
+    "movdqu     " MEMACCESS(1) ",%%xmm1        \n"
+    "movdqu     " MEMACCESS(2) ",%%xmm2        \n"
+    "pshufb     %11, %%xmm0                    \n"
+    "pshufb     %12, %%xmm1                    \n"
+    "pshufb     %13, %%xmm2                    \n"
+    "por        %%xmm1,%%xmm0                  \n"
+    "por        %%xmm2,%%xmm0                  \n"
+    "movdqu     %%xmm0," MEMACCESS2(32, 3) "   \n"
+
+    "lea        " MEMLEA(0x10,0) ",%0          \n"
+    "lea        " MEMLEA(0x10,1) ",%1          \n"
+    "lea        " MEMLEA(0x10,2) ",%2          \n"
+    "lea        " MEMLEA(0x30,3) ",%3          \n"
+    "sub        $0x10,%4                       \n"
+    "jg         1b                             \n"
+  : "+r"(src_r),                // %0
+    "+r"(src_g),                // %1
+    "+r"(src_b),                // %2
+    "+r"(dst_rgb),              // %3
+    "+r"(width)                 // %4
+  : "m"(kShuffleMaskRToRGB0),   // %5
+    "m"(kShuffleMaskGToRGB0),   // %6
+    "m"(kShuffleMaskBToRGB0),   // %7
+    "m"(kShuffleMaskRToRGB1),   // %8
+    "m"(kShuffleMaskGToRGB1),   // %9
+    "m"(kShuffleMaskBToRGB1),   // %10
+    "m"(kShuffleMaskRToRGB2),   // %11
+    "m"(kShuffleMaskGToRGB2),   // %12
+    "m"(kShuffleMaskBToRGB2)    // %13
+  : "memory", "cc", NACL_R14
+    "xmm0", "xmm1", "xmm2"
+  );
+}
+#endif  // HAS_MERGERGBROW_SSSE3
 
 #ifdef HAS_COPYROW_SSE2
@@ -5454 +5727 @@
 static float kScaleBias = 1.9259299444e-34f;
 void HalfFloatRow_SSE2(const uint16* src, uint16* dst, float scale, int width) {
+  scale *= kScaleBias;
   asm volatile (
     "pshufd      $0x0,%3,%%xmm4                \n"
@@ -5480 +5754 @@
     "+r"(dst),    // %1
     "+r"(width)   // %2
-  : "x"(scale * kScaleBias)   // %3
+#if defined(__x86_64__)
+  : "x"(scale)   // %3
+#else
+  : "m"(scale)   // %3
+#endif
   : "memory", "cc",
     "xmm2", "xmm3", "xmm4", "xmm5"
@@ -5489 +5767 @@
 #ifdef HAS_HALFFLOATROW_AVX2
 void HalfFloatRow_AVX2(const uint16* src, uint16* dst, float scale, int width) {
+  scale *= kScaleBias;
   asm volatile (
     "vbroadcastss  %3, %%ymm4                  \n"
@@ -5516 +5795 @@
     "+r"(dst),    // %1
     "+r"(width)   // %2
-  : "x"(scale * kScaleBias)   // %3
+#if defined(__x86_64__)
+  : "x"(scale)   // %3
+#else
+  : "m"(scale)   // %3
+#endif
   : "memory", "cc",
     "xmm2", "xmm3", "xmm4", "xmm5"
@@ -5549 +5832 @@
     "+r"(dst),   // %1
     "+r"(width)  // %2
+#if defined(__x86_64__)
   : "x"(scale)   // %3
+#else
+  : "m"(scale)   // %3
+#endif
   : "memory", "cc",
     "xmm2", "xmm3", "xmm4"

pjproject/trunk/third_party/yuv/source/row_msa.cc

@@ -2918 +2918 @@
 void ARGBSetRow_MSA(uint8* dst_argb, uint32 v32, int width) {
   int x;
-  v16u8 dst0 = (v16u8)__msa_fill_w(v32);
+  v4i32 dst0 = __builtin_msa_fill_w(v32);
 
   for (x = 0; x < width; x += 4) {
@@ -2970 +2970 @@
 }
 
+void ARGBExtractAlphaRow_MSA(const uint8* src_argb, uint8* dst_a, int width) {
+  int i;
+  v16u8 src0, src1, src2, src3, vec0, vec1, dst0;
+
+  for (i = 0; i < width; i += 16) {
+    src0 = (v16u8)__msa_ld_b((v16i8*)src_argb, 0);
+    src1 = (v16u8)__msa_ld_b((v16i8*)src_argb, 16);
+    src2 = (v16u8)__msa_ld_b((v16i8*)src_argb, 32);
+    src3 = (v16u8)__msa_ld_b((v16i8*)src_argb, 48);
+    vec0 = (v16u8)__msa_pckod_b((v16i8)src1, (v16i8)src0);
+    vec1 = (v16u8)__msa_pckod_b((v16i8)src3, (v16i8)src2);
+    dst0 = (v16u8)__msa_pckod_b((v16i8)vec1, (v16i8)vec0);
+    ST_UB(dst0, dst_a);
+    src_argb += 64;
+    dst_a += 16;
+  }
+}
+
+void ARGBBlendRow_MSA(const uint8* src_argb0,
+                      const uint8* src_argb1,
+                      uint8* dst_argb,
+                      int width) {
+  int x;
+  v16u8 src0, src1, src2, src3, dst0, dst1;
+  v8u16 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
+  v8u16 vec8, vec9, vec10, vec11, vec12, vec13;
+  v8u16 const_256 = (v8u16)__msa_ldi_h(256);
+  v16u8 const_255 = (v16u8)__msa_ldi_b(255);
+  v16u8 mask = {0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255};
+  v16i8 zero = {0};
+
+  for (x = 0; x < width; x += 8) {
+    src0 = (v16u8)__msa_ld_b((v16i8*)src_argb0, 0);
+    src1 = (v16u8)__msa_ld_b((v16i8*)src_argb0, 16);
+    src2 = (v16u8)__msa_ld_b((v16i8*)src_argb1, 0);
+    src3 = (v16u8)__msa_ld_b((v16i8*)src_argb1, 16);
+    vec0 = (v8u16)__msa_ilvr_b(zero, (v16i8)src0);
+    vec1 = (v8u16)__msa_ilvl_b(zero, (v16i8)src0);
+    vec2 = (v8u16)__msa_ilvr_b(zero, (v16i8)src1);
+    vec3 = (v8u16)__msa_ilvl_b(zero, (v16i8)src1);
+    vec4 = (v8u16)__msa_ilvr_b(zero, (v16i8)src2);
+    vec5 = (v8u16)__msa_ilvl_b(zero, (v16i8)src2);
+    vec6 = (v8u16)__msa_ilvr_b(zero, (v16i8)src3);
+    vec7 = (v8u16)__msa_ilvl_b(zero, (v16i8)src3);
+    vec8 = (v8u16)__msa_fill_h(vec0[3]);
+    vec9 = (v8u16)__msa_fill_h(vec0[7]);
+    vec10 = (v8u16)__msa_fill_h(vec1[3]);
+    vec11 = (v8u16)__msa_fill_h(vec1[7]);
+    vec8 = (v8u16)__msa_pckev_d((v2i64)vec9, (v2i64)vec8);
+    vec9 = (v8u16)__msa_pckev_d((v2i64)vec11, (v2i64)vec10);
+    vec10 = (v8u16)__msa_fill_h(vec2[3]);
+    vec11 = (v8u16)__msa_fill_h(vec2[7]);
+    vec12 = (v8u16)__msa_fill_h(vec3[3]);
+    vec13 = (v8u16)__msa_fill_h(vec3[7]);
+    vec10 = (v8u16)__msa_pckev_d((v2i64)vec11, (v2i64)vec10);
+    vec11 = (v8u16)__msa_pckev_d((v2i64)vec13, (v2i64)vec12);
+    vec8 = const_256 - vec8;
+    vec9 = const_256 - vec9;
+    vec10 = const_256 - vec10;
+    vec11 = const_256 - vec11;
+    vec8 *= vec4;
+    vec9 *= vec5;
+    vec10 *= vec6;
+    vec11 *= vec7;
+    vec8 = (v8u16)__msa_srai_h((v8i16)vec8, 8);
+    vec9 = (v8u16)__msa_srai_h((v8i16)vec9, 8);
+    vec10 = (v8u16)__msa_srai_h((v8i16)vec10, 8);
+    vec11 = (v8u16)__msa_srai_h((v8i16)vec11, 8);
+    vec0 += vec8;
+    vec1 += vec9;
+    vec2 += vec10;
+    vec3 += vec11;
+    dst0 = (v16u8)__msa_pckev_b((v16i8)vec1, (v16i8)vec0);
+    dst1 = (v16u8)__msa_pckev_b((v16i8)vec3, (v16i8)vec2);
+    dst0 = __msa_bmnz_v(dst0, const_255, mask);
+    dst1 = __msa_bmnz_v(dst1, const_255, mask);
+    ST_UB2(dst0, dst1, dst_argb, 16);
+    src_argb0 += 32;
+    src_argb1 += 32;
+    dst_argb += 32;
+  }
+}
+
+void ARGBQuantizeRow_MSA(uint8* dst_argb,
+                         int scale,
+                         int interval_size,
+                         int interval_offset,
+                         int width) {
+  int x;
+  v16u8 src0, src1, src2, src3, dst0, dst1, dst2, dst3;
+  v8i16 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
+  v4i32 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
+  v4i32 tmp8, tmp9, tmp10, tmp11, tmp12, tmp13, tmp14, tmp15;
+  v4i32 vec_scale = __msa_fill_w(scale);
+  v16u8 vec_int_sz = (v16u8)__msa_fill_b(interval_size);
+  v16u8 vec_int_ofst = (v16u8)__msa_fill_b(interval_offset);
+  v16i8 mask = {0, 1, 2, 19, 4, 5, 6, 23, 8, 9, 10, 27, 12, 13, 14, 31};
+  v16i8 zero = {0};
+
+  for (x = 0; x < width; x += 8) {
+    src0 = (v16u8)__msa_ld_b((v16i8*)dst_argb, 0);
+    src1 = (v16u8)__msa_ld_b((v16i8*)dst_argb, 16);
+    src2 = (v16u8)__msa_ld_b((v16i8*)dst_argb, 32);
+    src3 = (v16u8)__msa_ld_b((v16i8*)dst_argb, 48);
+    vec0 = (v8i16)__msa_ilvr_b(zero, (v16i8)src0);
+    vec1 = (v8i16)__msa_ilvl_b(zero, (v16i8)src0);
+    vec2 = (v8i16)__msa_ilvr_b(zero, (v16i8)src1);
+    vec3 = (v8i16)__msa_ilvl_b(zero, (v16i8)src1);
+    vec4 = (v8i16)__msa_ilvr_b(zero, (v16i8)src2);
+    vec5 = (v8i16)__msa_ilvl_b(zero, (v16i8)src2);
+    vec6 = (v8i16)__msa_ilvr_b(zero, (v16i8)src3);
+    vec7 = (v8i16)__msa_ilvl_b(zero, (v16i8)src3);
+    tmp0 = (v4i32)__msa_ilvr_h((v8i16)zero, (v8i16)vec0);
+    tmp1 = (v4i32)__msa_ilvl_h((v8i16)zero, (v8i16)vec0);
+    tmp2 = (v4i32)__msa_ilvr_h((v8i16)zero, (v8i16)vec1);
+    tmp3 = (v4i32)__msa_ilvl_h((v8i16)zero, (v8i16)vec1);
+    tmp4 = (v4i32)__msa_ilvr_h((v8i16)zero, (v8i16)vec2);
+    tmp5 = (v4i32)__msa_ilvl_h((v8i16)zero, (v8i16)vec2);
+    tmp6 = (v4i32)__msa_ilvr_h((v8i16)zero, (v8i16)vec3);
+    tmp7 = (v4i32)__msa_ilvl_h((v8i16)zero, (v8i16)vec3);
+    tmp8 = (v4i32)__msa_ilvr_h((v8i16)zero, (v8i16)vec4);
+    tmp9 = (v4i32)__msa_ilvl_h((v8i16)zero, (v8i16)vec4);
+    tmp10 = (v4i32)__msa_ilvr_h((v8i16)zero, (v8i16)vec5);
+    tmp11 = (v4i32)__msa_ilvl_h((v8i16)zero, (v8i16)vec5);
+    tmp12 = (v4i32)__msa_ilvr_h((v8i16)zero, (v8i16)vec6);
+    tmp13 = (v4i32)__msa_ilvl_h((v8i16)zero, (v8i16)vec6);
+    tmp14 = (v4i32)__msa_ilvr_h((v8i16)zero, (v8i16)vec7);
+    tmp15 = (v4i32)__msa_ilvl_h((v8i16)zero, (v8i16)vec7);
+    tmp0 *= vec_scale;
+    tmp1 *= vec_scale;
+    tmp2 *= vec_scale;
+    tmp3 *= vec_scale;
+    tmp4 *= vec_scale;
+    tmp5 *= vec_scale;
+    tmp6 *= vec_scale;
+    tmp7 *= vec_scale;
+    tmp8 *= vec_scale;
+    tmp9 *= vec_scale;
+    tmp10 *= vec_scale;
+    tmp11 *= vec_scale;
+    tmp12 *= vec_scale;
+    tmp13 *= vec_scale;
+    tmp14 *= vec_scale;
+    tmp15 *= vec_scale;
+    tmp0 >>= 16;
+    tmp1 >>= 16;
+    tmp2 >>= 16;
+    tmp3 >>= 16;
+    tmp4 >>= 16;
+    tmp5 >>= 16;
+    tmp6 >>= 16;
+    tmp7 >>= 16;
+    tmp8 >>= 16;
+    tmp9 >>= 16;
+    tmp10 >>= 16;
+    tmp11 >>= 16;
+    tmp12 >>= 16;
+    tmp13 >>= 16;
+    tmp14 >>= 16;
+    tmp15 >>= 16;
+    vec0 = (v8i16)__msa_pckev_h((v8i16)tmp1, (v8i16)tmp0);
+    vec1 = (v8i16)__msa_pckev_h((v8i16)tmp3, (v8i16)tmp2);
+    vec2 = (v8i16)__msa_pckev_h((v8i16)tmp5, (v8i16)tmp4);
+    vec3 = (v8i16)__msa_pckev_h((v8i16)tmp7, (v8i16)tmp6);
+    vec4 = (v8i16)__msa_pckev_h((v8i16)tmp9, (v8i16)tmp8);
+    vec5 = (v8i16)__msa_pckev_h((v8i16)tmp11, (v8i16)tmp10);
+    vec6 = (v8i16)__msa_pckev_h((v8i16)tmp13, (v8i16)tmp12);
+    vec7 = (v8i16)__msa_pckev_h((v8i16)tmp15, (v8i16)tmp14);
+    dst0 = (v16u8)__msa_pckev_b((v16i8)vec1, (v16i8)vec0);
+    dst1 = (v16u8)__msa_pckev_b((v16i8)vec3, (v16i8)vec2);
+    dst2 = (v16u8)__msa_pckev_b((v16i8)vec5, (v16i8)vec4);
+    dst3 = (v16u8)__msa_pckev_b((v16i8)vec7, (v16i8)vec6);
+    dst0 *= vec_int_sz;
+    dst1 *= vec_int_sz;
+    dst2 *= vec_int_sz;
+    dst3 *= vec_int_sz;
+    dst0 += vec_int_ofst;
+    dst1 += vec_int_ofst;
+    dst2 += vec_int_ofst;
+    dst3 += vec_int_ofst;
+    dst0 = (v16u8)__msa_vshf_b(mask, (v16i8)src0, (v16i8)dst0);
+    dst1 = (v16u8)__msa_vshf_b(mask, (v16i8)src1, (v16i8)dst1);
+    dst2 = (v16u8)__msa_vshf_b(mask, (v16i8)src2, (v16i8)dst2);
+    dst3 = (v16u8)__msa_vshf_b(mask, (v16i8)src3, (v16i8)dst3);
+    ST_UB4(dst0, dst1, dst2, dst3, dst_argb, 16);
+    dst_argb += 64;
+  }
+}
+
+void ARGBColorMatrixRow_MSA(const uint8* src_argb,
+                            uint8* dst_argb,
+                            const int8* matrix_argb,
+                            int width) {
+  int32 x;
+  v16i8 src0;
+  v16u8 src1, src2, dst0, dst1;
+  v8i16 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9;
+  v8i16 vec10, vec11, vec12, vec13, vec14, vec15, vec16, vec17;
+  v4i32 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
+  v4i32 tmp8, tmp9, tmp10, tmp11, tmp12, tmp13, tmp14, tmp15;
+  v16i8 zero = {0};
+  v8i16 max = __msa_ldi_h(255);
+
+  src0 = __msa_ld_b((v16i8*)matrix_argb, 0);
+  vec0 = (v8i16)__msa_ilvr_b(zero, src0);
+  vec1 = (v8i16)__msa_ilvl_b(zero, src0);
+
+  for (x = 0; x < width; x += 8) {
+    src1 = (v16u8)__msa_ld_b((v16i8*)src_argb, 0);
+    src2 = (v16u8)__msa_ld_b((v16i8*)src_argb, 16);
+    vec2 = (v8i16)__msa_ilvr_b(zero, (v16i8)src1);
+    vec3 = (v8i16)__msa_ilvl_b(zero, (v16i8)src1);
+    vec4 = (v8i16)__msa_ilvr_b(zero, (v16i8)src2);
+    vec5 = (v8i16)__msa_ilvl_b(zero, (v16i8)src2);
+    vec6 = (v8i16)__msa_pckod_d((v2i64)vec2, (v2i64)vec2);
+    vec7 = (v8i16)__msa_pckod_d((v2i64)vec3, (v2i64)vec3);
+    vec8 = (v8i16)__msa_pckod_d((v2i64)vec4, (v2i64)vec4);
+    vec9 = (v8i16)__msa_pckod_d((v2i64)vec5, (v2i64)vec5);
+    vec2 = (v8i16)__msa_pckev_d((v2i64)vec2, (v2i64)vec2);
+    vec3 = (v8i16)__msa_pckev_d((v2i64)vec3, (v2i64)vec3);
+    vec4 = (v8i16)__msa_pckev_d((v2i64)vec4, (v2i64)vec4);
+    vec5 = (v8i16)__msa_pckev_d((v2i64)vec5, (v2i64)vec5);
+    vec10 = vec2 * vec0;
+    vec11 = vec2 * vec1;
+    vec12 = vec6 * vec0;
+    vec13 = vec6 * vec1;
+    tmp0 = __msa_hadd_s_w(vec10, vec10);
+    tmp1 = __msa_hadd_s_w(vec11, vec11);
+    tmp2 = __msa_hadd_s_w(vec12, vec12);
+    tmp3 = __msa_hadd_s_w(vec13, vec13);
+    vec14 = vec3 * vec0;
+    vec15 = vec3 * vec1;
+    vec16 = vec7 * vec0;
+    vec17 = vec7 * vec1;
+    tmp4 = __msa_hadd_s_w(vec14, vec14);
+    tmp5 = __msa_hadd_s_w(vec15, vec15);
+    tmp6 = __msa_hadd_s_w(vec16, vec16);
+    tmp7 = __msa_hadd_s_w(vec17, vec17);
+    vec10 = __msa_pckev_h((v8i16)tmp1, (v8i16)tmp0);
+    vec11 = __msa_pckev_h((v8i16)tmp3, (v8i16)tmp2);
+    vec12 = __msa_pckev_h((v8i16)tmp5, (v8i16)tmp4);
+    vec13 = __msa_pckev_h((v8i16)tmp7, (v8i16)tmp6);
+    tmp0 = __msa_hadd_s_w(vec10, vec10);
+    tmp1 = __msa_hadd_s_w(vec11, vec11);
+    tmp2 = __msa_hadd_s_w(vec12, vec12);
+    tmp3 = __msa_hadd_s_w(vec13, vec13);
+    tmp0 = __msa_srai_w(tmp0, 6);
+    tmp1 = __msa_srai_w(tmp1, 6);
+    tmp2 = __msa_srai_w(tmp2, 6);
+    tmp3 = __msa_srai_w(tmp3, 6);
+    vec2 = vec4 * vec0;
+    vec6 = vec4 * vec1;
+    vec3 = vec8 * vec0;
+    vec7 = vec8 * vec1;
+    tmp8 = __msa_hadd_s_w(vec2, vec2);
+    tmp9 = __msa_hadd_s_w(vec6, vec6);
+    tmp10 = __msa_hadd_s_w(vec3, vec3);
+    tmp11 = __msa_hadd_s_w(vec7, vec7);
+    vec4 = vec5 * vec0;
+    vec8 = vec5 * vec1;
+    vec5 = vec9 * vec0;
+    vec9 = vec9 * vec1;
+    tmp12 = __msa_hadd_s_w(vec4, vec4);
+    tmp13 = __msa_hadd_s_w(vec8, vec8);
+    tmp14 = __msa_hadd_s_w(vec5, vec5);
+    tmp15 = __msa_hadd_s_w(vec9, vec9);
+    vec14 = __msa_pckev_h((v8i16)tmp9, (v8i16)tmp8);
+    vec15 = __msa_pckev_h((v8i16)tmp11, (v8i16)tmp10);
+    vec16 = __msa_pckev_h((v8i16)tmp13, (v8i16)tmp12);
+    vec17 = __msa_pckev_h((v8i16)tmp15, (v8i16)tmp14);
+    tmp4 = __msa_hadd_s_w(vec14, vec14);
+    tmp5 = __msa_hadd_s_w(vec15, vec15);
+    tmp6 = __msa_hadd_s_w(vec16, vec16);
+    tmp7 = __msa_hadd_s_w(vec17, vec17);
+    tmp4 = __msa_srai_w(tmp4, 6);
+    tmp5 = __msa_srai_w(tmp5, 6);
+    tmp6 = __msa_srai_w(tmp6, 6);
+    tmp7 = __msa_srai_w(tmp7, 6);
+    vec10 = __msa_pckev_h((v8i16)tmp1, (v8i16)tmp0);
+    vec11 = __msa_pckev_h((v8i16)tmp3, (v8i16)tmp2);
+    vec12 = __msa_pckev_h((v8i16)tmp5, (v8i16)tmp4);
+    vec13 = __msa_pckev_h((v8i16)tmp7, (v8i16)tmp6);
+    vec10 = __msa_maxi_s_h(vec10, 0);
+    vec11 = __msa_maxi_s_h(vec11, 0);
+    vec12 = __msa_maxi_s_h(vec12, 0);
+    vec13 = __msa_maxi_s_h(vec13, 0);
+    vec10 = __msa_min_s_h(vec10, max);
+    vec11 = __msa_min_s_h(vec11, max);
+    vec12 = __msa_min_s_h(vec12, max);
+    vec13 = __msa_min_s_h(vec13, max);
+    dst0 = (v16u8)__msa_pckev_b((v16i8)vec11, (v16i8)vec10);
+    dst1 = (v16u8)__msa_pckev_b((v16i8)vec13, (v16i8)vec12);
+    ST_UB2(dst0, dst1, dst_argb, 16);
+    src_argb += 32;
+    dst_argb += 32;
+  }
+}
+
+void SplitUVRow_MSA(const uint8* src_uv,
+                    uint8* dst_u,
+                    uint8* dst_v,
+                    int width) {
+  int x;
+  v16u8 src0, src1, src2, src3, dst0, dst1, dst2, dst3;
+
+  for (x = 0; x < width; x += 32) {
+    src0 = (v16u8)__msa_ld_b((v16i8*)src_uv, 0);
+    src1 = (v16u8)__msa_ld_b((v16i8*)src_uv, 16);
+    src2 = (v16u8)__msa_ld_b((v16i8*)src_uv, 32);
+    src3 = (v16u8)__msa_ld_b((v16i8*)src_uv, 48);
+    dst0 = (v16u8)__msa_pckev_b((v16i8)src1, (v16i8)src0);
+    dst1 = (v16u8)__msa_pckev_b((v16i8)src3, (v16i8)src2);
+    dst2 = (v16u8)__msa_pckod_b((v16i8)src1, (v16i8)src0);
+    dst3 = (v16u8)__msa_pckod_b((v16i8)src3, (v16i8)src2);
+    ST_UB2(dst0, dst1, dst_u, 16);
+    ST_UB2(dst2, dst3, dst_v, 16);
+    src_uv += 64;
+    dst_u += 32;
+    dst_v += 32;
+  }
+}
+
+void SetRow_MSA(uint8* dst, uint8 v8, int width) {
+  int x;
+  v16u8 dst0 = (v16u8)__msa_fill_b(v8);
+
+  for (x = 0; x < width; x += 16) {
+    ST_UB(dst0, dst);
+    dst += 16;
+  }
+}
+
+void MirrorUVRow_MSA(const uint8* src_uv,
+                     uint8* dst_u,
+                     uint8* dst_v,
+                     int width) {
+  int x;
+  v16u8 src0, src1, src2, src3;
+  v16u8 dst0, dst1, dst2, dst3;
+  v16i8 mask0 = {30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0};
+  v16i8 mask1 = {31, 29, 27, 25, 23, 21, 19, 17, 15, 13, 11, 9, 7, 5, 3, 1};
+
+  src_uv += (2 * width);
+
+  for (x = 0; x < width; x += 32) {
+    src_uv -= 64;
+    src2 = (v16u8)__msa_ld_b((v16i8*)src_uv, 0);
+    src3 = (v16u8)__msa_ld_b((v16i8*)src_uv, 16);
+    src0 = (v16u8)__msa_ld_b((v16i8*)src_uv, 32);
+    src1 = (v16u8)__msa_ld_b((v16i8*)src_uv, 48);
+    dst0 = (v16u8)__msa_vshf_b(mask1, (v16i8)src1, (v16i8)src0);
+    dst1 = (v16u8)__msa_vshf_b(mask1, (v16i8)src3, (v16i8)src2);
+    dst2 = (v16u8)__msa_vshf_b(mask0, (v16i8)src1, (v16i8)src0);
+    dst3 = (v16u8)__msa_vshf_b(mask0, (v16i8)src3, (v16i8)src2);
+    ST_UB2(dst0, dst1, dst_v, 16);
+    ST_UB2(dst2, dst3, dst_u, 16);
+    dst_u += 32;
+    dst_v += 32;
+  }
+}
+
+void SobelXRow_MSA(const uint8* src_y0,
+                   const uint8* src_y1,
+                   const uint8* src_y2,
+                   uint8* dst_sobelx,
+                   int32 width) {
+  int x;
+  v16u8 src0, src1, src2, src3, src4, src5, dst0;
+  v8i16 vec0, vec1, vec2, vec3, vec4, vec5;
+  v16i8 mask0 = {0, 2, 1, 3, 2, 4, 3, 5, 4, 6, 5, 7, 6, 8, 7, 9};
+  v16i8 tmp = __msa_ldi_b(8);
+  v16i8 mask1 = mask0 + tmp;
+  v8i16 zero = {0};
+  v8i16 max = __msa_ldi_h(255);
+
+  for (x = 0; x < width; x += 16) {
+    src0 = (v16u8)__msa_ld_b((v16i8*)src_y0, 0);
+    src1 = (v16u8)__msa_ld_b((v16i8*)src_y0, 16);
+    src2 = (v16u8)__msa_ld_b((v16i8*)src_y1, 0);
+    src3 = (v16u8)__msa_ld_b((v16i8*)src_y1, 16);
+    src4 = (v16u8)__msa_ld_b((v16i8*)src_y2, 0);
+    src5 = (v16u8)__msa_ld_b((v16i8*)src_y2, 16);
+    vec0 = (v8i16)__msa_vshf_b(mask0, (v16i8)src1, (v16i8)src0);
+    vec1 = (v8i16)__msa_vshf_b(mask1, (v16i8)src1, (v16i8)src0);
+    vec2 = (v8i16)__msa_vshf_b(mask0, (v16i8)src3, (v16i8)src2);
+    vec3 = (v8i16)__msa_vshf_b(mask1, (v16i8)src3, (v16i8)src2);
+    vec4 = (v8i16)__msa_vshf_b(mask0, (v16i8)src5, (v16i8)src4);
+    vec5 = (v8i16)__msa_vshf_b(mask1, (v16i8)src5, (v16i8)src4);
+    vec0 = (v8i16)__msa_hsub_u_h((v16u8)vec0, (v16u8)vec0);
+    vec1 = (v8i16)__msa_hsub_u_h((v16u8)vec1, (v16u8)vec1);
+    vec2 = (v8i16)__msa_hsub_u_h((v16u8)vec2, (v16u8)vec2);
+    vec3 = (v8i16)__msa_hsub_u_h((v16u8)vec3, (v16u8)vec3);
+    vec4 = (v8i16)__msa_hsub_u_h((v16u8)vec4, (v16u8)vec4);
+    vec5 = (v8i16)__msa_hsub_u_h((v16u8)vec5, (v16u8)vec5);
+    vec0 += vec2;
+    vec1 += vec3;
+    vec4 += vec2;
+    vec5 += vec3;
+    vec0 += vec4;
+    vec1 += vec5;
+    vec0 = __msa_add_a_h(zero, vec0);
+    vec1 = __msa_add_a_h(zero, vec1);
+    vec0 = __msa_maxi_s_h(vec0, 0);
+    vec1 = __msa_maxi_s_h(vec1, 0);
+    vec0 = __msa_min_s_h(max, vec0);
+    vec1 = __msa_min_s_h(max, vec1);
+    dst0 = (v16u8)__msa_pckev_b((v16i8)vec1, (v16i8)vec0);
+    ST_UB(dst0, dst_sobelx);
+    src_y0 += 16;
+    src_y1 += 16;
+    src_y2 += 16;
+    dst_sobelx += 16;
+  }
+}
+
+void SobelYRow_MSA(const uint8* src_y0,
+                   const uint8* src_y1,
+                   uint8* dst_sobely,
+                   int32 width) {
+  int x;
+  v16u8 src0, src1, dst0;
+  v8i16 vec0, vec1, vec2, vec3, vec4, vec5, vec6;
+  v8i16 zero = {0};
+  v8i16 max = __msa_ldi_h(255);
+
+  for (x = 0; x < width; x += 16) {
+    src0 = (v16u8)__msa_ld_b((v16i8*)src_y0, 0);
+    src1 = (v16u8)__msa_ld_b((v16i8*)src_y1, 0);
+    vec0 = (v8i16)__msa_ilvr_b((v16i8)zero, (v16i8)src0);
+    vec1 = (v8i16)__msa_ilvl_b((v16i8)zero, (v16i8)src0);
+    vec2 = (v8i16)__msa_ilvr_b((v16i8)zero, (v16i8)src1);
+    vec3 = (v8i16)__msa_ilvl_b((v16i8)zero, (v16i8)src1);
+    vec0 -= vec2;
+    vec1 -= vec3;
+    vec6[0] = src_y0[16] - src_y1[16];
+    vec6[1] = src_y0[17] - src_y1[17];
+    vec2 = (v8i16)__msa_sldi_b((v16i8)vec1, (v16i8)vec0, 2);
+    vec3 = (v8i16)__msa_sldi_b((v16i8)vec6, (v16i8)vec1, 2);
+    vec4 = (v8i16)__msa_sldi_b((v16i8)vec1, (v16i8)vec0, 4);
+    vec5 = (v8i16)__msa_sldi_b((v16i8)vec6, (v16i8)vec1, 4);
+    vec0 += vec2;
+    vec1 += vec3;
+    vec4 += vec2;
+    vec5 += vec3;
+    vec0 += vec4;
+    vec1 += vec5;
+    vec0 = __msa_add_a_h(zero, vec0);
+    vec1 = __msa_add_a_h(zero, vec1);
+    vec0 = __msa_maxi_s_h(vec0, 0);
+    vec1 = __msa_maxi_s_h(vec1, 0);
+    vec0 = __msa_min_s_h(max, vec0);
+    vec1 = __msa_min_s_h(max, vec1);
+    dst0 = (v16u8)__msa_pckev_b((v16i8)vec1, (v16i8)vec0);
+    ST_UB(dst0, dst_sobely);
+    src_y0 += 16;
+    src_y1 += 16;
+    dst_sobely += 16;
+  }
+}
+
+void HalfFloatRow_MSA(const uint16* src, uint16* dst, float scale, int width) {
+  int i;
+  v8u16 src0, src1, src2, src3, dst0, dst1, dst2, dst3;
+  v4u32 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
+  v4f32 fvec0, fvec1, fvec2, fvec3, fvec4, fvec5, fvec6, fvec7;
+  v4f32 mult_vec;
+  v8i16 zero = {0};
+  mult_vec[0] = 1.9259299444e-34f * scale;
+  mult_vec = (v4f32)__msa_splati_w((v4i32)mult_vec, 0);
+
+  for (i = 0; i < width; i += 32) {
+    src0 = (v8u16)__msa_ld_h((v8i16*)src, 0);
+    src1 = (v8u16)__msa_ld_h((v8i16*)src, 16);
+    src2 = (v8u16)__msa_ld_h((v8i16*)src, 32);
+    src3 = (v8u16)__msa_ld_h((v8i16*)src, 48);
+    vec0 = (v4u32)__msa_ilvr_h(zero, (v8i16)src0);
+    vec1 = (v4u32)__msa_ilvl_h(zero, (v8i16)src0);
+    vec2 = (v4u32)__msa_ilvr_h(zero, (v8i16)src1);
+    vec3 = (v4u32)__msa_ilvl_h(zero, (v8i16)src1);
+    vec4 = (v4u32)__msa_ilvr_h(zero, (v8i16)src2);
+    vec5 = (v4u32)__msa_ilvl_h(zero, (v8i16)src2);
+    vec6 = (v4u32)__msa_ilvr_h(zero, (v8i16)src3);
+    vec7 = (v4u32)__msa_ilvl_h(zero, (v8i16)src3);
+    fvec0 = __msa_ffint_u_w(vec0);
+    fvec1 = __msa_ffint_u_w(vec1);
+    fvec2 = __msa_ffint_u_w(vec2);
+    fvec3 = __msa_ffint_u_w(vec3);
+    fvec4 = __msa_ffint_u_w(vec4);
+    fvec5 = __msa_ffint_u_w(vec5);
+    fvec6 = __msa_ffint_u_w(vec6);
+    fvec7 = __msa_ffint_u_w(vec7);
+    fvec0 *= mult_vec;
+    fvec1 *= mult_vec;
+    fvec2 *= mult_vec;
+    fvec3 *= mult_vec;
+    fvec4 *= mult_vec;
+    fvec5 *= mult_vec;
+    fvec6 *= mult_vec;
+    fvec7 *= mult_vec;
+    vec0 = ((v4u32)fvec0) >> 13;
+    vec1 = ((v4u32)fvec1) >> 13;
+    vec2 = ((v4u32)fvec2) >> 13;
+    vec3 = ((v4u32)fvec3) >> 13;
+    vec4 = ((v4u32)fvec4) >> 13;
+    vec5 = ((v4u32)fvec5) >> 13;
+    vec6 = ((v4u32)fvec6) >> 13;
+    vec7 = ((v4u32)fvec7) >> 13;
+    dst0 = (v8u16)__msa_pckev_h((v8i16)vec1, (v8i16)vec0);
+    dst1 = (v8u16)__msa_pckev_h((v8i16)vec3, (v8i16)vec2);
+    dst2 = (v8u16)__msa_pckev_h((v8i16)vec5, (v8i16)vec4);
+    dst3 = (v8u16)__msa_pckev_h((v8i16)vec7, (v8i16)vec6);
+    ST_UH2(dst0, dst1, dst, 8);
+    ST_UH2(dst2, dst3, dst + 16, 8);
+    src += 32;
+    dst += 32;
+  }
+}
+
 #ifdef __cplusplus
 }  // extern "C"

pjproject/trunk/third_party/yuv/source/row_neon.cc

@@ -116 +116 @@
       YUVTORGB_SETUP
       "vmov.u8    d23, #255                      \n"
-      "1:                                          \n" READYUV444 YUVTORGB
+      "1:                                        \n" READYUV444 YUVTORGB
       "subs       %4, %4, #8                     \n"
       "vst4.8     {d20, d21, d22, d23}, [%3]!    \n"
@@ -142 +142 @@
       YUVTORGB_SETUP
       "vmov.u8    d23, #255                      \n"
-      "1:                                          \n" READYUV422 YUVTORGB
+      "1:                                        \n" READYUV422 YUVTORGB
       "subs       %4, %4, #8                     \n"
       "vst4.8     {d20, d21, d22, d23}, [%3]!    \n"
@@ -168 +168 @@
   asm volatile(
       YUVTORGB_SETUP
-      "1:                                          \n" READYUV422 YUVTORGB
+      "1:                                        \n" READYUV422 YUVTORGB
       "subs       %5, %5, #8                     \n"
       "vld1.8     {d23}, [%3]!                   \n"
@@ -195 +195 @@
   asm volatile(
       YUVTORGB_SETUP
-      "1:                                          \n" READYUV422 YUVTORGB
+      "1:                                        \n" READYUV422 YUVTORGB
       "subs       %4, %4, #8                     \n"
       "vmov.u8    d19, #255                      \n"  // d19 modified by
@@ -222 +222 @@
   asm volatile(
       YUVTORGB_SETUP
-      "1:                                          \n" READYUV422 YUVTORGB
+      "1:                                        \n" READYUV422 YUVTORGB
       "subs       %4, %4, #8                     \n"
       "vst3.8     {d20, d21, d22}, [%3]!         \n"
@@ -254 +254 @@
   asm volatile(
       YUVTORGB_SETUP
-      "1:                                          \n" READYUV422 YUVTORGB
+      "1:                                        \n" READYUV422 YUVTORGB
       "subs       %4, %4, #8                     \n" ARGBTORGB565
       "vst1.8     {q0}, [%3]!                    \n"  // store 8 pixels RGB565.
@@ -288 +288 @@
   asm volatile(
       YUVTORGB_SETUP
-      "1:                                          \n" READYUV422 YUVTORGB
+      "1:                                        \n" READYUV422 YUVTORGB
       "subs       %4, %4, #8                     \n"
       "vmov.u8    d23, #255                      \n" ARGBTOARGB1555
@@ -326 +326 @@
       "vmov.u8    d4, #0x0f                      \n"  // bits to clear with
                                                       // vbic.
-      "1:                                          \n" READYUV422 YUVTORGB
+      "1:                                        \n" READYUV422 YUVTORGB
       "subs       %4, %4, #8                     \n"
       "vmov.u8    d23, #255                      \n" ARGBTOARGB4444
@@ -349 +349 @@
       YUVTORGB_SETUP
       "vmov.u8    d23, #255                      \n"
-      "1:                                          \n" READYUV400 YUVTORGB
+      "1:                                        \n" READYUV400 YUVTORGB
       "subs       %2, %2, #8                     \n"
       "vst4.8     {d20, d21, d22, d23}, [%1]!    \n"
@@ -367 +367 @@
   asm volatile(
       "vmov.u8    d23, #255                      \n"
-      "1:                                          \n"
+      "1:                                        \n"
       "vld1.8     {d20}, [%0]!                   \n"
       "vmov       d21, d20                       \n"
@@ -386 +386 @@
                         const struct YuvConstants* yuvconstants,
                         int width) {
-  asm volatile(
-      YUVTORGB_SETUP
-      "vmov.u8    d23, #255                      \n"
-      "1:                                          \n" READNV12 YUVTORGB
-      "subs       %3, %3, #8                     \n"
-      "vst4.8     {d20, d21, d22, d23}, [%2]!    \n"
-      "bgt        1b                             \n"
-      : "+r"(src_y),     // %0
-        "+r"(src_uv),    // %1
-        "+r"(dst_argb),  // %2
-        "+r"(width)      // %3
-      : [kUVToRB] "r"(&yuvconstants->kUVToRB),
-        [kUVToG] "r"(&yuvconstants->kUVToG),
-        [kUVBiasBGR] "r"(&yuvconstants->kUVBiasBGR),
-        [kYToRgb] "r"(&yuvconstants->kYToRgb)
-      : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q8", "q9", "q10", "q11",
-        "q12", "q13", "q14", "q15");
+  asm volatile(YUVTORGB_SETUP
+               "vmov.u8    d23, #255                      \n"
+               "1:                                        \n" READNV12 YUVTORGB
+               "subs       %3, %3, #8                     \n"
+               "vst4.8     {d20, d21, d22, d23}, [%2]!    \n"
+               "bgt        1b                             \n"
+               : "+r"(src_y),     // %0
+                 "+r"(src_uv),    // %1
+                 "+r"(dst_argb),  // %2
+                 "+r"(width)      // %3
+               : [kUVToRB] "r"(&yuvconstants->kUVToRB),
+                 [kUVToG] "r"(&yuvconstants->kUVToG),
+                 [kUVBiasBGR] "r"(&yuvconstants->kUVBiasBGR),
+                 [kYToRgb] "r"(&yuvconstants->kYToRgb)
+               : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q8", "q9",
+                 "q10", "q11", "q12", "q13", "q14", "q15");
 }
 
@@ -410 +409 @@
                         const struct YuvConstants* yuvconstants,
                         int width) {
-  asm volatile(
-      YUVTORGB_SETUP
-      "vmov.u8    d23, #255                      \n"
-      "1:                                          \n" READNV21 YUVTORGB
-      "subs       %3, %3, #8                     \n"
-      "vst4.8     {d20, d21, d22, d23}, [%2]!    \n"
-      "bgt        1b                             \n"
-      : "+r"(src_y),     // %0
-        "+r"(src_vu),    // %1
-        "+r"(dst_argb),  // %2
-        "+r"(width)      // %3
-      : [kUVToRB] "r"(&yuvconstants->kUVToRB),
-        [kUVToG] "r"(&yuvconstants->kUVToG),
-        [kUVBiasBGR] "r"(&yuvconstants->kUVBiasBGR),
-        [kYToRgb] "r"(&yuvconstants->kYToRgb)
-      : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q8", "q9", "q10", "q11",
-        "q12", "q13", "q14", "q15");
+  asm volatile(YUVTORGB_SETUP
+               "vmov.u8    d23, #255                      \n"
+               "1:                                        \n" READNV21 YUVTORGB
+               "subs       %3, %3, #8                     \n"
+               "vst4.8     {d20, d21, d22, d23}, [%2]!    \n"
+               "bgt        1b                             \n"
+               : "+r"(src_y),     // %0
+                 "+r"(src_vu),    // %1
+                 "+r"(dst_argb),  // %2
+                 "+r"(width)      // %3
+               : [kUVToRB] "r"(&yuvconstants->kUVToRB),
+                 [kUVToG] "r"(&yuvconstants->kUVToG),
+                 [kUVBiasBGR] "r"(&yuvconstants->kUVBiasBGR),
+                 [kYToRgb] "r"(&yuvconstants->kYToRgb)
+               : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q8", "q9",
+                 "q10", "q11", "q12", "q13", "q14", "q15");
 }
 
@@ -436 +434 @@
   asm volatile(
       YUVTORGB_SETUP
-      "1:                                          \n" READNV12 YUVTORGB
+      "1:                                        \n" READNV12 YUVTORGB
       "subs       %3, %3, #8                     \n" ARGBTORGB565
       "vst1.8     {q0}, [%2]!                    \n"  // store 8 pixels RGB565.
@@ -456 +454 @@
                         const struct YuvConstants* yuvconstants,
                         int width) {
-  asm volatile(
-      YUVTORGB_SETUP
-      "vmov.u8    d23, #255                      \n"
-      "1:                                          \n" READYUY2 YUVTORGB
-      "subs       %2, %2, #8                     \n"
-      "vst4.8     {d20, d21, d22, d23}, [%1]!    \n"
-      "bgt        1b                             \n"
-      : "+r"(src_yuy2),  // %0
-        "+r"(dst_argb),  // %1
-        "+r"(width)      // %2
-      : [kUVToRB] "r"(&yuvconstants->kUVToRB),
-        [kUVToG] "r"(&yuvconstants->kUVToG),
-        [kUVBiasBGR] "r"(&yuvconstants->kUVBiasBGR),
-        [kYToRgb] "r"(&yuvconstants->kYToRgb)
-      : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q8", "q9", "q10", "q11",
-        "q12", "q13", "q14", "q15");
+  asm volatile(YUVTORGB_SETUP
+               "vmov.u8    d23, #255                      \n"
+               "1:                                        \n" READYUY2 YUVTORGB
+               "subs       %2, %2, #8                     \n"
+               "vst4.8     {d20, d21, d22, d23}, [%1]!    \n"
+               "bgt        1b                             \n"
+               : "+r"(src_yuy2),  // %0
+                 "+r"(dst_argb),  // %1
+                 "+r"(width)      // %2
+               : [kUVToRB] "r"(&yuvconstants->kUVToRB),
+                 [kUVToG] "r"(&yuvconstants->kUVToG),
+                 [kUVBiasBGR] "r"(&yuvconstants->kUVBiasBGR),
+                 [kYToRgb] "r"(&yuvconstants->kYToRgb)
+               : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q8", "q9",
+                 "q10", "q11", "q12", "q13", "q14", "q15");
 }
 
@@ -478 +475 @@
                         const struct YuvConstants* yuvconstants,
                         int width) {
-  asm volatile(
-      YUVTORGB_SETUP
-      "vmov.u8    d23, #255                      \n"
-      "1:                                          \n" READUYVY YUVTORGB
-      "subs       %2, %2, #8                     \n"
-      "vst4.8     {d20, d21, d22, d23}, [%1]!    \n"
-      "bgt        1b                             \n"
-      : "+r"(src_uyvy),  // %0
-        "+r"(dst_argb),  // %1
-        "+r"(width)      // %2
-      : [kUVToRB] "r"(&yuvconstants->kUVToRB),
-        [kUVToG] "r"(&yuvconstants->kUVToG),
-        [kUVBiasBGR] "r"(&yuvconstants->kUVBiasBGR),
-        [kYToRgb] "r"(&yuvconstants->kYToRgb)
-      : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q8", "q9", "q10", "q11",
-        "q12", "q13", "q14", "q15");
+  asm volatile(YUVTORGB_SETUP
+               "vmov.u8    d23, #255                      \n"
+               "1:                                        \n" READUYVY YUVTORGB
+               "subs       %2, %2, #8                     \n"
+               "vst4.8     {d20, d21, d22, d23}, [%1]!    \n"
+               "bgt        1b                             \n"
+               : "+r"(src_uyvy),  // %0
+                 "+r"(dst_argb),  // %1
+                 "+r"(width)      // %2
+               : [kUVToRB] "r"(&yuvconstants->kUVToRB),
+                 [kUVToG] "r"(&yuvconstants->kUVToG),
+                 [kUVBiasBGR] "r"(&yuvconstants->kUVBiasBGR),
+                 [kYToRgb] "r"(&yuvconstants->kYToRgb)
+               : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q8", "q9",
+                 "q10", "q11", "q12", "q13", "q14", "q15");
 }
 
@@ -502 +498 @@
                      int width) {
   asm volatile(
-      "1:                                          \n"
+      "1:                                        \n"
       "vld2.8     {q0, q1}, [%0]!                \n"  // load 16 pairs of UV
       "subs       %3, %3, #16                    \n"  // 16 processed per loop
@@ -523 +519 @@
                      int width) {
   asm volatile(
-      "1:                                          \n"
+      "1:                                        \n"
       "vld1.8     {q0}, [%0]!                    \n"  // load U
       "vld1.8     {q1}, [%1]!                    \n"  // load V
       "subs       %3, %3, #16                    \n"  // 16 processed per loop
-      "vst2.u8    {q0, q1}, [%2]!                \n"  // store 16 pairs of UV
+      "vst2.8     {q0, q1}, [%2]!                \n"  // store 16 pairs of UV
       "bgt        1b                             \n"
       : "+r"(src_u),                // %0
@@ -538 +534 @@
 }
 
+// Reads 16 packed RGB and write to planar dst_r, dst_g, dst_b.
+void SplitRGBRow_NEON(const uint8* src_rgb,
+                      uint8* dst_r,
+                      uint8* dst_g,
+                      uint8* dst_b,
+                      int width) {
+  asm volatile(
+      "1:                                        \n"
+      "vld3.8     {d0, d2, d4}, [%0]!            \n"  // load 8 RGB
+      "vld3.8     {d1, d3, d5}, [%0]!            \n"  // next 8 RGB
+      "subs       %4, %4, #16                    \n"  // 16 processed per loop
+      "vst1.8     {q0}, [%1]!                    \n"  // store R
+      "vst1.8     {q1}, [%2]!                    \n"  // store G
+      "vst1.8     {q2}, [%3]!                    \n"  // store B
+      "bgt        1b                             \n"
+      : "+r"(src_rgb),                    // %0
+        "+r"(dst_r),                      // %1
+        "+r"(dst_g),                      // %2
+        "+r"(dst_b),                      // %3
+        "+r"(width)                       // %4
+      :                                   // Input registers
+      : "cc", "memory", "d0", "d1", "d2"  // Clobber List
+      );
+}
+
+// Reads 16 planar R's, G's and B's and writes out 16 packed RGB at a time
+void MergeRGBRow_NEON(const uint8* src_r,
+                      const uint8* src_g,
+                      const uint8* src_b,
+                      uint8* dst_rgb,
+                      int width) {
+  asm volatile(
+      "1:                                        \n"
+      "vld1.8     {q0}, [%0]!                    \n"  // load R
+      "vld1.8     {q1}, [%1]!                    \n"  // load G
+      "vld1.8     {q2}, [%2]!                    \n"  // load B
+      "subs       %4, %4, #16                    \n"  // 16 processed per loop
+      "vst3.8     {d0, d2, d4}, [%3]!            \n"  // store 8 RGB
+      "vst3.8     {d1, d3, d5}, [%3]!            \n"  // next 8 RGB
+      "bgt        1b                             \n"
+      : "+r"(src_r),                      // %0
+        "+r"(src_g),                      // %1
+        "+r"(src_b),                      // %2
+        "+r"(dst_rgb),                    // %3
+        "+r"(width)                       // %4
+      :                                   // Input registers
+      : "cc", "memory", "q0", "q1", "q2"  // Clobber List
+      );
+}
+
 // Copy multiple of 32.  vld4.8  allow unaligned and is fastest on a15.
 void CopyRow_NEON(const uint8* src, uint8* dst, int count) {
   asm volatile(
-      "1:                                          \n"
+      "1:                                        \n"
       "vld1.8     {d0, d1, d2, d3}, [%0]!        \n"  // load 32
       "subs       %2, %2, #32                    \n"  // 32 processed per loop
@@ -558 +604 @@
   asm volatile(
       "vdup.8    q0, %2                          \n"  // duplicate 16 bytes
-      "1:                                          \n"
+      "1:                                        \n"
       "subs      %1, %1, #16                     \n"  // 16 bytes per loop
       "vst1.8    {q0}, [%0]!                     \n"  // store
@@ -572 +618 @@
   asm volatile(
       "vdup.u32  q0, %2                          \n"  // duplicate 4 ints
-      "1:                                          \n"
+      "1:                                        \n"
       "subs      %1, %1, #4                      \n"  // 4 pixels per loop
       "vst1.8    {q0}, [%0]!                     \n"  // store
@@ -589 +635 @@
       "sub        %0, #16                        \n"
 
-      "1:                                          \n"
+      "1:                                        \n"
       "vld1.8     {q0}, [%0], r3                 \n"  // src -= 16
       "subs       %2, #16                        \n"  // 16 pixels per loop.
@@ -613 +659 @@
       "sub        %0, #16                        \n"
 
-      "1:                                          \n"
+      "1:                                        \n"
       "vld2.8     {d0, d1}, [%0], r12            \n"  // src -= 16
       "subs       %3, #8                         \n"  // 8 pixels per loop.
@@ -635 +681 @@
       "sub        %0, #16                        \n"
 
-      "1:                                          \n"
+      "1:                                        \n"
       "vld1.8     {q0}, [%0], r3                 \n"  // src -= 16
       "subs       %2, #4                         \n"  // 4 pixels per loop.
@@ -652 +698 @@
   asm volatile(
       "vmov.u8    d4, #255                       \n"  // Alpha
-      "1:                                          \n"
+      "1:                                        \n"
       "vld3.8     {d1, d2, d3}, [%0]!            \n"  // load 8 pixels of RGB24.
       "subs       %2, %2, #8                     \n"  // 8 processed per loop.
@@ -668 +714 @@
   asm volatile(
       "vmov.u8    d4, #255                       \n"  // Alpha
-      "1:                                          \n"
+      "1:                                        \n"
       "vld3.8     {d1, d2, d3}, [%0]!            \n"  // load 8 pixels of RAW.
       "subs       %2, %2, #8                     \n"  // 8 processed per loop.
@@ -684 +730 @@
 void RAWToRGB24Row_NEON(const uint8* src_raw, uint8* dst_rgb24, int width) {
   asm volatile(
-      "1:                                          \n"
+      "1:                                        \n"
       "vld3.8     {d1, d2, d3}, [%0]!            \n"  // load 8 pixels of RAW.
       "subs       %2, %2, #8                     \n"  // 8 processed per loop.
@@ -714 +760 @@
   asm volatile(
       "vmov.u8    d3, #255                       \n"  // Alpha
-      "1:                                          \n"
+      "1:                                        \n"
       "vld1.8     {q0}, [%0]!                    \n"  // load 8 RGB565 pixels.
       "subs       %2, %2, #8                     \n"  // 8 processed per loop.
@@ -760 +806 @@
   asm volatile(
       "vmov.u8    d3, #255                       \n"  // Alpha
-      "1:                                          \n"
+      "1:                                        \n"
       "vld1.8     {q0}, [%0]!                    \n"  // load 8 ARGB1555 pixels.
       "subs       %2, %2, #8                     \n"  // 8 processed per loop.
@@ -789 +835 @@
   asm volatile(
       "vmov.u8    d3, #255                       \n"  // Alpha
-      "1:                                          \n"
+      "1:                                        \n"
       "vld1.8     {q0}, [%0]!                    \n"  // load 8 ARGB4444 pixels.
       "subs       %2, %2, #8                     \n"  // 8 processed per loop.
@@ -805 +851 @@
 void ARGBToRGB24Row_NEON(const uint8* src_argb, uint8* dst_rgb24, int width) {
   asm volatile(
-      "1:                                          \n"
+      "1:                                        \n"
       "vld4.8     {d1, d2, d3, d4}, [%0]!        \n"  // load 8 pixels of ARGB.
       "subs       %2, %2, #8                     \n"  // 8 processed per loop.
@@ -821 +867 @@
 void ARGBToRAWRow_NEON(const uint8* src_argb, uint8* dst_raw, int width) {
   asm volatile(
-      "1:                                          \n"
+      "1:                                        \n"
       "vld4.8     {d1, d2, d3, d4}, [%0]!        \n"  // load 8 pixels of ARGB.
       "subs       %2, %2, #8                     \n"  // 8 processed per loop.
@@ -837 +883 @@
 void YUY2ToYRow_NEON(const uint8* src_yuy2, uint8* dst_y, int width) {
   asm volatile(
-      "1:                                          \n"
+      "1:                                        \n"
       "vld2.8     {q0, q1}, [%0]!                \n"  // load 16 pixels of YUY2.
       "subs       %2, %2, #16                    \n"  // 16 processed per loop.
@@ -852 +898 @@
 void UYVYToYRow_NEON(const uint8* src_uyvy, uint8* dst_y, int width) {
   asm volatile(
-      "1:                                          \n"
+      "1:                                        \n"
       "vld2.8     {q0, q1}, [%0]!                \n"  // load 16 pixels of UYVY.
       "subs       %2, %2, #16                    \n"  // 16 processed per loop.
@@ -870 +916 @@
                          int width) {
   asm volatile(
-      "1:                                          \n"
+      "1:                                        \n"
       "vld4.8     {d0, d1, d2, d3}, [%0]!        \n"  // load 16 pixels of YUY2.
       "subs       %3, %3, #16                    \n"  // 16 pixels = 8 UVs.
@@ -890 +936 @@
                          int width) {
   asm volatile(
-      "1:                                          \n"
+      "1:                                        \n"
       "vld4.8     {d0, d1, d2, d3}, [%0]!        \n"  // load 16 pixels of UYVY.
       "subs       %3, %3, #16                    \n"  // 16 pixels = 8 UVs.
@@ -912 +958 @@
   asm volatile(
       "add        %1, %0, %1                     \n"  // stride + src_yuy2
-      "1:                                          \n"
+      "1:                                        \n"
       "vld4.8     {d0, d1, d2, d3}, [%0]!        \n"  // load 16 pixels of YUY2.
       "subs       %4, %4, #16                    \n"  // 16 pixels = 8 UVs.
@@ -939 +985 @@
   asm volatile(
       "add        %1, %0, %1                     \n"  // stride + src_uyvy
-      "1:                                          \n"
+      "1:                                        \n"
       "vld4.8     {d0, d1, d2, d3}, [%0]!        \n"  // load 16 pixels of UYVY.
       "subs       %4, %4, #16                    \n"  // 16 pixels = 8 UVs.
@@ -966 +1012 @@
   asm volatile(
       "vld1.8     {q2}, [%3]                     \n"  // shuffler
-      "1:                                          \n"
+      "1:                                        \n"
       "vld1.8     {q0}, [%0]!                    \n"  // load 4 pixels.
       "subs       %2, %2, #4                     \n"  // 4 processed per loop
@@ -987 +1033 @@
                         int width) {
   asm volatile(
-      "1:                                          \n"
+      "1:                                        \n"
       "vld2.8     {d0, d2}, [%0]!                \n"  // load 16 Ys
       "vld1.8     {d1}, [%1]!                    \n"  // load 8 Us
@@ -1009 +1055 @@
                         int width) {
   asm volatile(
-      "1:                                          \n"
+      "1:                                        \n"
       "vld2.8     {d1, d3}, [%0]!                \n"  // load 16 Ys
       "vld1.8     {d0}, [%1]!                    \n"  // load 8 Us
@@ -1027 +1073 @@
 void ARGBToRGB565Row_NEON(const uint8* src_argb, uint8* dst_rgb565, int width) {
   asm volatile(
-      "1:                                          \n"
+      "1:                                        \n"
       "vld4.8     {d20, d21, d22, d23}, [%0]!    \n"  // load 8 pixels of ARGB.
       "subs       %2, %2, #8                     \n"  // 8 processed per loop.
@@ -1046 +1092 @@
   asm volatile(
       "vdup.32    d2, %2                         \n"  // dither4
-      "1:                                          \n"
+      "1:                                        \n"
       "vld4.8     {d20, d21, d22, d23}, [%1]!    \n"  // load 8 pixels of ARGB.
       "subs       %3, %3, #8                     \n"  // 8 processed per loop.
       "vqadd.u8   d20, d20, d2                   \n"
       "vqadd.u8   d21, d21, d2                   \n"
-      "vqadd.u8   d22, d22, d2                   \n" ARGBTORGB565
-      "vst1.8     {q0}, [%0]!                    \n"  // store 8 pixels RGB565.
+      "vqadd.u8   d22, d22, d2                   \n"  // add for dither
+      ARGBTORGB565
+      "vst1.8     {q0}, [%0]!                    \n"  // store 8 RGB565.
       "bgt        1b                             \n"
       : "+r"(dst_rgb)   // %0
@@ -1065 +1112 @@
                             int width) {
   asm volatile(
-      "1:                                          \n"
+      "1:                                        \n"
       "vld4.8     {d20, d21, d22, d23}, [%0]!    \n"  // load 8 pixels of ARGB.
       "subs       %2, %2, #8                     \n"  // 8 processed per loop.
       ARGBTOARGB1555
-      "vst1.8     {q0}, [%1]!                    \n"  // store 8 pixels
-                                                      // ARGB1555.
+      "vst1.8     {q0}, [%1]!                    \n"  // store 8 ARGB1555.
       "bgt        1b                             \n"
       : "+r"(src_argb),      // %0
@@ -1085 +1131 @@
       "vmov.u8    d4, #0x0f                      \n"  // bits to clear with
                                                       // vbic.
-      "1:                                          \n"
+      "1:                                        \n"
       "vld4.8     {d20, d21, d22, d23}, [%0]!    \n"  // load 8 pixels of ARGB.
       "subs       %2, %2, #8                     \n"  // 8 processed per loop.
       ARGBTOARGB4444
-      "vst1.8     {q0}, [%1]!                    \n"  // store 8 pixels
-                                                      // ARGB4444.
+      "vst1.8     {q0}, [%1]!                    \n"  // store 8 ARGB4444.
       "bgt        1b                             \n"
       : "+r"(src_argb),      // %0
@@ -1105 +1150 @@
       "vmov.u8    d26, #33                       \n"  // R * 0.2578 coefficient
       "vmov.u8    d27, #16                       \n"  // Add 16 constant
-      "1:                                          \n"
+      "1:                                        \n"
       "vld4.8     {d0, d1, d2, d3}, [%0]!        \n"  // load 8 ARGB pixels.
       "subs       %2, %2, #8                     \n"  // 8 processed per loop.
@@ -1124 +1169 @@
 void ARGBExtractAlphaRow_NEON(const uint8* src_argb, uint8* dst_a, int width) {
   asm volatile(
-      "1:                                          \n"
+      "1:                                        \n"
       "vld4.8     {d0, d2, d4, d6}, [%0]!        \n"  // load 8 ARGB pixels
       "vld4.8     {d1, d3, d5, d7}, [%0]!        \n"  // load next 8 ARGB pixels
@@ -1143 +1188 @@
       "vmov.u8    d25, #75                       \n"  // G * 0.58700 coefficient
       "vmov.u8    d26, #38                       \n"  // R * 0.29900 coefficient
-      "1:                                          \n"
+      "1:                                        \n"
       "vld4.8     {d0, d1, d2, d3}, [%0]!        \n"  // load 8 ARGB pixels.
       "subs       %2, %2, #8                     \n"  // 8 processed per loop.
@@ -1172 +1217 @@
       "vmov.u8    d28, #94                       \n"  // VG -0.7344 coefficient
       "vmov.u16   q15, #0x8080                   \n"  // 128.5
-      "1:                                          \n"
+      "1:                                        \n"
       "vld4.8     {d0, d1, d2, d3}, [%0]!        \n"  // load 8 ARGB pixels.
       "subs       %3, %3, #8                     \n"  // 8 processed per loop.
@@ -1200 +1245 @@
 }
 
+// clang-format off
 // 16x2 pixels -> 8x1.  width is number of argb pixels. e.g. 16.
 #define RGBTOUV(QB, QG, QR)                                                 \
-  "vmul.s16   q8, " #QB                                                     \
-  ", q10               \n" /* B                    */                       \
-  "vmls.s16   q8, " #QG                                                     \
-  ", q11               \n" /* G                    */                       \
-  "vmls.s16   q8, " #QR                                                     \
-  ", q12               \n"                       /* R                    */ \
+  "vmul.s16   q8, " #QB ", q10               \n" /* B                    */ \
+  "vmls.s16   q8, " #QG ", q11               \n" /* G                    */ \
+  "vmls.s16   q8, " #QR ", q12               \n" /* R                    */ \
   "vadd.u16   q8, q8, q15                    \n" /* +128 -> unsigned     */ \
-  "vmul.s16   q9, " #QR                                                     \
-  ", q10               \n" /* R                    */                       \
-  "vmls.s16   q9, " #QG                                                     \
-  ", q14               \n" /* G                    */                       \
-  "vmls.s16   q9, " #QB                                                     \
-  ", q13               \n"                       /* B                    */ \
+  "vmul.s16   q9, " #QR ", q10               \n" /* R                    */ \
+  "vmls.s16   q9, " #QG ", q14               \n" /* G                    */ \
+  "vmls.s16   q9, " #QB ", q13               \n" /* B                    */ \
   "vadd.u16   q9, q9, q15                    \n" /* +128 -> unsigned     */ \
   "vqshrn.u16  d0, q8, #8                    \n" /* 16 bit to 8 bit U    */ \
   "vqshrn.u16  d1, q9, #8                    \n" /* 16 bit to 8 bit V    */
+// clang-format on
 
 // TODO(fbarchard): Consider vhadd vertical, then vpaddl horizontal, avoid shr.
@@ -1233 +1274 @@
     "vmov.s16   q14, #94 / 2                   \n"  // VG -0.7344 coefficient
     "vmov.u16   q15, #0x8080                   \n"  // 128.5
-  "1:                                          \n"
+    "1:                                        \n"
     "vld4.8     {d0, d2, d4, d6}, [%0]!        \n"  // load 8 ARGB pixels.
     "vld4.8     {d1, d3, d5, d7}, [%0]!        \n"  // load next 8 ARGB pixels.
@@ -1279 +1320 @@
     "vmov.s16   q14, #107 / 2                  \n"  // VG -0.41869 coefficient
     "vmov.u16   q15, #0x8080                   \n"  // 128.5
-  "1:                                          \n"
+    "1:                                        \n"
     "vld4.8     {d0, d2, d4, d6}, [%0]!        \n"  // load 8 ARGB pixels.
     "vld4.8     {d1, d3, d5, d7}, [%0]!        \n"  // load next 8 ARGB pixels.
@@ -1324 +1365 @@
     "vmov.s16   q14, #94 / 2                   \n"  // VG -0.7344 coefficient
     "vmov.u16   q15, #0x8080                   \n"  // 128.5
-  "1:                                          \n"
+    "1:                                        \n"
     "vld4.8     {d0, d2, d4, d6}, [%0]!        \n"  // load 8 BGRA pixels.
     "vld4.8     {d1, d3, d5, d7}, [%0]!        \n"  // load next 8 BGRA pixels.
@@ -1369 +1410 @@
     "vmov.s16   q14, #94 / 2                   \n"  // VG -0.7344 coefficient
     "vmov.u16   q15, #0x8080                   \n"  // 128.5
-  "1:                                          \n"
+    "1:                                        \n"
     "vld4.8     {d0, d2, d4, d6}, [%0]!        \n"  // load 8 ABGR pixels.
     "vld4.8     {d1, d3, d5, d7}, [%0]!        \n"  // load next 8 ABGR pixels.
@@ -1414 +1455 @@
     "vmov.s16   q14, #94 / 2                   \n"  // VG -0.7344 coefficient
     "vmov.u16   q15, #0x8080                   \n"  // 128.5
-  "1:                                          \n"
+    "1:                                        \n"
     "vld4.8     {d0, d2, d4, d6}, [%0]!        \n"  // load 8 RGBA pixels.
     "vld4.8     {d1, d3, d5, d7}, [%0]!        \n"  // load next 8 RGBA pixels.
@@ -1459 +1500 @@
     "vmov.s16   q14, #94 / 2                   \n"  // VG -0.7344 coefficient
     "vmov.u16   q15, #0x8080                   \n"  // 128.5
-  "1:                                          \n"
+    "1:                                        \n"
     "vld3.8     {d0, d2, d4}, [%0]!            \n"  // load 8 RGB24 pixels.
     "vld3.8     {d1, d3, d5}, [%0]!            \n"  // load next 8 RGB24 pixels.
@@ -1504 +1545 @@
     "vmov.s16   q14, #94 / 2                   \n"  // VG -0.7344 coefficient
     "vmov.u16   q15, #0x8080                   \n"  // 128.5
-  "1:                                          \n"
+    "1:                                        \n"
     "vld3.8     {d0, d2, d4}, [%0]!            \n"  // load 8 RAW pixels.
     "vld3.8     {d1, d3, d5}, [%0]!            \n"  // load next 8 RAW pixels.
@@ -1551 +1592 @@
       "vmov.s16   q14, #94 / 2                   \n"  // VG -0.7344 coefficient
       "vmov.u16   q15, #0x8080                   \n"  // 128.5
-      "1:                                          \n"
+      "1:                                        \n"
       "vld1.8     {q0}, [%0]!                    \n"  // load 8 RGB565 pixels.
       RGB565TOARGB
@@ -1617 +1658 @@
       "vmov.s16   q14, #94 / 2                   \n"  // VG -0.7344 coefficient
       "vmov.u16   q15, #0x8080                   \n"  // 128.5
-      "1:                                          \n"
+      "1:                                        \n"
       "vld1.8     {q0}, [%0]!                    \n"  // load 8 ARGB1555 pixels.
       RGB555TOARGB
@@ -1683 +1724 @@
       "vmov.s16   q14, #94 / 2                   \n"  // VG -0.7344 coefficient
       "vmov.u16   q15, #0x8080                   \n"  // 128.5
-      "1:                                          \n"
+      "1:                                        \n"
       "vld1.8     {q0}, [%0]!                    \n"  // load 8 ARGB4444 pixels.
       ARGB4444TOARGB
@@ -1740 +1781 @@
       "vmov.u8    d26, #33                       \n"  // R * 0.2578 coefficient
       "vmov.u8    d27, #16                       \n"  // Add 16 constant
-      "1:                                          \n"
+      "1:                                        \n"
       "vld1.8     {q0}, [%0]!                    \n"  // load 8 RGB565 pixels.
       "subs       %2, %2, #8                     \n"  // 8 processed per loop.
@@ -1764 +1805 @@
       "vmov.u8    d26, #33                       \n"  // R * 0.2578 coefficient
       "vmov.u8    d27, #16                       \n"  // Add 16 constant
-      "1:                                          \n"
+      "1:                                        \n"
       "vld1.8     {q0}, [%0]!                    \n"  // load 8 ARGB1555 pixels.
       "subs       %2, %2, #8                     \n"  // 8 processed per loop.
@@ -1788 +1829 @@
       "vmov.u8    d26, #33                       \n"  // R * 0.2578 coefficient
       "vmov.u8    d27, #16                       \n"  // Add 16 constant
-      "1:                                          \n"
+      "1:                                        \n"
       "vld1.8     {q0}, [%0]!                    \n"  // load 8 ARGB4444 pixels.
       "subs       %2, %2, #8                     \n"  // 8 processed per loop.
@@ -1812 +1853 @@
       "vmov.u8    d6, #13                        \n"  // B * 0.1016 coefficient
       "vmov.u8    d7, #16                        \n"  // Add 16 constant
-      "1:                                          \n"
+      "1:                                        \n"
       "vld4.8     {d0, d1, d2, d3}, [%0]!        \n"  // load 8 pixels of BGRA.
       "subs       %2, %2, #8                     \n"  // 8 processed per loop.
@@ -1835 +1876 @@
       "vmov.u8    d6, #13                        \n"  // B * 0.1016 coefficient
       "vmov.u8    d7, #16                        \n"  // Add 16 constant
-      "1:                                          \n"
+      "1:                                        \n"
       "vld4.8     {d0, d1, d2, d3}, [%0]!        \n"  // load 8 pixels of ABGR.
       "subs       %2, %2, #8                     \n"  // 8 processed per loop.
@@ -1858 +1899 @@
       "vmov.u8    d6, #33                        \n"  // R * 0.2578 coefficient
       "vmov.u8    d7, #16                        \n"  // Add 16 constant
-      "1:                                          \n"
+      "1:                                        \n"
       "vld4.8     {d0, d1, d2, d3}, [%0]!        \n"  // load 8 pixels of RGBA.
       "subs       %2, %2, #8                     \n"  // 8 processed per loop.
@@ -1881 +1922 @@
       "vmov.u8    d6, #33                        \n"  // R * 0.2578 coefficient
       "vmov.u8    d7, #16                        \n"  // Add 16 constant
-      "1:                                          \n"
+      "1:                                        \n"
       "vld3.8     {d0, d1, d2}, [%0]!            \n"  // load 8 pixels of RGB24.
       "subs       %2, %2, #8                     \n"  // 8 processed per loop.
@@ -1904 +1945 @@
       "vmov.u8    d6, #13                        \n"  // B * 0.1016 coefficient
       "vmov.u8    d7, #16                        \n"  // Add 16 constant
-      "1:                                          \n"
+      "1:                                        \n"
       "vld3.8     {d0, d1, d2}, [%0]!            \n"  // load 8 pixels of RAW.
       "subs       %2, %2, #8                     \n"  // 8 processed per loop.
@@ -1939 +1980 @@
       "vdup.8     d4, %4                         \n"
       // General purpose row blend.
-      "1:                                          \n"
+      "1:                                        \n"
       "vld1.8     {q0}, [%1]!                    \n"
       "vld1.8     {q1}, [%2]!                    \n"
@@ -1954 +1995 @@
 
       // Blend 50 / 50.
-      "50:                                         \n"
+      "50:                                       \n"
       "vld1.8     {q0}, [%1]!                    \n"
       "vld1.8     {q1}, [%2]!                    \n"
@@ -1964 +2005 @@
 
       // Blend 100 / 0 - Copy row unchanged.
-      "100:                                        \n"
+      "100:                                      \n"
       "vld1.8     {q0}, [%1]!                    \n"
       "subs       %3, %3, #16                    \n"
@@ -1970 +2011 @@
       "bgt        100b                           \n"
 
-      "99:                                         \n"
+      "99:                                       \n"
       : "+r"(dst_ptr),     // %0
         "+r"(src_ptr),     // %1
@@ -1989 +2030 @@
       "blt        89f                            \n"
       // Blend 8 pixels.
-      "8:                                          \n"
+      "8:                                        \n"
       "vld4.8     {d0, d1, d2, d3}, [%0]!        \n"  // load 8 pixels of ARGB0.
       "vld4.8     {d4, d5, d6, d7}, [%1]!        \n"  // load 8 pixels of ARGB1.
@@ -2007 +2048 @@
       "bge        8b                             \n"
 
-      "89:                                         \n"
+      "89:                                       \n"
       "adds       %3, #8-1                       \n"
       "blt        99f                            \n"
 
       // Blend 1 pixels.
-      "1:                                          \n"
+      "1:                                        \n"
       "vld4.8     {d0[0],d1[0],d2[0],d3[0]}, [%0]! \n"  // load 1 pixel ARGB0.
       "vld4.8     {d4[0],d5[0],d6[0],d7[0]}, [%1]! \n"  // load 1 pixel ARGB1.
@@ -2044 +2085 @@
   asm volatile(
       // Attenuate 8 pixels.
-      "1:                                          \n"
+      "1:                                        \n"
       "vld4.8     {d0, d1, d2, d3}, [%0]!        \n"  // load 8 pixels of ARGB.
       "subs       %2, %2, #8                     \n"  // 8 processed per loop.
@@ -2076 +2117 @@
 
       // 8 pixel loop.
-      "1:                                          \n"
+      "1:                                        \n"
       "vld4.8     {d0, d2, d4, d6}, [%0]         \n"  // load 8 pixels of ARGB.
       "subs       %1, %1, #8                     \n"  // 8 processed per loop.
@@ -2117 +2158 @@
 
       // 8 pixel loop.
-      "1:                                          \n"
+      "1:                                        \n"
       "vld4.8     {d20, d22, d24, d26}, [%0]!    \n"  // load 8 pixels of ARGB.
       "subs       %2, %2, #8                     \n"  // 8 processed per loop.
@@ -2149 +2190 @@
       "vmov.u8    d25, #75                       \n"  // G * 0.58700 coefficient
       "vmov.u8    d26, #38                       \n"  // R * 0.29900 coefficient
-      "1:                                          \n"
+      "1:                                        \n"
       "vld4.8     {d0, d1, d2, d3}, [%0]!        \n"  // load 8 ARGB pixels.
       "subs       %2, %2, #8                     \n"  // 8 processed per loop.
@@ -2182 +2223 @@
       "vmov.u8    d29, #98                       \n"  // BG coefficient
       "vmov.u8    d30, #50                       \n"  // BR coefficient
-      "1:                                          \n"
+      "1:                                        \n"
       "vld4.8     {d0, d1, d2, d3}, [%0]         \n"  // load 8 ARGB pixels.
       "subs       %1, %1, #8                     \n"  // 8 processed per loop.
@@ -2218 +2259 @@
       "vmovl.s8   q1, d5                         \n"  // R,A coefficients s16.
 
-      "1:                                          \n"
+      "1:                                        \n"
       "vld4.8     {d16, d18, d20, d22}, [%0]!    \n"  // load 8 ARGB pixels.
       "subs       %2, %2, #8                     \n"  // 8 processed per loop.
@@ -2274 +2315 @@
   asm volatile(
       // 8 pixel loop.
-      "1:                                          \n"
+      "1:                                        \n"
       "vld4.8     {d0, d2, d4, d6}, [%0]!        \n"  // load 8 ARGB pixels.
       "vld4.8     {d1, d3, d5, d7}, [%1]!        \n"  // load 8 more ARGB
-                                                      // pixels.
       "subs       %3, %3, #8                     \n"  // 8 processed per loop.
       "vmull.u8   q0, d0, d1                     \n"  // multiply B
@@ -2289 +2329 @@
       "vst4.8     {d0, d1, d2, d3}, [%2]!        \n"  // store 8 ARGB pixels.
       "bgt        1b                             \n"
-
       : "+r"(src_argb0),  // %0
         "+r"(src_argb1),  // %1
@@ -2305 +2344 @@
   asm volatile(
       // 8 pixel loop.
-      "1:                                          \n"
+      "1:                                        \n"
       "vld4.8     {d0, d1, d2, d3}, [%0]!        \n"  // load 8 ARGB pixels.
       "vld4.8     {d4, d5, d6, d7}, [%1]!        \n"  // load 8 more ARGB
-                                                      // pixels.
       "subs       %3, %3, #8                     \n"  // 8 processed per loop.
       "vqadd.u8   q0, q0, q2                     \n"  // add B, G
@@ -2314 +2352 @@
       "vst4.8     {d0, d1, d2, d3}, [%2]!        \n"  // store 8 ARGB pixels.
       "bgt        1b                             \n"
-
       : "+r"(src_argb0),  // %0
         "+r"(src_argb1),  // %1
@@ -2330 +2367 @@
   asm volatile(
       // 8 pixel loop.
-      "1:                                          \n"
+      "1:                                        \n"
       "vld4.8     {d0, d1, d2, d3}, [%0]!        \n"  // load 8 ARGB pixels.
       "vld4.8     {d4, d5, d6, d7}, [%1]!        \n"  // load 8 more ARGB
-                                                      // pixels.
       "subs       %3, %3, #8                     \n"  // 8 processed per loop.
       "vqsub.u8   q0, q0, q2                     \n"  // subtract B, G
@@ -2339 +2375 @@
       "vst4.8     {d0, d1, d2, d3}, [%2]!        \n"  // store 8 ARGB pixels.
       "bgt        1b                             \n"
-
       : "+r"(src_argb0),  // %0
         "+r"(src_argb1),  // %1
@@ -2360 +2395 @@
       "vmov.u8    d3, #255                       \n"  // alpha
       // 8 pixel loop.
-      "1:                                          \n"
+      "1:                                        \n"
       "vld1.8     {d0}, [%0]!                    \n"  // load 8 sobelx.
       "vld1.8     {d1}, [%1]!                    \n"  // load 8 sobely.
@@ -2384 +2419 @@
   asm volatile(
       // 16 pixel loop.
-      "1:                                          \n"
+      "1:                                        \n"
       "vld1.8     {q0}, [%0]!                    \n"  // load 16 sobelx.
       "vld1.8     {q1}, [%1]!                    \n"  // load 16 sobely.
@@ -2411 +2446 @@
       "vmov.u8    d3, #255                       \n"  // alpha
       // 8 pixel loop.
-      "1:                                          \n"
+      "1:                                        \n"
       "vld1.8     {d2}, [%0]!                    \n"  // load 8 sobelx.
       "vld1.8     {d0}, [%1]!                    \n"  // load 8 sobely.
@@ -2436 +2471 @@
                     int width) {
   asm volatile(
-      "1:                                          \n"
+      "1:                                        \n"
       "vld1.8     {d0}, [%0],%5                  \n"  // top
       "vld1.8     {d1}, [%0],%6                  \n"
@@ -2474 +2509 @@
                     int width) {
   asm volatile(
-      "1:                                          \n"
+      "1:                                        \n"
       "vld1.8     {d0}, [%0],%4                  \n"  // left
       "vld1.8     {d1}, [%1],%4                  \n"
@@ -2506 +2541 @@
       "vdup.32    q0, %3                         \n"
 
-      "1:                                          \n"
+      "1:                                        \n"
       "vld1.8     {q1}, [%0]!                    \n"  // load 8 shorts
       "subs       %2, %2, #8                     \n"  // 8 pixels per loop
@@ -2531 +2566 @@
       "vdup.32    q0, %3                         \n"
 
-      "1:                                          \n"
+      "1:                                        \n"
       "vld1.8     {q1}, [%0]!                    \n"  // load 8 shorts
       "subs       %2, %2, #8                     \n"  // 8 pixels per loop

pjproject/trunk/third_party/yuv/source/row_neon64.cc

@@ -274 +274 @@
   asm volatile(
       YUVTORGB_SETUP
-      "1:                                          \n" READYUV422 YUVTORGB(
+      "1:                                        \n" READYUV422 YUVTORGB(
           v22, v21,
           v20) "subs       %w4, %w4, #8                   \n" ARGBTORGB565
@@ -311 +311 @@
       YUVTORGB_SETUP
       "movi       v23.8b, #255                   \n"
-      "1:                                          \n" READYUV422 YUVTORGB(
+      "1:                                        \n" READYUV422 YUVTORGB(
           v22, v21,
           v20) "subs       %w4, %w4, #8                   \n" ARGBTOARGB1555
@@ -396 +396 @@
   asm volatile(
       "movi       v23.8b, #255                   \n"
-      "1:                                          \n"
+      "1:                                        \n"
       "ld1        {v20.8b}, [%0], #8             \n"
       "orr        v21.8b, v20.8b, v20.8b         \n"
@@ -471 +471 @@
   asm volatile(
       YUVTORGB_SETUP
-      "1:                                          \n" READNV12 YUVTORGB(
+      "1:                                        \n" READNV12 YUVTORGB(
           v22, v21,
           v20) "subs       %w3, %w3, #8                   \n" ARGBTORGB565
@@ -545 +545 @@
                      int width) {
   asm volatile(
-      "1:                                          \n"
+      "1:                                        \n"
       "ld2        {v0.16b,v1.16b}, [%0], #32     \n"  // load 16 pairs of UV
       "subs       %w3, %w3, #16                  \n"  // 16 processed per loop
@@ -566 +566 @@
                      int width) {
   asm volatile(
-      "1:                                          \n"
+      "1:                                        \n"
       "ld1        {v0.16b}, [%0], #16            \n"  // load U
       "ld1        {v1.16b}, [%1], #16            \n"  // load V
@@ -581 +581 @@
 }
 
-// Copy multiple of 32.  vld4.8  allow unaligned and is fastest on a15.
+// Reads 16 packed RGB and write to planar dst_r, dst_g, dst_b.
+void SplitRGBRow_NEON(const uint8* src_rgb,
+                      uint8* dst_r,
+                      uint8* dst_g,
+                      uint8* dst_b,
+                      int width) {
+  asm volatile(
+      "1:                                        \n"
+      "ld3        {v0.16b,v1.16b,v2.16b}, [%0], #48 \n"  // load 16 RGB
+      "subs       %w4, %w4, #16                  \n"  // 16 processed per loop
+      "st1        {v0.16b}, [%1], #16            \n"  // store R
+      "st1        {v1.16b}, [%2], #16            \n"  // store G
+      "st1        {v2.16b}, [%3], #16            \n"  // store B
+      "b.gt       1b                             \n"
+      : "+r"(src_rgb),                    // %0
+        "+r"(dst_r),                      // %1
+        "+r"(dst_g),                      // %2
+        "+r"(dst_b),                      // %3
+        "+r"(width)                       // %4
+      :                                   // Input registers
+      : "cc", "memory", "v0", "v1", "v2"  // Clobber List
+      );
+}
+
+// Reads 16 planar R's, G's and B's and writes out 16 packed RGB at a time
+void MergeRGBRow_NEON(const uint8* src_r,
+                      const uint8* src_g,
+                      const uint8* src_b,
+                      uint8* dst_rgb,
+                      int width) {
+  asm volatile(
+      "1:                                        \n"
+      "ld1        {v0.16b}, [%0], #16            \n"  // load R
+      "ld1        {v1.16b}, [%1], #16            \n"  // load G
+      "ld1        {v2.16b}, [%2], #16            \n"  // load B
+      "subs       %w4, %w4, #16                  \n"  // 16 processed per loop
+      "st3        {v0.16b,v1.16b,v2.16b}, [%3], #48 \n"  // store 16 RGB
+      "b.gt       1b                             \n"
+      : "+r"(src_r),                      // %0
+        "+r"(src_g),                      // %1
+        "+r"(src_b),                      // %2
+        "+r"(dst_rgb),                    // %3
+        "+r"(width)                       // %4
+      :                                   // Input registers
+      : "cc", "memory", "v0", "v1", "v2"  // Clobber List
+      );
+}
+
+// Copy multiple of 32.
 void CopyRow_NEON(const uint8* src, uint8* dst, int count) {
   asm volatile(
-      "1:                                          \n"
-      "ld1        {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32       \n"  // load 32
+      "1:                                        \n"
+      "ldp        q0, q1, [%0], #32              \n"
       "subs       %w2, %w2, #32                  \n"  // 32 processed per loop
-      "st1        {v0.8b,v1.8b,v2.8b,v3.8b}, [%1], #32       \n"  // store 32
-      "b.gt       1b                             \n"
-      : "+r"(src),                              // %0
-        "+r"(dst),                              // %1
-        "+r"(count)                             // %2  // Output registers
-      :                                         // Input registers
-      : "cc", "memory", "v0", "v1", "v2", "v3"  // Clobber List
+      "stp        q0, q1, [%1], #32              \n"
+      "b.gt       1b                             \n"
+      : "+r"(src),                  // %0
+        "+r"(dst),                  // %1
+        "+r"(count)                 // %2  // Output registers
+      :                             // Input registers
+      : "cc", "memory", "v0", "v1"  // Clobber List
       );
 }
@@ -601 +649 @@
   asm volatile(
       "dup        v0.16b, %w2                    \n"  // duplicate 16 bytes
-      "1:                                          \n"
+      "1:                                        \n"
       "subs       %w1, %w1, #16                  \n"  // 16 bytes per loop
       "st1        {v0.16b}, [%0], #16            \n"  // store
@@ -614 +662 @@
   asm volatile(
       "dup        v0.4s, %w2                     \n"  // duplicate 4 ints
-      "1:                                          \n"
+      "1:                                        \n"
       "subs       %w1, %w1, #4                   \n"  // 4 ints per loop
       "st1        {v0.16b}, [%0], #16            \n"  // store
@@ -629 +677 @@
       "add        %0, %0, %w2, sxtw              \n"
       "sub        %0, %0, #16                    \n"
-      "1:                                          \n"
+      "1:                                        \n"
       "ld1        {v0.16b}, [%0], %3             \n"  // src -= 16
       "subs       %w2, %w2, #16                  \n"  // 16 pixels per loop.
@@ -651 +699 @@
       "add        %0, %0, %w3, sxtw #1           \n"
       "sub        %0, %0, #16                    \n"
-      "1:                                          \n"
+      "1:                                        \n"
       "ld2        {v0.8b, v1.8b}, [%0], %4       \n"  // src -= 16
       "subs       %w3, %w3, #8                   \n"  // 8 pixels per loop.
@@ -672 +720 @@
       "add        %0, %0, %w2, sxtw #2           \n"
       "sub        %0, %0, #16                    \n"
-      "1:                                          \n"
+      "1:                                        \n"
       "ld1        {v0.16b}, [%0], %3             \n"  // src -= 16
       "subs       %w2, %w2, #4                   \n"  // 4 pixels per loop.
@@ -689 +737 @@
   asm volatile(
       "movi       v4.8b, #255                    \n"  // Alpha
-      "1:                                          \n"
+      "1:                                        \n"
       "ld3        {v1.8b,v2.8b,v3.8b}, [%0], #24 \n"  // load 8 pixels of RGB24.
       "subs       %w2, %w2, #8                   \n"  // 8 processed per loop.
       "st4        {v1.8b,v2.8b,v3.8b,v4.8b}, [%1], #32 \n"  // store 8 ARGB
-                                                            // pixels
       "b.gt       1b                             \n"
       : "+r"(src_rgb24),  // %0
@@ -706 +753 @@
   asm volatile(
       "movi       v5.8b, #255                    \n"  // Alpha
-      "1:                                          \n"
+      "1:                                        \n"
       "ld3        {v0.8b,v1.8b,v2.8b}, [%0], #24 \n"  // read r g b
       "subs       %w2, %w2, #8                   \n"  // 8 processed per loop.
@@ -723 +770 @@
 void RAWToRGB24Row_NEON(const uint8* src_raw, uint8* dst_rgb24, int width) {
   asm volatile(
-      "1:                                          \n"
+      "1:                                        \n"
       "ld3        {v0.8b,v1.8b,v2.8b}, [%0], #24 \n"  // read r g b
       "subs       %w2, %w2, #8                   \n"  // 8 processed per loop.
@@ -754 +801 @@
   asm volatile(
       "movi       v3.8b, #255                    \n"  // Alpha
-      "1:                                          \n"
+      "1:                                        \n"
       "ld1        {v0.16b}, [%0], #16            \n"  // load 8 RGB565 pixels.
       "subs       %w2, %w2, #8                   \n"  // 8 processed per loop.
       RGB565TOARGB
       "st4        {v0.8b,v1.8b,v2.8b,v3.8b}, [%1], #32 \n"  // store 8 ARGB
-                                                            // pixels
       "b.gt       1b                             \n"
       : "+r"(src_rgb565),  // %0
@@ -811 +857 @@
   asm volatile(
       "movi       v3.8b, #255                    \n"  // Alpha
-      "1:                                          \n"
+      "1:                                        \n"
       "ld1        {v0.16b}, [%0], #16            \n"  // load 8 ARGB1555 pixels.
       "subs       %w2, %w2, #8                   \n"  // 8 processed per loop.
@@ -842 +888 @@
                             int width) {
   asm volatile