target-arm: integrate upstream ARM translator.

The new translator has the following benefits:

- faster emulation of ARMv5TE code (through improved JIT)
- proper support for ARMv7 and NEON
- rebuilding the full-eng platform images for ARMv7-A results
  in additionnal speed increases (a.k.a. Thumb-2 rocks!).

Note that, as an interesting side effect, NEON machine code is generally
slower than the equivalent C code it is supposed to replace when run inside
the emulator. This can be explained by the fact that for now the translator
simply translates each NEON instruction into a series of sequential host
instructions (and also requires over-head for packing/unpacking/saturation/
etc...).

This change has been tested by running the "full-eng" platform image
rebuilt for ARMv7-A and Neon and using an appropriate kernel image
(prebuilt/android-arm/kernel/kernel-qemu-armv7). The system could boot
and seems to work perfectly. Not a single issue has been experienced
during testing. On a 2.4 GHz Xeon CPU, the image boots in about 25 seconds
(compared to 40 seconds for a vanilla one without this emulator patch).

Thanks to Peter Maydell at Linaro and ARM with his hard work to make this
happen (first in upstream, and now on Android).

This integration is based on the Meego git repository
(git://gitorious.org/qemu-maemo/qemu.git) using the following hash:

    7e2d65b0c95c865b1fa6d3d4948e8e822b9ac2fd

On top of which, the following upstream patch has been applied
(with recommendation from Peter):

    b7fa9214d8d4f57992c9acd0ccb125c54a095f00

(We chose this repository because it was the closest to the previous
integrate. We will probably use the Linaro ones for future work on this
part of the emulator).

Change-Id: I54837e3d2e908b2380d158411d7a9813630e7e4e

1 files changed, 736 insertions, 171 deletions

diff --git a/target-arm/neon_helper.c b/target-arm/neon_helper.c
index f32ecd6..9165519 100644
--- a/target-arm/neon_helper.c
+++ b/target-arm/neon_helper.c
@@ -10,39 +10,15 @@
 #include <stdio.h>
 
 #include "cpu.h"
-#include "exec-all.h"
-#include "helpers.h"
+#include "exec.h"
+#include "helper.h"
 
 #define SIGNBIT (uint32_t)0x80000000
 #define SIGNBIT64 ((uint64_t)1 << 63)
 
 #define SET_QC() env->vfp.xregs[ARM_VFP_FPSCR] = CPSR_Q
 
-static float_status neon_float_status;
-#define NFS &neon_float_status
-
-/* Helper routines to perform bitwise copies between float and int.  */
-static inline float32 vfp_itos(uint32_t i)
-{
-    union {
-        uint32_t i;
-        float32 s;
-    } v;
-
-    v.i = i;
-    return v.s;
-}
-
-static inline uint32_t vfp_stoi(float32 s)
-{
-    union {
-        uint32_t i;
-        float32 s;
-    } v;
-
-    v.s = s;
-    return v.i;
-}
+#define NFS (&env->vfp.standard_fp_status)
 
 #define NEON_TYPE1(name, type) \
 typedef struct \
@@ -139,10 +115,6 @@ NEON_TYPE1(u32, uint32_t)
 uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
 NEON_VOP_BODY(vtype, n)
 
-#define NEON_VOP_ENV(name, vtype, n) \
-uint32_t HELPER(glue(neon_,name))(CPUState *env, uint32_t arg1, uint32_t arg2) \
-NEON_VOP_BODY(vtype, n)
-
 /* Pairwise operations.  */
 /* For 32-bit elements each segment only contains a single element, so
    the elementwise and pairwise operations are the same.  */
@@ -191,13 +163,35 @@ uint32_t HELPER(glue(neon_,name))(uint32_t arg) \
         dest = tmp; \
     }} while(0)
 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
-NEON_VOP_ENV(qadd_u8, neon_u8, 4)
+NEON_VOP(qadd_u8, neon_u8, 4)
 #undef NEON_FN
 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
-NEON_VOP_ENV(qadd_u16, neon_u16, 2)
+NEON_VOP(qadd_u16, neon_u16, 2)
 #undef NEON_FN
 #undef NEON_USAT
 
+uint32_t HELPER(neon_qadd_u32)(uint32_t a, uint32_t b)
+{
+    uint32_t res = a + b;
+    if (res < a) {
+        SET_QC();
+        res = ~0;
+    }
+    return res;
+}
+
+uint64_t HELPER(neon_qadd_u64)(uint64_t src1, uint64_t src2)
+{
+    uint64_t res;
+
+    res = src1 + src2;
+    if (res < src1) {
+        SET_QC();
+        res = ~(uint64_t)0;
+    }
+    return res;
+}
+
 #define NEON_SSAT(dest, src1, src2, type) do { \
     int32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
     if (tmp != (type)tmp) { \
@@ -211,13 +205,35 @@ NEON_VOP_ENV(qadd_u16, neon_u16, 2)
     dest = tmp; \
     } while(0)
 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
-NEON_VOP_ENV(qadd_s8, neon_s8, 4)
+NEON_VOP(qadd_s8, neon_s8, 4)
 #undef NEON_FN
 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
-NEON_VOP_ENV(qadd_s16, neon_s16, 2)
+NEON_VOP(qadd_s16, neon_s16, 2)
 #undef NEON_FN
 #undef NEON_SSAT
 
+uint32_t HELPER(neon_qadd_s32)(uint32_t a, uint32_t b)
+{
+    uint32_t res = a + b;
+    if (((res ^ a) & SIGNBIT) && !((a ^ b) & SIGNBIT)) {
+        SET_QC();
+        res = ~(((int32_t)a >> 31) ^ SIGNBIT);
+    }
+    return res;
+}
+
+uint64_t HELPER(neon_qadd_s64)(uint64_t src1, uint64_t src2)
+{
+    uint64_t res;
+
+    res = src1 + src2;
+    if (((res ^ src1) & SIGNBIT64) && !((src1 ^ src2) & SIGNBIT64)) {
+        SET_QC();
+        res = ((int64_t)src1 >> 63) ^ ~SIGNBIT64;
+    }
+    return res;
+}
+
 #define NEON_USAT(dest, src1, src2, type) do { \
     uint32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
     if (tmp != (type)tmp) { \
@@ -227,13 +243,36 @@ NEON_VOP_ENV(qadd_s16, neon_s16, 2)
         dest = tmp; \
     }} while(0)
 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
-NEON_VOP_ENV(qsub_u8, neon_u8, 4)
+NEON_VOP(qsub_u8, neon_u8, 4)
 #undef NEON_FN
 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
-NEON_VOP_ENV(qsub_u16, neon_u16, 2)
+NEON_VOP(qsub_u16, neon_u16, 2)
 #undef NEON_FN
 #undef NEON_USAT
 
+uint32_t HELPER(neon_qsub_u32)(uint32_t a, uint32_t b)
+{
+    uint32_t res = a - b;
+    if (res > a) {
+        SET_QC();
+        res = 0;
+    }
+    return res;
+}
+
+uint64_t HELPER(neon_qsub_u64)(uint64_t src1, uint64_t src2)
+{
+    uint64_t res;
+
+    if (src1 < src2) {
+        SET_QC();
+        res = 0;
+    } else {
+        res = src1 - src2;
+    }
+    return res;
+}
+
 #define NEON_SSAT(dest, src1, src2, type) do { \
     int32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
     if (tmp != (type)tmp) { \
@@ -247,13 +286,35 @@ NEON_VOP_ENV(qsub_u16, neon_u16, 2)
     dest = tmp; \
     } while(0)
 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
-NEON_VOP_ENV(qsub_s8, neon_s8, 4)
+NEON_VOP(qsub_s8, neon_s8, 4)
 #undef NEON_FN
 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
-NEON_VOP_ENV(qsub_s16, neon_s16, 2)
+NEON_VOP(qsub_s16, neon_s16, 2)
 #undef NEON_FN
 #undef NEON_SSAT
 
+uint32_t HELPER(neon_qsub_s32)(uint32_t a, uint32_t b)
+{
+    uint32_t res = a - b;
+    if (((res ^ a) & SIGNBIT) && ((a ^ b) & SIGNBIT)) {
+        SET_QC();
+        res = ~(((int32_t)a >> 31) ^ SIGNBIT);
+    }
+    return res;
+}
+
+uint64_t HELPER(neon_qsub_s64)(uint64_t src1, uint64_t src2)
+{
+    uint64_t res;
+
+    res = src1 - src2;
+    if (((res ^ src1) & SIGNBIT64) && ((src1 ^ src2) & SIGNBIT64)) {
+        SET_QC();
+        res = ((int64_t)src1 >> 63) ^ ~SIGNBIT64;
+    }
+    return res;
+}
+
 #define NEON_FN(dest, src1, src2) dest = (src1 + src2) >> 1
 NEON_VOP(hadd_s8, neon_s8, 4)
 NEON_VOP(hadd_u8, neon_u8, 4)
@@ -392,7 +453,8 @@ NEON_VOP(abd_u32, neon_u32, 1)
 #define NEON_FN(dest, src1, src2) do { \
     int8_t tmp; \
     tmp = (int8_t)src2; \
-    if (tmp >= sizeof(src1) * 8 || tmp <= -sizeof(src1) * 8) { \
+    if (tmp >= (ssize_t)sizeof(src1) * 8 || \
+        tmp <= -(ssize_t)sizeof(src1) * 8) { \
         dest = 0; \
     } else if (tmp < 0) { \
         dest = src1 >> -tmp; \
@@ -420,9 +482,9 @@ uint64_t HELPER(neon_shl_u64)(uint64_t val, uint64_t shiftop)
 #define NEON_FN(dest, src1, src2) do { \
     int8_t tmp; \
     tmp = (int8_t)src2; \
-    if (tmp >= sizeof(src1) * 8) { \
+    if (tmp >= (ssize_t)sizeof(src1) * 8) { \
         dest = 0; \
-    } else if (tmp <= -sizeof(src1) * 8) { \
+    } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
         dest = src1 >> (sizeof(src1) * 8 - 1); \
     } else if (tmp < 0) { \
         dest = src1 >> -tmp; \
@@ -453,14 +515,9 @@ uint64_t HELPER(neon_shl_s64)(uint64_t valop, uint64_t shiftop)
 #define NEON_FN(dest, src1, src2) do { \
     int8_t tmp; \
     tmp = (int8_t)src2; \
-    if (tmp >= sizeof(src1) * 8) { \
+    if ((tmp >= (ssize_t)sizeof(src1) * 8) \
+        || (tmp <= -(ssize_t)sizeof(src1) * 8)) { \
         dest = 0; \
-    } else if (tmp < -sizeof(src1) * 8) { \
-        dest = src1 >> (sizeof(src1) * 8 - 1); \
-    } else if (tmp == -sizeof(src1) * 8) { \
-        dest = src1 >> (tmp - 1); \
-        dest++; \
-        dest >>= 1; \
     } else if (tmp < 0) { \
         dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
     } else { \
@@ -468,23 +525,45 @@ uint64_t HELPER(neon_shl_s64)(uint64_t valop, uint64_t shiftop)
     }} while (0)
 NEON_VOP(rshl_s8, neon_s8, 4)
 NEON_VOP(rshl_s16, neon_s16, 2)
-NEON_VOP(rshl_s32, neon_s32, 1)
 #undef NEON_FN
 
+/* The addition of the rounding constant may overflow, so we use an
+ * intermediate 64 bits accumulator.  */
+uint32_t HELPER(neon_rshl_s32)(uint32_t valop, uint32_t shiftop)
+{
+    int32_t dest;
+    int32_t val = (int32_t)valop;
+    int8_t shift = (int8_t)shiftop;
+    if ((shift >= 32) || (shift <= -32)) {
+        dest = 0;
+    } else if (shift < 0) {
+        int64_t big_dest = ((int64_t)val + (1 << (-1 - shift)));
+        dest = big_dest >> -shift;
+    } else {
+        dest = val << shift;
+    }
+    return dest;
+}
+
+/* Handling addition overflow with 64 bits inputs values is more
+ * tricky than with 32 bits values.  */
 uint64_t HELPER(neon_rshl_s64)(uint64_t valop, uint64_t shiftop)
 {
     int8_t shift = (int8_t)shiftop;
     int64_t val = valop;
-    if (shift >= 64) {
+    if ((shift >= 64) || (shift <= -64)) {
         val = 0;
-    } else if (shift < -64) {
-        val >>= 63;
-    } else if (shift == -63) {
-        val >>= 63;
-        val++;
-        val >>= 1;
     } else if (shift < 0) {
-        val = (val + ((int64_t)1 << (-1 - shift))) >> -shift;
+        val >>= (-shift - 1);
+        if (val == INT64_MAX) {
+            /* In this case, it means that the rounding constant is 1,
+             * and the addition would overflow. Return the actual
+             * result directly.  */
+            val = 0x4000000000000000LL;
+        } else {
+            val++;
+            val >>= 1;
+        }
     } else {
         val <<= shift;
     }
@@ -494,10 +573,11 @@ uint64_t HELPER(neon_rshl_s64)(uint64_t valop, uint64_t shiftop)
 #define NEON_FN(dest, src1, src2) do { \
     int8_t tmp; \
     tmp = (int8_t)src2; \
-    if (tmp >= sizeof(src1) * 8 || tmp < -sizeof(src1) * 8) { \
+    if (tmp >= (ssize_t)sizeof(src1) * 8 || \
+        tmp < -(ssize_t)sizeof(src1) * 8) { \
         dest = 0; \
-    } else if (tmp == -sizeof(src1) * 8) { \
-        dest = src1 >> (tmp - 1); \
+    } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
+        dest = src1 >> (-tmp - 1); \
     } else if (tmp < 0) { \
         dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
     } else { \
@@ -505,20 +585,48 @@ uint64_t HELPER(neon_rshl_s64)(uint64_t valop, uint64_t shiftop)
     }} while (0)
 NEON_VOP(rshl_u8, neon_u8, 4)
 NEON_VOP(rshl_u16, neon_u16, 2)
-NEON_VOP(rshl_u32, neon_u32, 1)
 #undef NEON_FN
 
+/* The addition of the rounding constant may overflow, so we use an
+ * intermediate 64 bits accumulator.  */
+uint32_t HELPER(neon_rshl_u32)(uint32_t val, uint32_t shiftop)
+{
+    uint32_t dest;
+    int8_t shift = (int8_t)shiftop;
+    if (shift >= 32 || shift < -32) {
+        dest = 0;
+    } else if (shift == -32) {
+        dest = val >> 31;
+    } else if (shift < 0) {
+        uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift)));
+        dest = big_dest >> -shift;
+    } else {
+        dest = val << shift;
+    }
+    return dest;
+}
+
+/* Handling addition overflow with 64 bits inputs values is more
+ * tricky than with 32 bits values.  */
 uint64_t HELPER(neon_rshl_u64)(uint64_t val, uint64_t shiftop)
 {
     int8_t shift = (uint8_t)shiftop;
-    if (shift >= 64 || shift < 64) {
+    if (shift >= 64 || shift < -64) {
         val = 0;
     } else if (shift == -64) {
         /* Rounding a 1-bit result just preserves that bit.  */
         val >>= 63;
-    } if (shift < 0) {
-        val = (val + ((uint64_t)1 << (-1 - shift))) >> -shift;
-        val >>= -shift;
+    } else if (shift < 0) {
+        val >>= (-shift - 1);
+        if (val == UINT64_MAX) {
+            /* In this case, it means that the rounding constant is 1,
+             * and the addition would overflow. Return the actual
+             * result directly.  */
+            val = 0x8000000000000000ULL;
+        } else {
+            val++;
+            val >>= 1;
+        }
     } else {
         val <<= shift;
     }
@@ -528,14 +636,14 @@ uint64_t HELPER(neon_rshl_u64)(uint64_t val, uint64_t shiftop)
 #define NEON_FN(dest, src1, src2) do { \
     int8_t tmp; \
     tmp = (int8_t)src2; \
-    if (tmp >= sizeof(src1) * 8) { \
+    if (tmp >= (ssize_t)sizeof(src1) * 8) { \
         if (src1) { \
             SET_QC(); \
             dest = ~0; \
         } else { \
             dest = 0; \
         } \
-    } else if (tmp <= -sizeof(src1) * 8) { \
+    } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
         dest = 0; \
     } else if (tmp < 0) { \
         dest = src1 >> -tmp; \
@@ -546,20 +654,18 @@ uint64_t HELPER(neon_rshl_u64)(uint64_t val, uint64_t shiftop)
             dest = ~0; \
         } \
     }} while (0)
-NEON_VOP_ENV(qshl_u8, neon_u8, 4)
-NEON_VOP_ENV(qshl_u16, neon_u16, 2)
-NEON_VOP_ENV(qshl_u32, neon_u32, 1)
+NEON_VOP(qshl_u8, neon_u8, 4)
+NEON_VOP(qshl_u16, neon_u16, 2)
+NEON_VOP(qshl_u32, neon_u32, 1)
 #undef NEON_FN
 
-uint64_t HELPER(neon_qshl_u64)(CPUState *env, uint64_t val, uint64_t shiftop)
+uint64_t HELPER(neon_qshl_u64)(uint64_t val, uint64_t shiftop)
 {
     int8_t shift = (int8_t)shiftop;
     if (shift >= 64) {
         if (val) {
             val = ~(uint64_t)0;
             SET_QC();
-        } else {
-            val = 0;
         }
     } else if (shift <= -64) {
         val = 0;
@@ -579,11 +685,17 @@ uint64_t HELPER(neon_qshl_u64)(CPUState *env, uint64_t val, uint64_t shiftop)
 #define NEON_FN(dest, src1, src2) do { \
     int8_t tmp; \
     tmp = (int8_t)src2; \
-    if (tmp >= sizeof(src1) * 8) { \
-        if (src1) \
+    if (tmp >= (ssize_t)sizeof(src1) * 8) { \
+        if (src1) { \
             SET_QC(); \
-        dest = src1 >> 31; \
-    } else if (tmp <= -sizeof(src1) * 8) { \
+            dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
+            if (src1 > 0) { \
+                dest--; \
+            } \
+        } else { \
+            dest = src1; \
+        } \
+    } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
         dest = src1 >> 31; \
     } else if (tmp < 0) { \
         dest = src1 >> -tmp; \
@@ -591,24 +703,27 @@ uint64_t HELPER(neon_qshl_u64)(CPUState *env, uint64_t val, uint64_t shiftop)
         dest = src1 << tmp; \
         if ((dest >> tmp) != src1) { \
             SET_QC(); \
-            dest = src2 >> 31; \
+            dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
+            if (src1 > 0) { \
+                dest--; \
+            } \
         } \
     }} while (0)
-NEON_VOP_ENV(qshl_s8, neon_s8, 4)
-NEON_VOP_ENV(qshl_s16, neon_s16, 2)
-NEON_VOP_ENV(qshl_s32, neon_s32, 1)
+NEON_VOP(qshl_s8, neon_s8, 4)
+NEON_VOP(qshl_s16, neon_s16, 2)
+NEON_VOP(qshl_s32, neon_s32, 1)
 #undef NEON_FN
 
-uint64_t HELPER(neon_qshl_s64)(CPUState *env, uint64_t valop, uint64_t shiftop)
+uint64_t HELPER(neon_qshl_s64)(uint64_t valop, uint64_t shiftop)
 {
     int8_t shift = (uint8_t)shiftop;
     int64_t val = valop;
     if (shift >= 64) {
         if (val) {
             SET_QC();
-            val = (val >> 63) & ~SIGNBIT64;
+            val = (val >> 63) ^ ~SIGNBIT64;
         }
-    } else if (shift <= 64) {
+    } else if (shift <= -64) {
         val >>= 63;
     } else if (shift < 0) {
         val >>= -shift;
@@ -623,12 +738,70 @@ uint64_t HELPER(neon_qshl_s64)(CPUState *env, uint64_t valop, uint64_t shiftop)
     return val;
 }
 
+#define NEON_FN(dest, src1, src2) do { \
+    if (src1 & (1 << (sizeof(src1) * 8 - 1))) { \
+        SET_QC(); \
+        dest = 0; \
+    } else { \
+        int8_t tmp; \
+        tmp = (int8_t)src2; \
+        if (tmp >= (ssize_t)sizeof(src1) * 8) { \
+            if (src1) { \
+                SET_QC(); \
+                dest = ~0; \
+            } else { \
+                dest = 0; \
+            } \
+        } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
+            dest = 0; \
+        } else if (tmp < 0) { \
+            dest = src1 >> -tmp; \
+        } else { \
+            dest = src1 << tmp; \
+            if ((dest >> tmp) != src1) { \
+                SET_QC(); \
+                dest = ~0; \
+            } \
+        } \
+    }} while (0)
+NEON_VOP(qshlu_s8, neon_u8, 4)
+NEON_VOP(qshlu_s16, neon_u16, 2)
+#undef NEON_FN
+
+uint32_t HELPER(neon_qshlu_s32)(uint32_t valop, uint32_t shiftop)
+{
+    if ((int32_t)valop < 0) {
+        SET_QC();
+        return 0;
+    }
+    return helper_neon_qshl_u32(valop, shiftop);
+}
+
+uint64_t HELPER(neon_qshlu_s64)(uint64_t valop, uint64_t shiftop)
+{
+    if ((int64_t)valop < 0) {
+        SET_QC();
+        return 0;
+    }
+    return helper_neon_qshl_u64(valop, shiftop);
+}
 
 /* FIXME: This is wrong.  */
 #define NEON_FN(dest, src1, src2) do { \
     int8_t tmp; \
     tmp = (int8_t)src2; \
-    if (tmp < 0) { \
+    if (tmp >= (ssize_t)sizeof(src1) * 8) { \
+        if (src1) { \
+            SET_QC(); \
+            dest = ~0; \
+        } else { \
+            dest = 0; \
+        } \
+    } else if (tmp < -(ssize_t)sizeof(src1) * 8) { \
+        dest = 0; \
+    } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
+        dest = src1 >> (sizeof(src1) * 8 - 1); \
+    } else if (tmp < 0) { \
         dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
     } else { \
         dest = src1 << tmp; \
@@ -637,16 +810,65 @@ uint64_t HELPER(neon_qshl_s64)(CPUState *env, uint64_t valop, uint64_t shiftop)
             dest = ~0; \
         } \
     }} while (0)
-NEON_VOP_ENV(qrshl_u8, neon_u8, 4)
-NEON_VOP_ENV(qrshl_u16, neon_u16, 2)
-NEON_VOP_ENV(qrshl_u32, neon_u32, 1)
+NEON_VOP(qrshl_u8, neon_u8, 4)
+NEON_VOP(qrshl_u16, neon_u16, 2)
 #undef NEON_FN
 
-uint64_t HELPER(neon_qrshl_u64)(CPUState *env, uint64_t val, uint64_t shiftop)
+/* The addition of the rounding constant may overflow, so we use an
+ * intermediate 64 bits accumulator.  */
+uint32_t HELPER(neon_qrshl_u32)(uint32_t val, uint32_t shiftop)
 {
+    uint32_t dest;
     int8_t shift = (int8_t)shiftop;
-    if (shift < 0) {
-        val = (val + (1 << (-1 - shift))) >> -shift;
+    if (shift >= 32) {
+        if (val) {
+            SET_QC();
+            dest = ~0;
+        } else {
+            dest = 0;
+        }
+    } else if (shift < -32) {
+        dest = 0;
+    } else if (shift == -32) {
+        dest = val >> 31;
+    } else if (shift < 0) {
+        uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift)));
+        dest = big_dest >> -shift;
+    } else {
+        dest = val << shift;
+        if ((dest >> shift) != val) {
+            SET_QC();
+            dest = ~0;
+        }
+    }
+    return dest;
+}
+
+/* Handling addition overflow with 64 bits inputs values is more
+ * tricky than with 32 bits values.  */
+uint64_t HELPER(neon_qrshl_u64)(uint64_t val, uint64_t shiftop)
+{
+    int8_t shift = (int8_t)shiftop;
+    if (shift >= 64) {
+        if (val) {
+            SET_QC();
+            val = ~0;
+        }
+    } else if (shift < -64) {
+        val = 0;
+    } else if (shift == -64) {
+        val >>= 63;
+    } else if (shift < 0) {
+        val >>= (-shift - 1);
+        if (val == UINT64_MAX) {
+            /* In this case, it means that the rounding constant is 1,
+             * and the addition would overflow. Return the actual
+             * result directly.  */
+            val = 0x8000000000000000ULL;
+        } else {
+            val++;
+            val >>= 1;
+        }
     } else { \
         uint64_t tmp = val;
         val <<= shift;
@@ -661,33 +883,94 @@ uint64_t HELPER(neon_qrshl_u64)(CPUState *env, uint64_t val, uint64_t shiftop)
 #define NEON_FN(dest, src1, src2) do { \
     int8_t tmp; \
     tmp = (int8_t)src2; \
-    if (tmp < 0) { \
+    if (tmp >= (ssize_t)sizeof(src1) * 8) { \
+        if (src1) { \
+            SET_QC(); \
+            dest = (1 << (sizeof(src1) * 8 - 1)); \
+            if (src1 > 0) { \
+                dest--; \
+            } \
+        } else { \
+            dest = 0; \
+        } \
+    } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
+        dest = 0; \
+    } else if (tmp < 0) { \
         dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
     } else { \
         dest = src1 << tmp; \
         if ((dest >> tmp) != src1) { \
             SET_QC(); \
-            dest = src1 >> 31; \
+            dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
+            if (src1 > 0) { \
+                dest--; \
+            } \
         } \
     }} while (0)
-NEON_VOP_ENV(qrshl_s8, neon_s8, 4)
-NEON_VOP_ENV(qrshl_s16, neon_s16, 2)
-NEON_VOP_ENV(qrshl_s32, neon_s32, 1)
+NEON_VOP(qrshl_s8, neon_s8, 4)
+NEON_VOP(qrshl_s16, neon_s16, 2)
 #undef NEON_FN
 
-uint64_t HELPER(neon_qrshl_s64)(CPUState *env, uint64_t valop, uint64_t shiftop)
+/* The addition of the rounding constant may overflow, so we use an
+ * intermediate 64 bits accumulator.  */
+uint32_t HELPER(neon_qrshl_s32)(uint32_t valop, uint32_t shiftop)
+{
+    int32_t dest;
+    int32_t val = (int32_t)valop;
+    int8_t shift = (int8_t)shiftop;
+    if (shift >= 32) {
+        if (val) {
+            SET_QC();
+            dest = (val >> 31) ^ ~SIGNBIT;
+        } else {
+            dest = 0;
+        }
+    } else if (shift <= -32) {
+        dest = 0;
+    } else if (shift < 0) {
+        int64_t big_dest = ((int64_t)val + (1 << (-1 - shift)));
+        dest = big_dest >> -shift;
+    } else {
+        dest = val << shift;
+        if ((dest >> shift) != val) {
+            SET_QC();
+            dest = (val >> 31) ^ ~SIGNBIT;
+        }
+    }
+    return dest;
+}
+
+/* Handling addition overflow with 64 bits inputs values is more
+ * tricky than with 32 bits values.  */
+uint64_t HELPER(neon_qrshl_s64)(uint64_t valop, uint64_t shiftop)
 {
     int8_t shift = (uint8_t)shiftop;
     int64_t val = valop;
 
-    if (shift < 0) {
-        val = (val + (1 << (-1 - shift))) >> -shift;
+    if (shift >= 64) {
+        if (val) {
+            SET_QC();
+            val = (val >> 63) ^ ~SIGNBIT64;
+        }
+    } else if (shift <= -64) {
+        val = 0;
+    } else if (shift < 0) {
+        val >>= (-shift - 1);
+        if (val == INT64_MAX) {
+            /* In this case, it means that the rounding constant is 1,
+             * and the addition would overflow. Return the actual
+             * result directly.  */
+            val = 0x4000000000000000ULL;
+        } else {
+            val++;
+            val >>= 1;
+        }
     } else {
-        int64_t tmp = val;;
+        int64_t tmp = val;
         val <<= shift;
         if ((val >> shift) != tmp) {
             SET_QC();
-            val = tmp >> 31;
+            val = (tmp >> 63) ^ ~SIGNBIT64;
         }
     }
     return val;
@@ -750,6 +1033,36 @@ uint32_t HELPER(neon_mul_p8)(uint32_t op1, uint32_t op2)
     return result;
 }
 
+uint64_t HELPER(neon_mull_p8)(uint32_t op1, uint32_t op2)
+{
+    uint64_t result = 0;
+    uint64_t mask;
+    uint64_t op2ex = op2;
+    op2ex = (op2ex & 0xff) |
+        ((op2ex & 0xff00) << 8) |
+        ((op2ex & 0xff0000) << 16) |
+        ((op2ex & 0xff000000) << 24);
+    while (op1) {
+        mask = 0;
+        if (op1 & 1) {
+            mask |= 0xffff;
+        }
+        if (op1 & (1 << 8)) {
+            mask |= (0xffffU << 16);
+        }
+        if (op1 & (1 << 16)) {
+            mask |= (0xffffULL << 32);
+        }
+        if (op1 & (1 << 24)) {
+            mask |= (0xffffULL << 48);
+        }
+        result ^= op2ex & mask;
+        op1 = (op1 >> 1) & 0x7f7f7f7f;
+        op2ex <<= 1;
+    }
+    return result;
+}
+
 #define NEON_FN(dest, src1, src2) dest = (src1 & src2) ? -1 : 0
 NEON_VOP(tst_u8, neon_u8, 4)
 NEON_VOP(tst_u16, neon_u16, 2)
@@ -824,8 +1137,9 @@ uint32_t HELPER(neon_cnt_u8)(uint32_t x)
     if ((tmp ^ (tmp << 1)) & SIGNBIT) { \
         SET_QC(); \
         tmp = (tmp >> 31) ^ ~SIGNBIT; \
+    } else { \
+        tmp <<= 1; \
     } \
-    tmp <<= 1; \
     if (round) { \
         int32_t old = tmp; \
         tmp += 1 << 15; \
@@ -837,10 +1151,10 @@ uint32_t HELPER(neon_cnt_u8)(uint32_t x)
     dest = tmp >> 16; \
     } while(0)
 #define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 0)
-NEON_VOP_ENV(qdmulh_s16, neon_s16, 2)
+NEON_VOP(qdmulh_s16, neon_s16, 2)
 #undef NEON_FN
 #define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 1)
-NEON_VOP_ENV(qrdmulh_s16, neon_s16, 2)
+NEON_VOP(qrdmulh_s16, neon_s16, 2)
 #undef NEON_FN
 #undef NEON_QDMULH16
 
@@ -863,10 +1177,10 @@ NEON_VOP_ENV(qrdmulh_s16, neon_s16, 2)
     dest = tmp >> 32; \
     } while(0)
 #define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 0)
-NEON_VOP_ENV(qdmulh_s32, neon_s32, 1)
+NEON_VOP(qdmulh_s32, neon_s32, 1)
 #undef NEON_FN
 #define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 1)
-NEON_VOP_ENV(qrdmulh_s32, neon_s32, 1)
+NEON_VOP(qrdmulh_s32, neon_s32, 1)
 #undef NEON_FN
 #undef NEON_QDMULH32
 
@@ -907,7 +1221,34 @@ uint32_t HELPER(neon_narrow_round_high_u16)(uint64_t x)
     return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000);
 }
 
-uint32_t HELPER(neon_narrow_sat_u8)(CPUState *env, uint64_t x)
+uint32_t HELPER(neon_unarrow_sat8)(uint64_t x)
+{
+    uint16_t s;
+    uint8_t d;
+    uint32_t res = 0;
+#define SAT8(n) \
+    s = x >> n; \
+    if (s & 0x8000) { \
+        SET_QC(); \
+    } else { \
+        if (s > 0xff) { \
+            d = 0xff; \
+            SET_QC(); \
+        } else  { \
+            d = s; \
+        } \
+        res |= (uint32_t)d << (n / 2); \
+    }
+
+    SAT8(0);
+    SAT8(16);
+    SAT8(32);
+    SAT8(48);
+#undef SAT8
+    return res;
+}
+
+uint32_t HELPER(neon_narrow_sat_u8)(uint64_t x)
 {
     uint16_t s;
     uint8_t d;
@@ -930,7 +1271,7 @@ uint32_t HELPER(neon_narrow_sat_u8)(CPUState *env, uint64_t x)
     return res;
 }
 
-uint32_t HELPER(neon_narrow_sat_s8)(CPUState *env, uint64_t x)
+uint32_t HELPER(neon_narrow_sat_s8)(uint64_t x)
 {
     int16_t s;
     uint8_t d;
@@ -953,7 +1294,30 @@ uint32_t HELPER(neon_narrow_sat_s8)(CPUState *env, uint64_t x)
     return res;
 }
 
-uint32_t HELPER(neon_narrow_sat_u16)(CPUState *env, uint64_t x)
+uint32_t HELPER(neon_unarrow_sat16)(uint64_t x)
+{
+    uint32_t high;
+    uint32_t low;
+    low = x;
+    if (low & 0x80000000) {
+        low = 0;
+        SET_QC();
+    } else if (low > 0xffff) {
+        low = 0xffff;
+        SET_QC();
+    }
+    high = x >> 32;
+    if (high & 0x80000000) {
+        high = 0;
+        SET_QC();
+    } else if (high > 0xffff) {
+        high = 0xffff;
+        SET_QC();
+    }
+    return low | (high << 16);
+}
+
+uint32_t HELPER(neon_narrow_sat_u16)(uint64_t x)
 {
     uint32_t high;
     uint32_t low;
@@ -970,7 +1334,7 @@ uint32_t HELPER(neon_narrow_sat_u16)(CPUState *env, uint64_t x)
     return low | (high << 16);
 }
 
-uint32_t HELPER(neon_narrow_sat_s16)(CPUState *env, uint64_t x)
+uint32_t HELPER(neon_narrow_sat_s16)(uint64_t x)
 {
     int32_t low;
     int32_t high;
@@ -987,7 +1351,20 @@ uint32_t HELPER(neon_narrow_sat_s16)(CPUState *env, uint64_t x)
     return (uint16_t)low | (high << 16);
 }
 
-uint32_t HELPER(neon_narrow_sat_u32)(CPUState *env, uint64_t x)
+uint32_t HELPER(neon_unarrow_sat32)(uint64_t x)
+{
+    if (x & 0x8000000000000000ull) {
+        SET_QC();
+        return 0;
+    }
+    if (x > 0xffffffffu) {
+        SET_QC();
+        return 0xffffffffu;
+    }
+    return x;
+}
+
+uint32_t HELPER(neon_narrow_sat_u32)(uint64_t x)
 {
     if (x > 0xffffffffu) {
         SET_QC();
@@ -996,11 +1373,11 @@ uint32_t HELPER(neon_narrow_sat_u32)(CPUState *env, uint64_t x)
     return x;
 }
 
-uint32_t HELPER(neon_narrow_sat_s32)(CPUState *env, uint64_t x)
+uint32_t HELPER(neon_narrow_sat_s32)(uint64_t x)
 {
     if ((int64_t)x != (int32_t)x) {
         SET_QC();
-        return (x >> 63) ^ 0x7fffffff;
+        return ((int64_t)x >> 63) ^ 0x7fffffff;
     }
     return x;
 }
@@ -1103,7 +1480,7 @@ uint64_t HELPER(neon_subl_u32)(uint64_t a, uint64_t b)
     return (a - b) ^ mask;
 }
 
-uint64_t HELPER(neon_addl_saturate_s32)(CPUState *env, uint64_t a, uint64_t b)
+uint64_t HELPER(neon_addl_saturate_s32)(uint64_t a, uint64_t b)
 {
     uint32_t x, y;
     uint32_t low, high;
@@ -1125,7 +1502,7 @@ uint64_t HELPER(neon_addl_saturate_s32)(CPUState *env, uint64_t a, uint64_t b)
     return low | ((uint64_t)high << 32);
 }
 
-uint64_t HELPER(neon_addl_saturate_s64)(CPUState *env, uint64_t a, uint64_t b)
+uint64_t HELPER(neon_addl_saturate_s64)(uint64_t a, uint64_t b)
 {
     uint64_t result;
 
@@ -1137,9 +1514,13 @@ uint64_t HELPER(neon_addl_saturate_s64)(CPUState *env, uint64_t a, uint64_t b)
     return result;
 }
 
-#define DO_ABD(dest, x, y, type) do { \
-    type tmp_x = x; \
-    type tmp_y = y; \
+/* We have to do the arithmetic in a larger type than
+ * the input type, because for example with a signed 32 bit
+ * op the absolute difference can overflow a signed 32 bit value.
+ */
+#define DO_ABD(dest, x, y, intype, arithtype) do {            \
+    arithtype tmp_x = (intype)(x);                            \
+    arithtype tmp_y = (intype)(y);                            \
     dest = ((tmp_x > tmp_y) ? tmp_x - tmp_y : tmp_y - tmp_x); \
     } while(0)
 
@@ -1147,12 +1528,12 @@ uint64_t HELPER(neon_abdl_u16)(uint32_t a, uint32_t b)
 {
     uint64_t tmp;
     uint64_t result;
-    DO_ABD(result, a, b, uint8_t);
-    DO_ABD(tmp, a >> 8, b >> 8, uint8_t);
+    DO_ABD(result, a, b, uint8_t, uint32_t);
+    DO_ABD(tmp, a >> 8, b >> 8, uint8_t, uint32_t);
     result |= tmp << 16;
-    DO_ABD(tmp, a >> 16, b >> 16, uint8_t);
+    DO_ABD(tmp, a >> 16, b >> 16, uint8_t, uint32_t);
     result |= tmp << 32;
-    DO_ABD(tmp, a >> 24, b >> 24, uint8_t);
+    DO_ABD(tmp, a >> 24, b >> 24, uint8_t, uint32_t);
     result |= tmp << 48;
     return result;
 }
@@ -1161,12 +1542,12 @@ uint64_t HELPER(neon_abdl_s16)(uint32_t a, uint32_t b)
 {
     uint64_t tmp;
     uint64_t result;
-    DO_ABD(result, a, b, int8_t);
-    DO_ABD(tmp, a >> 8, b >> 8, int8_t);
+    DO_ABD(result, a, b, int8_t, int32_t);
+    DO_ABD(tmp, a >> 8, b >> 8, int8_t, int32_t);
     result |= tmp << 16;
-    DO_ABD(tmp, a >> 16, b >> 16, int8_t);
+    DO_ABD(tmp, a >> 16, b >> 16, int8_t, int32_t);
     result |= tmp << 32;
-    DO_ABD(tmp, a >> 24, b >> 24, int8_t);
+    DO_ABD(tmp, a >> 24, b >> 24, int8_t, int32_t);
     result |= tmp << 48;
     return result;
 }
@@ -1175,8 +1556,8 @@ uint64_t HELPER(neon_abdl_u32)(uint32_t a, uint32_t b)
 {
     uint64_t tmp;
     uint64_t result;
-    DO_ABD(result, a, b, uint16_t);
-    DO_ABD(tmp, a >> 16, b >> 16, uint16_t);
+    DO_ABD(result, a, b, uint16_t, uint32_t);
+    DO_ABD(tmp, a >> 16, b >> 16, uint16_t, uint32_t);
     return result | (tmp << 32);
 }
 
@@ -1184,22 +1565,22 @@ uint64_t HELPER(neon_abdl_s32)(uint32_t a, uint32_t b)
 {
     uint64_t tmp;
     uint64_t result;
-    DO_ABD(result, a, b, int16_t);
-    DO_ABD(tmp, a >> 16, b >> 16, int16_t);
+    DO_ABD(result, a, b, int16_t, int32_t);
+    DO_ABD(tmp, a >> 16, b >> 16, int16_t, int32_t);
     return result | (tmp << 32);
 }
 
 uint64_t HELPER(neon_abdl_u64)(uint32_t a, uint32_t b)
 {
     uint64_t result;
-    DO_ABD(result, a, b, uint32_t);
+    DO_ABD(result, a, b, uint32_t, uint64_t);
     return result;
 }
 
 uint64_t HELPER(neon_abdl_s64)(uint32_t a, uint32_t b)
 {
     uint64_t result;
-    DO_ABD(result, a, b, int32_t);
+    DO_ABD(result, a, b, int32_t, int64_t);
     return result;
 }
 #undef DO_ABD
@@ -1275,7 +1656,6 @@ uint64_t HELPER(neon_negl_u16)(uint64_t x)
     return result;
 }
 
-#include <stdio.h>
 uint64_t HELPER(neon_negl_u32)(uint64_t x)
 {
     uint32_t low = -x;
@@ -1298,7 +1678,7 @@ uint64_t HELPER(neon_negl_u64)(uint64_t x)
     } else if (x < 0) { \
         x = -x; \
     }} while (0)
-uint32_t HELPER(neon_qabs_s8)(CPUState *env, uint32_t x)
+uint32_t HELPER(neon_qabs_s8)(uint32_t x)
 {
     neon_s8 vec;
     NEON_UNPACK(neon_s8, vec, x);
@@ -1318,7 +1698,7 @@ uint32_t HELPER(neon_qabs_s8)(CPUState *env, uint32_t x)
     } else { \
         x = -x; \
     }} while (0)
-uint32_t HELPER(neon_qneg_s8)(CPUState *env, uint32_t x)
+uint32_t HELPER(neon_qneg_s8)(uint32_t x)
 {
     neon_s8 vec;
     NEON_UNPACK(neon_s8, vec, x);
@@ -1338,7 +1718,7 @@ uint32_t HELPER(neon_qneg_s8)(CPUState *env, uint32_t x)
     } else if (x < 0) { \
         x = -x; \
     }} while (0)
-uint32_t HELPER(neon_qabs_s16)(CPUState *env, uint32_t x)
+uint32_t HELPER(neon_qabs_s16)(uint32_t x)
 {
     neon_s16 vec;
     NEON_UNPACK(neon_s16, vec, x);
@@ -1356,7 +1736,7 @@ uint32_t HELPER(neon_qabs_s16)(CPUState *env, uint32_t x)
     } else { \
         x = -x; \
     }} while (0)
-uint32_t HELPER(neon_qneg_s16)(CPUState *env, uint32_t x)
+uint32_t HELPER(neon_qneg_s16)(uint32_t x)
 {
     neon_s16 vec;
     NEON_UNPACK(neon_s16, vec, x);
@@ -1367,7 +1747,7 @@ uint32_t HELPER(neon_qneg_s16)(CPUState *env, uint32_t x)
 }
 #undef DO_QNEG16
 
-uint32_t HELPER(neon_qabs_s32)(CPUState *env, uint32_t x)
+uint32_t HELPER(neon_qabs_s32)(uint32_t x)
 {
     if (x == SIGNBIT) {
         SET_QC();
@@ -1378,7 +1758,7 @@ uint32_t HELPER(neon_qabs_s32)(CPUState *env, uint32_t x)
     return x;
 }
 
-uint32_t HELPER(neon_qneg_s32)(CPUState *env, uint32_t x)
+uint32_t HELPER(neon_qneg_s32)(uint32_t x)
 {
     if (x == SIGNBIT) {
         SET_QC();
@@ -1392,66 +1772,251 @@ uint32_t HELPER(neon_qneg_s32)(CPUState *env, uint32_t x)
 /* NEON Float helpers.  */
 uint32_t HELPER(neon_min_f32)(uint32_t a, uint32_t b)
 {
-    float32 f0 = vfp_itos(a);
-    float32 f1 = vfp_itos(b);
-    return (float32_compare_quiet(f0, f1, NFS) == -1) ? a : b;
+    return float32_val(float32_min(make_float32(a), make_float32(b), NFS));
 }
 
 uint32_t HELPER(neon_max_f32)(uint32_t a, uint32_t b)
 {
-    float32 f0 = vfp_itos(a);
-    float32 f1 = vfp_itos(b);
-    return (float32_compare_quiet(f0, f1, NFS) == 1) ? a : b;
+    return float32_val(float32_max(make_float32(a), make_float32(b), NFS));
 }
 
 uint32_t HELPER(neon_abd_f32)(uint32_t a, uint32_t b)
 {
-    float32 f0 = vfp_itos(a);
-    float32 f1 = vfp_itos(b);
-    return vfp_stoi((float32_compare_quiet(f0, f1, NFS) == 1)
-                    ? float32_sub(f0, f1, NFS)
-                    : float32_sub(f1, f0, NFS));
+    float32 f0 = make_float32(a);
+    float32 f1 = make_float32(b);
+    return float32_val(float32_abs(float32_sub(f0, f1, NFS)));
 }
 
 uint32_t HELPER(neon_add_f32)(uint32_t a, uint32_t b)
 {
-    return vfp_stoi(float32_add(vfp_itos(a), vfp_itos(b), NFS));
+    return float32_val(float32_add(make_float32(a), make_float32(b), NFS));
 }
 
 uint32_t HELPER(neon_sub_f32)(uint32_t a, uint32_t b)
 {
-    return vfp_stoi(float32_sub(vfp_itos(a), vfp_itos(b), NFS));
+    return float32_val(float32_sub(make_float32(a), make_float32(b), NFS));
 }
 
 uint32_t HELPER(neon_mul_f32)(uint32_t a, uint32_t b)
 {
-    return vfp_stoi(float32_mul(vfp_itos(a), vfp_itos(b), NFS));
+    return float32_val(float32_mul(make_float32(a), make_float32(b), NFS));
 }
 
-/* Floating point comparisons produce an integer result.  */
-#define NEON_VOP_FCMP(name, cmp) \
-uint32_t HELPER(neon_##name)(uint32_t a, uint32_t b) \
-{ \
-    if (float32_compare_quiet(vfp_itos(a), vfp_itos(b), NFS) cmp 0) \
-        return ~0; \
-    else \
-        return 0; \
+/* Floating point comparisons produce an integer result.
+ * Note that EQ doesn't signal InvalidOp for QNaNs but GE and GT do.
+ * Softfloat routines return 0/1, which we convert to the 0/-1 Neon requires.
+ */
+uint32_t HELPER(neon_ceq_f32)(uint32_t a, uint32_t b)
+{
+    return -float32_eq_quiet(make_float32(a), make_float32(b), NFS);
+}
+
+uint32_t HELPER(neon_cge_f32)(uint32_t a, uint32_t b)
+{
+    return -float32_le(make_float32(b), make_float32(a), NFS);
 }
 
-NEON_VOP_FCMP(ceq_f32, ==)
-NEON_VOP_FCMP(cge_f32, >=)
-NEON_VOP_FCMP(cgt_f32, >)
+uint32_t HELPER(neon_cgt_f32)(uint32_t a, uint32_t b)
+{
+    return -float32_lt(make_float32(b), make_float32(a), NFS);
+}
 
 uint32_t HELPER(neon_acge_f32)(uint32_t a, uint32_t b)
 {
-    float32 f0 = float32_abs(vfp_itos(a));
-    float32 f1 = float32_abs(vfp_itos(b));
-    return (float32_compare_quiet(f0, f1,NFS) >= 0) ? ~0 : 0;
+    float32 f0 = float32_abs(make_float32(a));
+    float32 f1 = float32_abs(make_float32(b));
+    return -float32_le(f1, f0, NFS);
 }
 
 uint32_t HELPER(neon_acgt_f32)(uint32_t a, uint32_t b)
 {
-    float32 f0 = float32_abs(vfp_itos(a));
-    float32 f1 = float32_abs(vfp_itos(b));
-    return (float32_compare_quiet(f0, f1, NFS) > 0) ? ~0 : 0;
+    float32 f0 = float32_abs(make_float32(a));
+    float32 f1 = float32_abs(make_float32(b));
+    return -float32_lt(f1, f0, NFS);
+}
+
+#define ELEM(V, N, SIZE) (((V) >> ((N) * (SIZE))) & ((1ull << (SIZE)) - 1))
+
+void HELPER(neon_qunzip8)(uint32_t rd, uint32_t rm)
+{
+    uint64_t zm0 = float64_val(env->vfp.regs[rm]);
+    uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
+    uint64_t zd0 = float64_val(env->vfp.regs[rd]);
+    uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
+    uint64_t d0 = ELEM(zd0, 0, 8) | (ELEM(zd0, 2, 8) << 8)
+        | (ELEM(zd0, 4, 8) << 16) | (ELEM(zd0, 6, 8) << 24)
+        | (ELEM(zd1, 0, 8) << 32) | (ELEM(zd1, 2, 8) << 40)
+        | (ELEM(zd1, 4, 8) << 48) | (ELEM(zd1, 6, 8) << 56);
+    uint64_t d1 = ELEM(zm0, 0, 8) | (ELEM(zm0, 2, 8) << 8)
+        | (ELEM(zm0, 4, 8) << 16) | (ELEM(zm0, 6, 8) << 24)
+        | (ELEM(zm1, 0, 8) << 32) | (ELEM(zm1, 2, 8) << 40)
+        | (ELEM(zm1, 4, 8) << 48) | (ELEM(zm1, 6, 8) << 56);
+    uint64_t m0 = ELEM(zd0, 1, 8) | (ELEM(zd0, 3, 8) << 8)
+        | (ELEM(zd0, 5, 8) << 16) | (ELEM(zd0, 7, 8) << 24)
+        | (ELEM(zd1, 1, 8) << 32) | (ELEM(zd1, 3, 8) << 40)
+        | (ELEM(zd1, 5, 8) << 48) | (ELEM(zd1, 7, 8) << 56);
+    uint64_t m1 = ELEM(zm0, 1, 8) | (ELEM(zm0, 3, 8) << 8)
+        | (ELEM(zm0, 5, 8) << 16) | (ELEM(zm0, 7, 8) << 24)
+        | (ELEM(zm1, 1, 8) << 32) | (ELEM(zm1, 3, 8) << 40)
+        | (ELEM(zm1, 5, 8) << 48) | (ELEM(zm1, 7, 8) << 56);
+    env->vfp.regs[rm] = make_float64(m0);
+    env->vfp.regs[rm + 1] = make_float64(m1);
+    env->vfp.regs[rd] = make_float64(d0);
+    env->vfp.regs[rd + 1] = make_float64(d1);
+}
+
+void HELPER(neon_qunzip16)(uint32_t rd, uint32_t rm)
+{
+    uint64_t zm0 = float64_val(env->vfp.regs[rm]);
+    uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
+    uint64_t zd0 = float64_val(env->vfp.regs[rd]);
+    uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
+    uint64_t d0 = ELEM(zd0, 0, 16) | (ELEM(zd0, 2, 16) << 16)
+        | (ELEM(zd1, 0, 16) << 32) | (ELEM(zd1, 2, 16) << 48);
+    uint64_t d1 = ELEM(zm0, 0, 16) | (ELEM(zm0, 2, 16) << 16)
+        | (ELEM(zm1, 0, 16) << 32) | (ELEM(zm1, 2, 16) << 48);
+    uint64_t m0 = ELEM(zd0, 1, 16) | (ELEM(zd0, 3, 16) << 16)
+        | (ELEM(zd1, 1, 16) << 32) | (ELEM(zd1, 3, 16) << 48);
+    uint64_t m1 = ELEM(zm0, 1, 16) | (ELEM(zm0, 3, 16) << 16)
+        | (ELEM(zm1, 1, 16) << 32) | (ELEM(zm1, 3, 16) << 48);
+    env->vfp.regs[rm] = make_float64(m0);
+    env->vfp.regs[rm + 1] = make_float64(m1);
+    env->vfp.regs[rd] = make_float64(d0);
+    env->vfp.regs[rd + 1] = make_float64(d1);
+}
+
+void HELPER(neon_qunzip32)(uint32_t rd, uint32_t rm)
+{
+    uint64_t zm0 = float64_val(env->vfp.regs[rm]);
+    uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
+    uint64_t zd0 = float64_val(env->vfp.regs[rd]);
+    uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
+    uint64_t d0 = ELEM(zd0, 0, 32) | (ELEM(zd1, 0, 32) << 32);
+    uint64_t d1 = ELEM(zm0, 0, 32) | (ELEM(zm1, 0, 32) << 32);
+    uint64_t m0 = ELEM(zd0, 1, 32) | (ELEM(zd1, 1, 32) << 32);
+    uint64_t m1 = ELEM(zm0, 1, 32) | (ELEM(zm1, 1, 32) << 32);
+    env->vfp.regs[rm] = make_float64(m0);
+    env->vfp.regs[rm + 1] = make_float64(m1);
+    env->vfp.regs[rd] = make_float64(d0);
+    env->vfp.regs[rd + 1] = make_float64(d1);
+}
+
+void HELPER(neon_unzip8)(uint32_t rd, uint32_t rm)
+{
+    uint64_t zm = float64_val(env->vfp.regs[rm]);
+    uint64_t zd = float64_val(env->vfp.regs[rd]);
+    uint64_t d0 = ELEM(zd, 0, 8) | (ELEM(zd, 2, 8) << 8)
+        | (ELEM(zd, 4, 8) << 16) | (ELEM(zd, 6, 8) << 24)
+        | (ELEM(zm, 0, 8) << 32) | (ELEM(zm, 2, 8) << 40)
+        | (ELEM(zm, 4, 8) << 48) | (ELEM(zm, 6, 8) << 56);
+    uint64_t m0 = ELEM(zd, 1, 8) | (ELEM(zd, 3, 8) << 8)
+        | (ELEM(zd, 5, 8) << 16) | (ELEM(zd, 7, 8) << 24)
+        | (ELEM(zm, 1, 8) << 32) | (ELEM(zm, 3, 8) << 40)
+        | (ELEM(zm, 5, 8) << 48) | (ELEM(zm, 7, 8) << 56);
+    env->vfp.regs[rm] = make_float64(m0);
+    env->vfp.regs[rd] = make_float64(d0);
+}
+
+void HELPER(neon_unzip16)(uint32_t rd, uint32_t rm)
+{
+    uint64_t zm = float64_val(env->vfp.regs[rm]);
+    uint64_t zd = float64_val(env->vfp.regs[rd]);
+    uint64_t d0 = ELEM(zd, 0, 16) | (ELEM(zd, 2, 16) << 16)
+        | (ELEM(zm, 0, 16) << 32) | (ELEM(zm, 2, 16) << 48);
+    uint64_t m0 = ELEM(zd, 1, 16) | (ELEM(zd, 3, 16) << 16)
+        | (ELEM(zm, 1, 16) << 32) | (ELEM(zm, 3, 16) << 48);
+    env->vfp.regs[rm] = make_float64(m0);
+    env->vfp.regs[rd] = make_float64(d0);
+}
+
+void HELPER(neon_qzip8)(uint32_t rd, uint32_t rm)
+{
+    uint64_t zm0 = float64_val(env->vfp.regs[rm]);
+    uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
+    uint64_t zd0 = float64_val(env->vfp.regs[rd]);
+    uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
+    uint64_t d0 = ELEM(zd0, 0, 8) | (ELEM(zm0, 0, 8) << 8)
+        | (ELEM(zd0, 1, 8) << 16) | (ELEM(zm0, 1, 8) << 24)
+        | (ELEM(zd0, 2, 8) << 32) | (ELEM(zm0, 2, 8) << 40)
+        | (ELEM(zd0, 3, 8) << 48) | (ELEM(zm0, 3, 8) << 56);
+    uint64_t d1 = ELEM(zd0, 4, 8) | (ELEM(zm0, 4, 8) << 8)
+        | (ELEM(zd0, 5, 8) << 16) | (ELEM(zm0, 5, 8) << 24)
+        | (ELEM(zd0, 6, 8) << 32) | (ELEM(zm0, 6, 8) << 40)
+        | (ELEM(zd0, 7, 8) << 48) | (ELEM(zm0, 7, 8) << 56);
+    uint64_t m0 = ELEM(zd1, 0, 8) | (ELEM(zm1, 0, 8) << 8)
+        | (ELEM(zd1, 1, 8) << 16) | (ELEM(zm1, 1, 8) << 24)
+        | (ELEM(zd1, 2, 8) << 32) | (ELEM(zm1, 2, 8) << 40)
+        | (ELEM(zd1, 3, 8) << 48) | (ELEM(zm1, 3, 8) << 56);
+    uint64_t m1 = ELEM(zd1, 4, 8) | (ELEM(zm1, 4, 8) << 8)
+        | (ELEM(zd1, 5, 8) << 16) | (ELEM(zm1, 5, 8) << 24)
+        | (ELEM(zd1, 6, 8) << 32) | (ELEM(zm1, 6, 8) << 40)
+        | (ELEM(zd1, 7, 8) << 48) | (ELEM(zm1, 7, 8) << 56);
+    env->vfp.regs[rm] = make_float64(m0);
+    env->vfp.regs[rm + 1] = make_float64(m1);
+    env->vfp.regs[rd] = make_float64(d0);
+    env->vfp.regs[rd + 1] = make_float64(d1);
+}
+
+void HELPER(neon_qzip16)(uint32_t rd, uint32_t rm)
+{
+    uint64_t zm0 = float64_val(env->vfp.regs[rm]);
+    uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
+    uint64_t zd0 = float64_val(env->vfp.regs[rd]);
+    uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
+    uint64_t d0 = ELEM(zd0, 0, 16) | (ELEM(zm0, 0, 16) << 16)
+        | (ELEM(zd0, 1, 16) << 32) | (ELEM(zm0, 1, 16) << 48);
+    uint64_t d1 = ELEM(zd0, 2, 16) | (ELEM(zm0, 2, 16) << 16)
+        | (ELEM(zd0, 3, 16) << 32) | (ELEM(zm0, 3, 16) << 48);
+    uint64_t m0 = ELEM(zd1, 0, 16) | (ELEM(zm1, 0, 16) << 16)
+        | (ELEM(zd1, 1, 16) << 32) | (ELEM(zm1, 1, 16) << 48);
+    uint64_t m1 = ELEM(zd1, 2, 16) | (ELEM(zm1, 2, 16) << 16)
+        | (ELEM(zd1, 3, 16) << 32) | (ELEM(zm1, 3, 16) << 48);
+    env->vfp.regs[rm] = make_float64(m0);
+    env->vfp.regs[rm + 1] = make_float64(m1);
+    env->vfp.regs[rd] = make_float64(d0);
+    env->vfp.regs[rd + 1] = make_float64(d1);
+}
+
+void HELPER(neon_qzip32)(uint32_t rd, uint32_t rm)
+{
+    uint64_t zm0 = float64_val(env->vfp.regs[rm]);
+    uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
+    uint64_t zd0 = float64_val(env->vfp.regs[rd]);
+    uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
+    uint64_t d0 = ELEM(zd0, 0, 32) | (ELEM(zm0, 0, 32) << 32);
+    uint64_t d1 = ELEM(zd0, 1, 32) | (ELEM(zm0, 1, 32) << 32);
+    uint64_t m0 = ELEM(zd1, 0, 32) | (ELEM(zm1, 0, 32) << 32);
+    uint64_t m1 = ELEM(zd1, 1, 32) | (ELEM(zm1, 1, 32) << 32);
+    env->vfp.regs[rm] = make_float64(m0);
+    env->vfp.regs[rm + 1] = make_float64(m1);
+    env->vfp.regs[rd] = make_float64(d0);
+    env->vfp.regs[rd + 1] = make_float64(d1);
+}
+
+void HELPER(neon_zip8)(uint32_t rd, uint32_t rm)
+{
+    uint64_t zm = float64_val(env->vfp.regs[rm]);
+    uint64_t zd = float64_val(env->vfp.regs[rd]);
+    uint64_t d0 = ELEM(zd, 0, 8) | (ELEM(zm, 0, 8) << 8)
+        | (ELEM(zd, 1, 8) << 16) | (ELEM(zm, 1, 8) << 24)
+        | (ELEM(zd, 2, 8) << 32) | (ELEM(zm, 2, 8) << 40)
+        | (ELEM(zd, 3, 8) << 48) | (ELEM(zm, 3, 8) << 56);
+    uint64_t m0 = ELEM(zd, 4, 8) | (ELEM(zm, 4, 8) << 8)
+        | (ELEM(zd, 5, 8) << 16) | (ELEM(zm, 5, 8) << 24)
+        | (ELEM(zd, 6, 8) << 32) | (ELEM(zm, 6, 8) << 40)
+        | (ELEM(zd, 7, 8) << 48) | (ELEM(zm, 7, 8) << 56);
+    env->vfp.regs[rm] = make_float64(m0);
+    env->vfp.regs[rd] = make_float64(d0);
+}
+
+void HELPER(neon_zip16)(uint32_t rd, uint32_t rm)
+{
+    uint64_t zm = float64_val(env->vfp.regs[rm]);
+    uint64_t zd = float64_val(env->vfp.regs[rd]);
+    uint64_t d0 = ELEM(zd, 0, 16) | (ELEM(zm, 0, 16) << 16)
+        | (ELEM(zd, 1, 16) << 32) | (ELEM(zm, 1, 16) << 48);
+    uint64_t m0 = ELEM(zd, 2, 16) | (ELEM(zm, 2, 16) << 16)
+        | (ELEM(zd, 3, 16) << 32) | (ELEM(zm, 3, 16) << 48);
+    env->vfp.regs[rm] = make_float64(m0);
+    env->vfp.regs[rd] = make_float64(d0);
 }