Overflow fixes for amrwbenc

Most of these were encountered while running a mixed sanitized/unsanitized
AMR encoder, so may not be reachable in real conditions.

Change-Id: I85af7d40214133234009323e7e64432fc1be39ca

4 files changed, 26 insertions, 16 deletions

diff --git a/media/libstagefright/codecs/amrwbenc/inc/basic_op.h b/media/libstagefright/codecs/amrwbenc/inc/basic_op.h
index e63a55a..d36f455 100644
--- a/media/libstagefright/codecs/amrwbenc/inc/basic_op.h
+++ b/media/libstagefright/codecs/amrwbenc/inc/basic_op.h
@@ -222,14 +222,18 @@ static_vo Word16 shl (Word16 var1, Word16 var2)
     }
     else
     {
-        result = (Word32) var1 *((Word32) 1 << var2);
-        if ((var2 > 15 && var1 != 0) || (result != (Word32) ((Word16) result)))
+        if (var2 > 15 && var1 != 0)
         {
             var_out = (Word16)((var1 > 0) ? MAX_16 : MIN_16);
         }
         else
         {
-            var_out = extract_l (result);
+            result = (Word32) var1 *((Word32) 1 << var2);
+            if ((result != (Word32) ((Word16) result))) {
+                var_out = (Word16)((var1 > 0) ? MAX_16 : MIN_16);
+            } else {
+                var_out = extract_l (result);
+            }
         }
     }
     return (var_out);
diff --git a/media/libstagefright/codecs/amrwbenc/src/c2t64fx.c b/media/libstagefright/codecs/amrwbenc/src/c2t64fx.c
index c7c9279..dbb94c6 100644
--- a/media/libstagefright/codecs/amrwbenc/src/c2t64fx.c
+++ b/media/libstagefright/codecs/amrwbenc/src/c2t64fx.c
@@ -93,7 +93,7 @@ void ACELP_2t64_fx(
 #endif
 
     Isqrt_n(&s, &exp);
-    k_dn = vo_round(L_shl(s, (exp + 8)));    /* k_dn = 256..4096 */
+    k_dn = voround(L_shl(s, (exp + 8)));    /* k_dn = 256..4096 */
     k_dn = vo_mult_r(alp, k_dn);              /* alp in Q12 */
 
     /* mix normalized cn[] and dn[] */
diff --git a/media/libstagefright/codecs/amrwbenc/src/c4t64fx.c b/media/libstagefright/codecs/amrwbenc/src/c4t64fx.c
index b9a9e26..49a89a1 100644
--- a/media/libstagefright/codecs/amrwbenc/src/c4t64fx.c
+++ b/media/libstagefright/codecs/amrwbenc/src/c4t64fx.c
@@ -257,7 +257,7 @@ void ACELP_4t64_fx(
 #endif
 
     Isqrt_n(&s, &exp);
-    k_dn = (L_shl(s, (exp + 5 + 3)) + 0x8000) >> 16;    /* k_dn = 256..4096 */
+    k_dn = voround(L_shl(s, (exp + 5 + 3)));    /* k_dn = 256..4096 */
     k_dn = vo_mult_r(alp, k_dn);              /* alp in Q12 */
 
     /* mix normalized cn[] and dn[] */
@@ -1005,7 +1005,7 @@ void search_ixiy(
     for (x = track_x; x < L_SUBFR; x += STEP)
     {
         ps1 = *ps + dn[x];
-        alp1 = alp0 + ((*p0++)<<13);
+        alp1 = L_add(alp0, ((*p0++)<<13));
 
         if (dn2[x] < thres_ix)
         {
@@ -1018,7 +1018,7 @@ void search_ixiy(
                 alp2 = alp2 + ((*p2++)<<14);
                 alp_16 = extract_h(alp2);
                 sq = vo_mult(ps2, ps2);
-                s = vo_L_mult(alpk, sq) - ((sqk * alp_16)<<1);
+                s = L_sub(vo_L_mult(alpk, sq), L_mult(sqk, alp_16));
 
                 if (s > 0)
                 {
diff --git a/media/libstagefright/codecs/amrwbenc/src/cor_h_x.c b/media/libstagefright/codecs/amrwbenc/src/cor_h_x.c
index b2aa759..e834396 100644
--- a/media/libstagefright/codecs/amrwbenc/src/cor_h_x.c
+++ b/media/libstagefright/codecs/amrwbenc/src/cor_h_x.c
@@ -55,10 +55,10 @@ void cor_h_x(
         p1 = &x[i];
         p2 = &h[0];
         for (j = i; j < L_SUBFR; j++)
-            L_tmp += vo_L_mult(*p1++, *p2++);
+            L_tmp = L_add(L_tmp, vo_L_mult(*p1++, *p2++));
 
         y32[i] = L_tmp;
-        L_tmp = (L_tmp > 0)? L_tmp:-L_tmp;
+        L_tmp = (L_tmp > 0)? L_tmp: (L_tmp == INT_MIN ? INT_MAX : -L_tmp);
         if(L_tmp > L_max)
         {
             L_max = L_tmp;
@@ -68,10 +68,10 @@ void cor_h_x(
         p1 = &x[i+1];
         p2 = &h[0];
         for (j = i+1; j < L_SUBFR; j++)
-            L_tmp += vo_L_mult(*p1++, *p2++);
+            L_tmp = L_add(L_tmp, vo_L_mult(*p1++, *p2++));
 
         y32[i+1] = L_tmp;
-        L_tmp = (L_tmp > 0)? L_tmp:-L_tmp;
+        L_tmp = (L_tmp > 0)? L_tmp: (L_tmp == INT_MIN ? INT_MAX : -L_tmp);
         if(L_tmp > L_max1)
         {
             L_max1 = L_tmp;
@@ -81,10 +81,10 @@ void cor_h_x(
         p1 = &x[i+2];
         p2 = &h[0];
         for (j = i+2; j < L_SUBFR; j++)
-            L_tmp += vo_L_mult(*p1++, *p2++);
+            L_tmp = L_add(L_tmp, vo_L_mult(*p1++, *p2++));
 
         y32[i+2] = L_tmp;
-        L_tmp = (L_tmp > 0)? L_tmp:-L_tmp;
+        L_tmp = (L_tmp > 0)? L_tmp: (L_tmp == INT_MIN ? INT_MAX : -L_tmp);
         if(L_tmp > L_max2)
         {
             L_max2 = L_tmp;
@@ -94,17 +94,23 @@ void cor_h_x(
         p1 = &x[i+3];
         p2 = &h[0];
         for (j = i+3; j < L_SUBFR; j++)
-            L_tmp += vo_L_mult(*p1++, *p2++);
+            L_tmp = L_add(L_tmp, vo_L_mult(*p1++, *p2++));
 
         y32[i+3] = L_tmp;
-        L_tmp = (L_tmp > 0)? L_tmp:-L_tmp;
+        L_tmp = (L_tmp > 0)? L_tmp: (L_tmp == INT_MIN ? INT_MAX : -L_tmp);
         if(L_tmp > L_max3)
         {
             L_max3 = L_tmp;
         }
     }
     /* tot += 3*max / 8 */
-    L_max = ((L_max + L_max1 + L_max2 + L_max3) >> 2);
+    if (L_max > INT_MAX - L_max1 ||
+            L_max + L_max1 > INT_MAX - L_max2 ||
+            L_max + L_max1 + L_max2 > INT_MAX - L_max3) {
+        L_max = INT_MAX >> 2;
+    } else {
+        L_max = ((L_max + L_max1 + L_max2 + L_max3) >> 2);
+    }
     L_tot = vo_L_add(L_tot, L_max);       /* +max/4 */
     L_tot = vo_L_add(L_tot, (L_max >> 1));  /* +max/8 */