Fix overflows in amrwbenc

Revert 3cdaed88daeeebfe05e7913837f41c2d92f411cc and
883ff4f4c41ae9bf84d1912ab054fe38f7505dd0, and fix the overflows that
should have been clamped instead of allowed to overflow.

Bug: 25843966
Bug: 23752600

Change-Id: I9af1726d058eb8eeaa80fa8df74fe0a3759486c1

4 files changed, 34 insertions, 3 deletions

diff --git a/media/libstagefright/codecs/amrwbenc/src/c2t64fx.c b/media/libstagefright/codecs/amrwbenc/src/c2t64fx.c
index 18698e2..519924d 100644
--- a/media/libstagefright/codecs/amrwbenc/src/c2t64fx.c
+++ b/media/libstagefright/codecs/amrwbenc/src/c2t64fx.c
@@ -80,6 +80,9 @@ void ACELP_2t64_fx(
 
 	Isqrt_n(&s, &exp);
 	s = L_shl(s, add1(exp, 5));
+	if (s > INT_MAX - 0x8000) {
+		s = INT_MAX - 0x8000;
+	}
 	k_cn = vo_round(s);
 
 	/* set k_dn = 32..512 (ener_dn = 2^30..2^22) */
diff --git a/media/libstagefright/codecs/amrwbenc/src/c4t64fx.c b/media/libstagefright/codecs/amrwbenc/src/c4t64fx.c
index 1ecc11f..6505e5d 100644
--- a/media/libstagefright/codecs/amrwbenc/src/c4t64fx.c
+++ b/media/libstagefright/codecs/amrwbenc/src/c4t64fx.c
@@ -628,8 +628,16 @@ void ACELP_4t64_fx(
 			L_tmp = 0L;
 			for(i = 0; i < L_SUBFR; i++)
 			{
+				Word32 vecSq2;
 				vec[i]  = add1(add1(add1(*p0++, *p1++), *p2++), *p3++);
-				L_tmp  += (vec[i] * vec[i]) << 1;
+				vecSq2 = (vec[i] * vec[i]) << 1;
+				if (vecSq2 > 0 && L_tmp > INT_MAX - vecSq2) {
+					L_tmp = INT_MAX;
+				} else if (vecSq2 < 0 && L_tmp < INT_MIN - vecSq2) {
+					L_tmp = INT_MIN;
+				} else {
+					L_tmp  += vecSq2;
+				}
 			}
 
 			alp = ((L_tmp >> 3) + 0x8000) >> 16;
diff --git a/media/libstagefright/codecs/amrwbenc/src/deemph.c b/media/libstagefright/codecs/amrwbenc/src/deemph.c
index 0c49d6b..5eae6a7 100644
--- a/media/libstagefright/codecs/amrwbenc/src/deemph.c
+++ b/media/libstagefright/codecs/amrwbenc/src/deemph.c
@@ -68,8 +68,19 @@ void Deemph2(
 	x[0] = (L_tmp + 0x8000)>>16;
 	for (i = 1; i < L; i++)
 	{
+		Word32 tmp;
 		L_tmp = x[i] << 15;
-		L_tmp += (x[i - 1] * mu)<<1;
+		tmp = (x[i - 1] * mu)<<1;
+		if (tmp > 0 && L_tmp > INT_MAX - tmp) {
+			L_tmp = INT_MAX;
+		} else if (tmp < 0 && L_tmp < INT_MIN - tmp) {
+			L_tmp = INT_MIN;
+		} else {
+			L_tmp += tmp;
+		}
+		if (L_tmp > INT32_MAX - 0x8000) {
+			L_tmp = INT_MAX - 0x8000;
+		}
 		x[i] = (L_tmp + 0x8000)>>16;
 	}
 	*mem = x[L - 1];
diff --git a/media/libstagefright/codecs/amrwbenc/src/preemph.c b/media/libstagefright/codecs/amrwbenc/src/preemph.c
index c867bf7..6c26da5 100644
--- a/media/libstagefright/codecs/amrwbenc/src/preemph.c
+++ b/media/libstagefright/codecs/amrwbenc/src/preemph.c
@@ -69,14 +69,23 @@ void Preemph2(
 	for (i = (Word16) (lg - 1); i > 0; i--)
 	{
 		L_tmp = L_deposit_h(x[i]);
-		L_tmp -= (x[i - 1] * mu)<<1;
+		L_tmp -= (x[i - 1] * mu)<<1; // only called with mu == 22282, so this won't overflow
+		if (L_tmp > INT32_MAX / 2) {
+			L_tmp = INT32_MAX / 2;
+		}
 		L_tmp = (L_tmp << 1);
 		x[i] = (L_tmp + 0x8000)>>16;
 	}
 
 	L_tmp = L_deposit_h(x[0]);
 	L_tmp -= ((*mem) * mu)<<1;
+	if (L_tmp > INT32_MAX / 2) {
+		L_tmp = INT32_MAX / 2;
+	}
 	L_tmp = (L_tmp << 1);
+	if (L_tmp > INT32_MAX - 0x8000) {
+		L_tmp = INT32_MAX - 0x8000;
+	}
 	x[0] = (L_tmp + 0x8000)>>16;
 
 	*mem = temp;