AArch64: Correction for LP64

In the Intra16x16PlanePrediction function, a u32 (unsigned 32-bit
integer) was being used for the "i" variable, and being used with a
value of 7 in the loop "for (i = 0, b = 0; i < 8; i++)" to access the
"above[6-i]" location where "above" is defined as "u8 *". Because "i"
was unsigned there was no sign extension on use with the __LP64__
64-bit pointer, so rather than 0xFFFFFFFF being treated as -1, it was
treated as 4,292,967,295 and that resulted in a SIGSEGV at an invalid
address. By changing the type of "i" to an i32 (signed 32-bit integer)
the expected sign extension occurs and the value is treated as -1.

This change fixes android.media.cts.DecoderTest#testCodeBasicH264 on
64-bit platforms

Change-Id: I85df58b0dc1d39f89ab421d04ab5481356520f0c
Signed-off-by: Marcus Oakland <marcus.oakland@arm.com>

1 files changed, 5 insertions, 5 deletions

diff --git a/media/libstagefright/codecs/on2/h264dec/source/h264bsd_intra_prediction.c b/media/libstagefright/codecs/on2/h264dec/source/h264bsd_intra_prediction.c
index 15eabfb..52c85e5 100755
--- a/media/libstagefright/codecs/on2/h264dec/source/h264bsd_intra_prediction.c
+++ b/media/libstagefright/codecs/on2/h264dec/source/h264bsd_intra_prediction.c
@@ -1110,7 +1110,7 @@ void Intra16x16PlanePrediction(u8 *data, u8 *above, u8 *left)
 
 /* Variables */
 
-    u32 i, j;
+    i32 i, j;
     i32 a, b, c;
     i32 tmp;
 
@@ -1123,20 +1123,20 @@ void Intra16x16PlanePrediction(u8 *data, u8 *above, u8 *left)
     a = 16 * (above[15] + left[15]);
 
     for (i = 0, b = 0; i < 8; i++)
-        b += ((i32)i + 1) * (above[8+i] - above[6-i]);
+        b += (i + 1) * (above[8+i] - above[6-i]);
     b = (5 * b + 32) >> 6;
 
     for (i = 0, c = 0; i < 7; i++)
-        c += ((i32)i + 1) * (left[8+i] - left[6-i]);
+        c += (i + 1) * (left[8+i] - left[6-i]);
     /* p[-1,-1] has to be accessed through above pointer */
-    c += ((i32)i + 1) * (left[8+i] - above[-1]);
+    c += (i + 1) * (left[8+i] - above[-1]);
     c = (5 * c + 32) >> 6;
 
     for (i = 0; i < 16; i++)
     {
         for (j = 0; j < 16; j++)
         {
-            tmp = (a + b * ((i32)j - 7) + c * ((i32)i - 7) + 16) >> 5;
+            tmp = (a + b * (j - 7) + c * (i - 7) + 16) >> 5;
             data[i*16+j] = (u8)CLIP1(tmp);
         }
     }