From 1f797f960412b5a50bf8e92a0c8b460f30e947d7 Mon Sep 17 00:00:00 2001 From: Marcus Oakland Date: Mon, 17 Feb 2014 08:28:25 +0000 Subject: AArch64: Correction for LP64 In the Intra16x16PlanePrediction function, a u32 (unsigned 32-bit integer) was being used for the "i" variable, and being used with a value of 7 in the loop "for (i = 0, b = 0; i < 8; i++)" to access the "above[6-i]" location where "above" is defined as "u8 *". Because "i" was unsigned there was no sign extension on use with the __LP64__ 64-bit pointer, so rather than 0xFFFFFFFF being treated as -1, it was treated as 4,292,967,295 and that resulted in a SIGSEGV at an invalid address. By changing the type of "i" to an i32 (signed 32-bit integer) the expected sign extension occurs and the value is treated as -1. This change fixes android.media.cts.DecoderTest#testCodeBasicH264 on 64-bit platforms Change-Id: I85df58b0dc1d39f89ab421d04ab5481356520f0c Signed-off-by: Marcus Oakland --- .../codecs/on2/h264dec/source/h264bsd_intra_prediction.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/media/libstagefright/codecs/on2/h264dec/source/h264bsd_intra_prediction.c b/media/libstagefright/codecs/on2/h264dec/source/h264bsd_intra_prediction.c index 15eabfb..52c85e5 100755 --- a/media/libstagefright/codecs/on2/h264dec/source/h264bsd_intra_prediction.c +++ b/media/libstagefright/codecs/on2/h264dec/source/h264bsd_intra_prediction.c @@ -1110,7 +1110,7 @@ void Intra16x16PlanePrediction(u8 *data, u8 *above, u8 *left) /* Variables */ - u32 i, j; + i32 i, j; i32 a, b, c; i32 tmp; @@ -1123,20 +1123,20 @@ void Intra16x16PlanePrediction(u8 *data, u8 *above, u8 *left) a = 16 * (above[15] + left[15]); for (i = 0, b = 0; i < 8; i++) - b += ((i32)i + 1) * (above[8+i] - above[6-i]); + b += (i + 1) * (above[8+i] - above[6-i]); b = (5 * b + 32) >> 6; for (i = 0, c = 0; i < 7; i++) - c += ((i32)i + 1) * (left[8+i] - left[6-i]); + c += (i + 1) * (left[8+i] - left[6-i]); /* p[-1,-1] has to be accessed through above pointer */ - c += ((i32)i + 1) * (left[8+i] - above[-1]); + c += (i + 1) * (left[8+i] - above[-1]); c = (5 * c + 32) >> 6; for (i = 0; i < 16; i++) { for (j = 0; j < 16; j++) { - tmp = (a + b * ((i32)j - 7) + c * ((i32)i - 7) + 16) >> 5; + tmp = (a + b * (j - 7) + c * (i - 7) + 16) >> 5; data[i*16+j] = (u8)CLIP1(tmp); } } -- cgit v1.1