From 86a1600fb6291e3ba31d1366aa3ab8bff7d87dd7 Mon Sep 17 00:00:00 2001 From: Henrik Smiding Date: Fri, 16 May 2014 13:26:08 +0200 Subject: Port libcutils memset16/32 to x86_64. Port libcutils memset16/32 assembly SSE2 optimizations to x86_64 architecture. Ensures the same performance on 64-bit arch. Change-Id: I874a71a884c0d28a152933ddff9cb886c9a6e99e Signed-off-by: Henrik Smiding --- libcutils/Android.mk | 43 +- libcutils/arch-x86_64/android_memset16_SSE2-atom.S | 564 +++++++++++++++++++++ libcutils/arch-x86_64/android_memset32_SSE2-atom.S | 372 ++++++++++++++ libcutils/arch-x86_64/cache.h | 31 ++ 4 files changed, 994 insertions(+), 16 deletions(-) create mode 100644 libcutils/arch-x86_64/android_memset16_SSE2-atom.S create mode 100644 libcutils/arch-x86_64/android_memset32_SSE2-atom.S create mode 100644 libcutils/arch-x86_64/cache.h diff --git a/libcutils/Android.mk b/libcutils/Android.mk index ca1fd15..c0faed4 100644 --- a/libcutils/Android.mk +++ b/libcutils/Android.mk @@ -82,6 +82,7 @@ ifneq ($(HOST_OS),windows) LOCAL_CFLAGS += -Werror endif LOCAL_MULTILIB := both +LOCAL_ADDITIONAL_DEPENDENCIES := $(LOCAL_PATH)/Android.mk include $(BUILD_HOST_STATIC_LIBRARY) @@ -95,6 +96,7 @@ LOCAL_CFLAGS += $(hostSmpFlag) -m64 ifneq ($(HOST_OS),windows) LOCAL_CFLAGS += -Werror endif +LOCAL_ADDITIONAL_DEPENDENCIES := $(LOCAL_PATH)/Android.mk include $(BUILD_HOST_STATIC_LIBRARY) # Tests for host @@ -108,6 +110,7 @@ endif LOCAL_SRC_FILES := str_parms.c hashmap.c memory.c LOCAL_STATIC_LIBRARIES := liblog LOCAL_MODULE_TAGS := optional +LOCAL_ADDITIONAL_DEPENDENCIES := $(LOCAL_PATH)/Android.mk include $(BUILD_HOST_EXECUTABLE) @@ -121,30 +124,36 @@ LOCAL_SRC_FILES := $(commonSources) \ ashmem-dev.c \ debugger.c \ klog.c \ + memory.c \ partition_utils.c \ properties.c \ qtaguid.c \ trace.c \ - uevent.c - -ifeq ($(TARGET_ARCH),arm) - LOCAL_SRC_FILES += arch-arm/memset32.S -else # !arm - ifeq ($(TARGET_ARCH),x86) - LOCAL_CFLAGS += -DHAVE_MEMSET16 -DHAVE_MEMSET32 - LOCAL_SRC_FILES += arch-x86/android_memset16.S arch-x86/android_memset32.S memory.c - else # !x86 - ifeq ($(TARGET_ARCH),mips) - LOCAL_SRC_FILES += arch-mips/android_memset.c - else # !mips - LOCAL_SRC_FILES += memory.c - endif # !mips - endif # !x86 -endif # !arm + uevent.c \ + +LOCAL_SRC_FILES_arm += \ + arch-arm/memset32.S \ + +LOCAL_SRC_FILES_mips += \ + arch-mips/android_memset.c \ + +LOCAL_SRC_FILES_x86 += \ + arch-x86/android_memset16.S \ + arch-x86/android_memset32.S \ + +LOCAL_SRC_FILES_x86_64 += \ + arch-x86_64/android_memset16_SSE2-atom.S \ + arch-x86_64/android_memset32_SSE2-atom.S \ + +LOCAL_CFLAGS_arm += -DHAVE_MEMSET16 -DHAVE_MEMSET32 +LOCAL_CFLAGS_mips += -DHAVE_MEMSET16 -DHAVE_MEMSET32 +LOCAL_CFLAGS_x86 += -DHAVE_MEMSET16 -DHAVE_MEMSET32 +LOCAL_CFLAGS_x86_64 += -DHAVE_MEMSET16 -DHAVE_MEMSET32 LOCAL_C_INCLUDES := $(libcutils_c_includes) LOCAL_STATIC_LIBRARIES := liblog LOCAL_CFLAGS += $(targetSmpFlag) -Werror +LOCAL_ADDITIONAL_DEPENDENCIES := $(LOCAL_PATH)/Android.mk include $(BUILD_STATIC_LIBRARY) include $(CLEAR_VARS) @@ -155,6 +164,7 @@ LOCAL_WHOLE_STATIC_LIBRARIES := libcutils liblog LOCAL_SHARED_LIBRARIES := liblog LOCAL_CFLAGS += $(targetSmpFlag) -Werror LOCAL_C_INCLUDES := $(libcutils_c_includes) +LOCAL_ADDITIONAL_DEPENDENCIES := $(LOCAL_PATH)/Android.mk include $(BUILD_SHARED_LIBRARY) include $(CLEAR_VARS) @@ -163,6 +173,7 @@ LOCAL_CFLAGS += -DTEST_STR_PARMS -Werror LOCAL_SRC_FILES := str_parms.c hashmap.c memory.c LOCAL_SHARED_LIBRARIES := liblog LOCAL_MODULE_TAGS := optional +LOCAL_ADDITIONAL_DEPENDENCIES := $(LOCAL_PATH)/Android.mk include $(BUILD_EXECUTABLE) include $(call all-makefiles-under,$(LOCAL_PATH)) diff --git a/libcutils/arch-x86_64/android_memset16_SSE2-atom.S b/libcutils/arch-x86_64/android_memset16_SSE2-atom.S new file mode 100644 index 0000000..48a10ed --- /dev/null +++ b/libcutils/arch-x86_64/android_memset16_SSE2-atom.S @@ -0,0 +1,564 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + * Contributed by: Intel Corporation + */ + +#include "cache.h" + +#ifndef L +# define L(label) .L##label +#endif + +#ifndef ALIGN +# define ALIGN(n) .p2align n +#endif + +#ifndef cfi_startproc +# define cfi_startproc .cfi_startproc +#endif + +#ifndef cfi_endproc +# define cfi_endproc .cfi_endproc +#endif + +#ifndef ENTRY +# define ENTRY(name) \ + .type name, @function; \ + .globl name; \ + .p2align 4; \ +name: \ + cfi_startproc +#endif + +#ifndef END +# define END(name) \ + cfi_endproc; \ + .size name, .-name +#endif + +#define JMPTBL(I, B) I - B + +/* Branch to an entry in a jump table. TABLE is a jump table with + relative offsets. INDEX is a register contains the index into the + jump table. SCALE is the scale of INDEX. */ +#define BRANCH_TO_JMPTBL_ENTRY(TABLE, INDEX, SCALE) \ + lea TABLE(%rip), %r11; \ + movslq (%r11, INDEX, SCALE), INDEX; \ + lea (%r11, INDEX), INDEX; \ + jmp *INDEX + + .section .text.sse2,"ax",@progbits + ALIGN (4) +ENTRY (android_memset16) // Address in rdi + shr $1, %rdx // Count in rdx + movzwl %si, %ecx + /* Fill the whole ECX with pattern. */ + shl $16, %esi + or %esi, %ecx // Pattern in ecx + + cmp $32, %rdx + jae L(32wordsormore) + +L(write_less32words): + lea (%rdi, %rdx, 2), %rdi + BRANCH_TO_JMPTBL_ENTRY (L(table_less32words), %rdx, 4) + + .pushsection .rodata.sse2,"a",@progbits + ALIGN (2) +L(table_less32words): + .int JMPTBL (L(write_0words), L(table_less32words)) + .int JMPTBL (L(write_1words), L(table_less32words)) + .int JMPTBL (L(write_2words), L(table_less32words)) + .int JMPTBL (L(write_3words), L(table_less32words)) + .int JMPTBL (L(write_4words), L(table_less32words)) + .int JMPTBL (L(write_5words), L(table_less32words)) + .int JMPTBL (L(write_6words), L(table_less32words)) + .int JMPTBL (L(write_7words), L(table_less32words)) + .int JMPTBL (L(write_8words), L(table_less32words)) + .int JMPTBL (L(write_9words), L(table_less32words)) + .int JMPTBL (L(write_10words), L(table_less32words)) + .int JMPTBL (L(write_11words), L(table_less32words)) + .int JMPTBL (L(write_12words), L(table_less32words)) + .int JMPTBL (L(write_13words), L(table_less32words)) + .int JMPTBL (L(write_14words), L(table_less32words)) + .int JMPTBL (L(write_15words), L(table_less32words)) + .int JMPTBL (L(write_16words), L(table_less32words)) + .int JMPTBL (L(write_17words), L(table_less32words)) + .int JMPTBL (L(write_18words), L(table_less32words)) + .int JMPTBL (L(write_19words), L(table_less32words)) + .int JMPTBL (L(write_20words), L(table_less32words)) + .int JMPTBL (L(write_21words), L(table_less32words)) + .int JMPTBL (L(write_22words), L(table_less32words)) + .int JMPTBL (L(write_23words), L(table_less32words)) + .int JMPTBL (L(write_24words), L(table_less32words)) + .int JMPTBL (L(write_25words), L(table_less32words)) + .int JMPTBL (L(write_26words), L(table_less32words)) + .int JMPTBL (L(write_27words), L(table_less32words)) + .int JMPTBL (L(write_28words), L(table_less32words)) + .int JMPTBL (L(write_29words), L(table_less32words)) + .int JMPTBL (L(write_30words), L(table_less32words)) + .int JMPTBL (L(write_31words), L(table_less32words)) + .popsection + + ALIGN (4) +L(write_28words): + movl %ecx, -56(%rdi) + movl %ecx, -52(%rdi) +L(write_24words): + movl %ecx, -48(%rdi) + movl %ecx, -44(%rdi) +L(write_20words): + movl %ecx, -40(%rdi) + movl %ecx, -36(%rdi) +L(write_16words): + movl %ecx, -32(%rdi) + movl %ecx, -28(%rdi) +L(write_12words): + movl %ecx, -24(%rdi) + movl %ecx, -20(%rdi) +L(write_8words): + movl %ecx, -16(%rdi) + movl %ecx, -12(%rdi) +L(write_4words): + movl %ecx, -8(%rdi) + movl %ecx, -4(%rdi) +L(write_0words): + ret + + ALIGN (4) +L(write_29words): + movl %ecx, -58(%rdi) + movl %ecx, -54(%rdi) +L(write_25words): + movl %ecx, -50(%rdi) + movl %ecx, -46(%rdi) +L(write_21words): + movl %ecx, -42(%rdi) + movl %ecx, -38(%rdi) +L(write_17words): + movl %ecx, -34(%rdi) + movl %ecx, -30(%rdi) +L(write_13words): + movl %ecx, -26(%rdi) + movl %ecx, -22(%rdi) +L(write_9words): + movl %ecx, -18(%rdi) + movl %ecx, -14(%rdi) +L(write_5words): + movl %ecx, -10(%rdi) + movl %ecx, -6(%rdi) +L(write_1words): + mov %cx, -2(%rdi) + ret + + ALIGN (4) +L(write_30words): + movl %ecx, -60(%rdi) + movl %ecx, -56(%rdi) +L(write_26words): + movl %ecx, -52(%rdi) + movl %ecx, -48(%rdi) +L(write_22words): + movl %ecx, -44(%rdi) + movl %ecx, -40(%rdi) +L(write_18words): + movl %ecx, -36(%rdi) + movl %ecx, -32(%rdi) +L(write_14words): + movl %ecx, -28(%rdi) + movl %ecx, -24(%rdi) +L(write_10words): + movl %ecx, -20(%rdi) + movl %ecx, -16(%rdi) +L(write_6words): + movl %ecx, -12(%rdi) + movl %ecx, -8(%rdi) +L(write_2words): + movl %ecx, -4(%rdi) + ret + + ALIGN (4) +L(write_31words): + movl %ecx, -62(%rdi) + movl %ecx, -58(%rdi) +L(write_27words): + movl %ecx, -54(%rdi) + movl %ecx, -50(%rdi) +L(write_23words): + movl %ecx, -46(%rdi) + movl %ecx, -42(%rdi) +L(write_19words): + movl %ecx, -38(%rdi) + movl %ecx, -34(%rdi) +L(write_15words): + movl %ecx, -30(%rdi) + movl %ecx, -26(%rdi) +L(write_11words): + movl %ecx, -22(%rdi) + movl %ecx, -18(%rdi) +L(write_7words): + movl %ecx, -14(%rdi) + movl %ecx, -10(%rdi) +L(write_3words): + movl %ecx, -6(%rdi) + movw %cx, -2(%rdi) + ret + + ALIGN (4) +L(32wordsormore): + shl $1, %rdx + test $0x01, %edi + jz L(aligned2bytes) + mov %ecx, (%rdi) + mov %ecx, -4(%rdi, %rdx) + sub $2, %rdx + add $1, %rdi + rol $8, %ecx +L(aligned2bytes): + /* Fill xmm0 with the pattern. */ + movd %ecx, %xmm0 + pshufd $0, %xmm0, %xmm0 + + testl $0xf, %edi + jz L(aligned_16) +/* RDX > 32 and RDI is not 16 byte aligned. */ + movdqu %xmm0, (%rdi) + mov %rdi, %rsi + and $-16, %rdi + add $16, %rdi + sub %rdi, %rsi + add %rsi, %rdx + + ALIGN (4) +L(aligned_16): + cmp $128, %rdx + jge L(128bytesormore) + +L(aligned_16_less128bytes): + add %rdx, %rdi + shr $1, %rdx + BRANCH_TO_JMPTBL_ENTRY (L(table_16_128bytes), %rdx, 4) + + ALIGN (4) +L(128bytesormore): + cmp $SHARED_CACHE_SIZE, %rdx + jg L(128bytesormore_nt) + +L(128bytesormore_normal): + sub $128, %rdx + movdqa %xmm0, (%rdi) + movdqa %xmm0, 0x10(%rdi) + movdqa %xmm0, 0x20(%rdi) + movdqa %xmm0, 0x30(%rdi) + movdqa %xmm0, 0x40(%rdi) + movdqa %xmm0, 0x50(%rdi) + movdqa %xmm0, 0x60(%rdi) + movdqa %xmm0, 0x70(%rdi) + lea 128(%rdi), %rdi + cmp $128, %rdx + jl L(128bytesless_normal) + + sub $128, %rdx + movdqa %xmm0, (%rdi) + movdqa %xmm0, 0x10(%rdi) + movdqa %xmm0, 0x20(%rdi) + movdqa %xmm0, 0x30(%rdi) + movdqa %xmm0, 0x40(%rdi) + movdqa %xmm0, 0x50(%rdi) + movdqa %xmm0, 0x60(%rdi) + movdqa %xmm0, 0x70(%rdi) + lea 128(%rdi), %rdi + cmp $128, %rdx + jl L(128bytesless_normal) + + sub $128, %rdx + movdqa %xmm0, (%rdi) + movdqa %xmm0, 0x10(%rdi) + movdqa %xmm0, 0x20(%rdi) + movdqa %xmm0, 0x30(%rdi) + movdqa %xmm0, 0x40(%rdi) + movdqa %xmm0, 0x50(%rdi) + movdqa %xmm0, 0x60(%rdi) + movdqa %xmm0, 0x70(%rdi) + lea 128(%rdi), %rdi + cmp $128, %rdx + jl L(128bytesless_normal) + + sub $128, %rdx + movdqa %xmm0, (%rdi) + movdqa %xmm0, 0x10(%rdi) + movdqa %xmm0, 0x20(%rdi) + movdqa %xmm0, 0x30(%rdi) + movdqa %xmm0, 0x40(%rdi) + movdqa %xmm0, 0x50(%rdi) + movdqa %xmm0, 0x60(%rdi) + movdqa %xmm0, 0x70(%rdi) + lea 128(%rdi), %rdi + cmp $128, %rdx + jge L(128bytesormore_normal) + +L(128bytesless_normal): + add %rdx, %rdi + shr $1, %rdx + BRANCH_TO_JMPTBL_ENTRY (L(table_16_128bytes), %rdx, 4) + + ALIGN (4) +L(128bytesormore_nt): + sub $128, %rdx + movntdq %xmm0, (%rdi) + movntdq %xmm0, 0x10(%rdi) + movntdq %xmm0, 0x20(%rdi) + movntdq %xmm0, 0x30(%rdi) + movntdq %xmm0, 0x40(%rdi) + movntdq %xmm0, 0x50(%rdi) + movntdq %xmm0, 0x60(%rdi) + movntdq %xmm0, 0x70(%rdi) + lea 128(%rdi), %rdi + cmp $128, %rdx + jge L(128bytesormore_nt) + + sfence + add %rdx, %rdi + shr $1, %rdx + BRANCH_TO_JMPTBL_ENTRY (L(table_16_128bytes), %rdx, 4) + + .pushsection .rodata.sse2,"a",@progbits + ALIGN (2) +L(table_16_128bytes): + .int JMPTBL (L(aligned_16_0bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_2bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_4bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_6bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_8bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_10bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_12bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_14bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_16bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_18bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_20bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_22bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_24bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_26bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_28bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_30bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_32bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_34bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_36bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_38bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_40bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_42bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_44bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_46bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_48bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_50bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_52bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_54bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_56bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_58bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_60bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_62bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_64bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_66bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_68bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_70bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_72bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_74bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_76bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_78bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_80bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_82bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_84bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_86bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_88bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_90bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_92bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_94bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_96bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_98bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_100bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_102bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_104bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_106bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_108bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_110bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_112bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_114bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_116bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_118bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_120bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_122bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_124bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_126bytes), L(table_16_128bytes)) + .popsection + + ALIGN (4) +L(aligned_16_112bytes): + movdqa %xmm0, -112(%rdi) +L(aligned_16_96bytes): + movdqa %xmm0, -96(%rdi) +L(aligned_16_80bytes): + movdqa %xmm0, -80(%rdi) +L(aligned_16_64bytes): + movdqa %xmm0, -64(%rdi) +L(aligned_16_48bytes): + movdqa %xmm0, -48(%rdi) +L(aligned_16_32bytes): + movdqa %xmm0, -32(%rdi) +L(aligned_16_16bytes): + movdqa %xmm0, -16(%rdi) +L(aligned_16_0bytes): + ret + + ALIGN (4) +L(aligned_16_114bytes): + movdqa %xmm0, -114(%rdi) +L(aligned_16_98bytes): + movdqa %xmm0, -98(%rdi) +L(aligned_16_82bytes): + movdqa %xmm0, -82(%rdi) +L(aligned_16_66bytes): + movdqa %xmm0, -66(%rdi) +L(aligned_16_50bytes): + movdqa %xmm0, -50(%rdi) +L(aligned_16_34bytes): + movdqa %xmm0, -34(%rdi) +L(aligned_16_18bytes): + movdqa %xmm0, -18(%rdi) +L(aligned_16_2bytes): + movw %cx, -2(%rdi) + ret + + ALIGN (4) +L(aligned_16_116bytes): + movdqa %xmm0, -116(%rdi) +L(aligned_16_100bytes): + movdqa %xmm0, -100(%rdi) +L(aligned_16_84bytes): + movdqa %xmm0, -84(%rdi) +L(aligned_16_68bytes): + movdqa %xmm0, -68(%rdi) +L(aligned_16_52bytes): + movdqa %xmm0, -52(%rdi) +L(aligned_16_36bytes): + movdqa %xmm0, -36(%rdi) +L(aligned_16_20bytes): + movdqa %xmm0, -20(%rdi) +L(aligned_16_4bytes): + movl %ecx, -4(%rdi) + ret + + ALIGN (4) +L(aligned_16_118bytes): + movdqa %xmm0, -118(%rdi) +L(aligned_16_102bytes): + movdqa %xmm0, -102(%rdi) +L(aligned_16_86bytes): + movdqa %xmm0, -86(%rdi) +L(aligned_16_70bytes): + movdqa %xmm0, -70(%rdi) +L(aligned_16_54bytes): + movdqa %xmm0, -54(%rdi) +L(aligned_16_38bytes): + movdqa %xmm0, -38(%rdi) +L(aligned_16_22bytes): + movdqa %xmm0, -22(%rdi) +L(aligned_16_6bytes): + movl %ecx, -6(%rdi) + movw %cx, -2(%rdi) + ret + + ALIGN (4) +L(aligned_16_120bytes): + movdqa %xmm0, -120(%rdi) +L(aligned_16_104bytes): + movdqa %xmm0, -104(%rdi) +L(aligned_16_88bytes): + movdqa %xmm0, -88(%rdi) +L(aligned_16_72bytes): + movdqa %xmm0, -72(%rdi) +L(aligned_16_56bytes): + movdqa %xmm0, -56(%rdi) +L(aligned_16_40bytes): + movdqa %xmm0, -40(%rdi) +L(aligned_16_24bytes): + movdqa %xmm0, -24(%rdi) +L(aligned_16_8bytes): + movq %xmm0, -8(%rdi) + ret + + ALIGN (4) +L(aligned_16_122bytes): + movdqa %xmm0, -122(%rdi) +L(aligned_16_106bytes): + movdqa %xmm0, -106(%rdi) +L(aligned_16_90bytes): + movdqa %xmm0, -90(%rdi) +L(aligned_16_74bytes): + movdqa %xmm0, -74(%rdi) +L(aligned_16_58bytes): + movdqa %xmm0, -58(%rdi) +L(aligned_16_42bytes): + movdqa %xmm0, -42(%rdi) +L(aligned_16_26bytes): + movdqa %xmm0, -26(%rdi) +L(aligned_16_10bytes): + movq %xmm0, -10(%rdi) + movw %cx, -2(%rdi) + ret + + ALIGN (4) +L(aligned_16_124bytes): + movdqa %xmm0, -124(%rdi) +L(aligned_16_108bytes): + movdqa %xmm0, -108(%rdi) +L(aligned_16_92bytes): + movdqa %xmm0, -92(%rdi) +L(aligned_16_76bytes): + movdqa %xmm0, -76(%rdi) +L(aligned_16_60bytes): + movdqa %xmm0, -60(%rdi) +L(aligned_16_44bytes): + movdqa %xmm0, -44(%rdi) +L(aligned_16_28bytes): + movdqa %xmm0, -28(%rdi) +L(aligned_16_12bytes): + movq %xmm0, -12(%rdi) + movl %ecx, -4(%rdi) + ret + + ALIGN (4) +L(aligned_16_126bytes): + movdqa %xmm0, -126(%rdi) +L(aligned_16_110bytes): + movdqa %xmm0, -110(%rdi) +L(aligned_16_94bytes): + movdqa %xmm0, -94(%rdi) +L(aligned_16_78bytes): + movdqa %xmm0, -78(%rdi) +L(aligned_16_62bytes): + movdqa %xmm0, -62(%rdi) +L(aligned_16_46bytes): + movdqa %xmm0, -46(%rdi) +L(aligned_16_30bytes): + movdqa %xmm0, -30(%rdi) +L(aligned_16_14bytes): + movq %xmm0, -14(%rdi) + movl %ecx, -6(%rdi) + movw %cx, -2(%rdi) + ret + +END (android_memset16) diff --git a/libcutils/arch-x86_64/android_memset32_SSE2-atom.S b/libcutils/arch-x86_64/android_memset32_SSE2-atom.S new file mode 100644 index 0000000..4bdea8e --- /dev/null +++ b/libcutils/arch-x86_64/android_memset32_SSE2-atom.S @@ -0,0 +1,372 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + * Contributed by: Intel Corporation + */ + +#include "cache.h" + +#ifndef L +# define L(label) .L##label +#endif + +#ifndef ALIGN +# define ALIGN(n) .p2align n +#endif + +#ifndef cfi_startproc +# define cfi_startproc .cfi_startproc +#endif + +#ifndef cfi_endproc +# define cfi_endproc .cfi_endproc +#endif + +#ifndef ENTRY +# define ENTRY(name) \ + .type name, @function; \ + .globl name; \ + .p2align 4; \ +name: \ + cfi_startproc +#endif + +#ifndef END +# define END(name) \ + cfi_endproc; \ + .size name, .-name +#endif + +#define JMPTBL(I, B) I - B + +/* Branch to an entry in a jump table. TABLE is a jump table with + relative offsets. INDEX is a register contains the index into the + jump table. SCALE is the scale of INDEX. */ +#define BRANCH_TO_JMPTBL_ENTRY(TABLE, INDEX, SCALE) \ + lea TABLE(%rip), %r11; \ + movslq (%r11, INDEX, SCALE), INDEX; \ + lea (%r11, INDEX), INDEX; \ + jmp *INDEX + + .section .text.sse2,"ax",@progbits + ALIGN (4) +ENTRY (android_memset32) // Address in rdi + shr $2, %rdx // Count in rdx + movl %esi, %ecx // Pattern in ecx + + cmp $16, %rdx + jae L(16dbwordsormore) + +L(write_less16dbwords): + lea (%rdi, %rdx, 4), %rdi + BRANCH_TO_JMPTBL_ENTRY (L(table_less16dbwords), %rdx, 4) + + .pushsection .rodata.sse2,"a",@progbits + ALIGN (2) +L(table_less16dbwords): + .int JMPTBL (L(write_0dbwords), L(table_less16dbwords)) + .int JMPTBL (L(write_1dbwords), L(table_less16dbwords)) + .int JMPTBL (L(write_2dbwords), L(table_less16dbwords)) + .int JMPTBL (L(write_3dbwords), L(table_less16dbwords)) + .int JMPTBL (L(write_4dbwords), L(table_less16dbwords)) + .int JMPTBL (L(write_5dbwords), L(table_less16dbwords)) + .int JMPTBL (L(write_6dbwords), L(table_less16dbwords)) + .int JMPTBL (L(write_7dbwords), L(table_less16dbwords)) + .int JMPTBL (L(write_8dbwords), L(table_less16dbwords)) + .int JMPTBL (L(write_9dbwords), L(table_less16dbwords)) + .int JMPTBL (L(write_10dbwords), L(table_less16dbwords)) + .int JMPTBL (L(write_11dbwords), L(table_less16dbwords)) + .int JMPTBL (L(write_12dbwords), L(table_less16dbwords)) + .int JMPTBL (L(write_13dbwords), L(table_less16dbwords)) + .int JMPTBL (L(write_14dbwords), L(table_less16dbwords)) + .int JMPTBL (L(write_15dbwords), L(table_less16dbwords)) + .popsection + + ALIGN (4) +L(write_15dbwords): + movl %ecx, -60(%rdi) +L(write_14dbwords): + movl %ecx, -56(%rdi) +L(write_13dbwords): + movl %ecx, -52(%rdi) +L(write_12dbwords): + movl %ecx, -48(%rdi) +L(write_11dbwords): + movl %ecx, -44(%rdi) +L(write_10dbwords): + movl %ecx, -40(%rdi) +L(write_9dbwords): + movl %ecx, -36(%rdi) +L(write_8dbwords): + movl %ecx, -32(%rdi) +L(write_7dbwords): + movl %ecx, -28(%rdi) +L(write_6dbwords): + movl %ecx, -24(%rdi) +L(write_5dbwords): + movl %ecx, -20(%rdi) +L(write_4dbwords): + movl %ecx, -16(%rdi) +L(write_3dbwords): + movl %ecx, -12(%rdi) +L(write_2dbwords): + movl %ecx, -8(%rdi) +L(write_1dbwords): + movl %ecx, -4(%rdi) +L(write_0dbwords): + ret + + ALIGN (4) +L(16dbwordsormore): + test $3, %edi + jz L(aligned4bytes) + mov %ecx, (%rdi) + mov %ecx, -4(%rdi, %rdx, 4) + sub $1, %rdx + rol $24, %ecx + add $1, %rdi + test $3, %edi + jz L(aligned4bytes) + ror $8, %ecx + add $1, %rdi + test $3, %edi + jz L(aligned4bytes) + ror $8, %ecx + add $1, %rdi +L(aligned4bytes): + shl $2, %rdx + + /* Fill xmm0 with the pattern. */ + movd %ecx, %xmm0 + pshufd $0, %xmm0, %xmm0 + + testl $0xf, %edi + jz L(aligned_16) +/* RDX > 32 and RDI is not 16 byte aligned. */ + movdqu %xmm0, (%rdi) + mov %rdi, %rsi + and $-16, %rdi + add $16, %rdi + sub %rdi, %rsi + add %rsi, %rdx + + ALIGN (4) +L(aligned_16): + cmp $128, %rdx + jge L(128bytesormore) + +L(aligned_16_less128bytes): + add %rdx, %rdi + shr $2, %rdx + BRANCH_TO_JMPTBL_ENTRY (L(table_16_128bytes), %rdx, 4) + + ALIGN (4) +L(128bytesormore): + cmp $SHARED_CACHE_SIZE, %rdx + jg L(128bytesormore_nt) + +L(128bytesormore_normal): + sub $128, %rdx + movdqa %xmm0, (%rdi) + movdqa %xmm0, 0x10(%rdi) + movdqa %xmm0, 0x20(%rdi) + movdqa %xmm0, 0x30(%rdi) + movdqa %xmm0, 0x40(%rdi) + movdqa %xmm0, 0x50(%rdi) + movdqa %xmm0, 0x60(%rdi) + movdqa %xmm0, 0x70(%rdi) + lea 128(%rdi), %rdi + cmp $128, %rdx + jl L(128bytesless_normal) + + sub $128, %rdx + movdqa %xmm0, (%rdi) + movdqa %xmm0, 0x10(%rdi) + movdqa %xmm0, 0x20(%rdi) + movdqa %xmm0, 0x30(%rdi) + movdqa %xmm0, 0x40(%rdi) + movdqa %xmm0, 0x50(%rdi) + movdqa %xmm0, 0x60(%rdi) + movdqa %xmm0, 0x70(%rdi) + lea 128(%rdi), %rdi + cmp $128, %rdx + jl L(128bytesless_normal) + + sub $128, %rdx + movdqa %xmm0, (%rdi) + movdqa %xmm0, 0x10(%rdi) + movdqa %xmm0, 0x20(%rdi) + movdqa %xmm0, 0x30(%rdi) + movdqa %xmm0, 0x40(%rdi) + movdqa %xmm0, 0x50(%rdi) + movdqa %xmm0, 0x60(%rdi) + movdqa %xmm0, 0x70(%rdi) + lea 128(%rdi), %rdi + cmp $128, %rdx + jl L(128bytesless_normal) + + sub $128, %rdx + movdqa %xmm0, (%rdi) + movdqa %xmm0, 0x10(%rdi) + movdqa %xmm0, 0x20(%rdi) + movdqa %xmm0, 0x30(%rdi) + movdqa %xmm0, 0x40(%rdi) + movdqa %xmm0, 0x50(%rdi) + movdqa %xmm0, 0x60(%rdi) + movdqa %xmm0, 0x70(%rdi) + lea 128(%rdi), %rdi + cmp $128, %rdx + jge L(128bytesormore_normal) + +L(128bytesless_normal): + add %rdx, %rdi + shr $2, %rdx + BRANCH_TO_JMPTBL_ENTRY (L(table_16_128bytes), %rdx, 4) + + ALIGN (4) +L(128bytesormore_nt): + sub $128, %rdx + movntdq %xmm0, (%rdi) + movntdq %xmm0, 0x10(%rdi) + movntdq %xmm0, 0x20(%rdi) + movntdq %xmm0, 0x30(%rdi) + movntdq %xmm0, 0x40(%rdi) + movntdq %xmm0, 0x50(%rdi) + movntdq %xmm0, 0x60(%rdi) + movntdq %xmm0, 0x70(%rdi) + lea 128(%rdi), %rdi + cmp $128, %rdx + jge L(128bytesormore_nt) + + sfence + add %rdx, %rdi + shr $2, %rdx + BRANCH_TO_JMPTBL_ENTRY (L(table_16_128bytes), %rdx, 4) + + .pushsection .rodata.sse2,"a",@progbits + ALIGN (2) +L(table_16_128bytes): + .int JMPTBL (L(aligned_16_0bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_4bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_8bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_12bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_16bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_20bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_24bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_28bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_32bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_36bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_40bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_44bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_48bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_52bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_56bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_60bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_64bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_68bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_72bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_76bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_80bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_84bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_88bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_92bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_96bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_100bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_104bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_108bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_112bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_116bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_120bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_124bytes), L(table_16_128bytes)) + .popsection + + ALIGN (4) +L(aligned_16_112bytes): + movdqa %xmm0, -112(%rdi) +L(aligned_16_96bytes): + movdqa %xmm0, -96(%rdi) +L(aligned_16_80bytes): + movdqa %xmm0, -80(%rdi) +L(aligned_16_64bytes): + movdqa %xmm0, -64(%rdi) +L(aligned_16_48bytes): + movdqa %xmm0, -48(%rdi) +L(aligned_16_32bytes): + movdqa %xmm0, -32(%rdi) +L(aligned_16_16bytes): + movdqa %xmm0, -16(%rdi) +L(aligned_16_0bytes): + ret + + ALIGN (4) +L(aligned_16_116bytes): + movdqa %xmm0, -116(%rdi) +L(aligned_16_100bytes): + movdqa %xmm0, -100(%rdi) +L(aligned_16_84bytes): + movdqa %xmm0, -84(%rdi) +L(aligned_16_68bytes): + movdqa %xmm0, -68(%rdi) +L(aligned_16_52bytes): + movdqa %xmm0, -52(%rdi) +L(aligned_16_36bytes): + movdqa %xmm0, -36(%rdi) +L(aligned_16_20bytes): + movdqa %xmm0, -20(%rdi) +L(aligned_16_4bytes): + movl %ecx, -4(%rdi) + ret + + ALIGN (4) +L(aligned_16_120bytes): + movdqa %xmm0, -120(%rdi) +L(aligned_16_104bytes): + movdqa %xmm0, -104(%rdi) +L(aligned_16_88bytes): + movdqa %xmm0, -88(%rdi) +L(aligned_16_72bytes): + movdqa %xmm0, -72(%rdi) +L(aligned_16_56bytes): + movdqa %xmm0, -56(%rdi) +L(aligned_16_40bytes): + movdqa %xmm0, -40(%rdi) +L(aligned_16_24bytes): + movdqa %xmm0, -24(%rdi) +L(aligned_16_8bytes): + movq %xmm0, -8(%rdi) + ret + + ALIGN (4) +L(aligned_16_124bytes): + movdqa %xmm0, -124(%rdi) +L(aligned_16_108bytes): + movdqa %xmm0, -108(%rdi) +L(aligned_16_92bytes): + movdqa %xmm0, -92(%rdi) +L(aligned_16_76bytes): + movdqa %xmm0, -76(%rdi) +L(aligned_16_60bytes): + movdqa %xmm0, -60(%rdi) +L(aligned_16_44bytes): + movdqa %xmm0, -44(%rdi) +L(aligned_16_28bytes): + movdqa %xmm0, -28(%rdi) +L(aligned_16_12bytes): + movq %xmm0, -12(%rdi) + movl %ecx, -4(%rdi) + ret + +END (android_memset32) diff --git a/libcutils/arch-x86_64/cache.h b/libcutils/arch-x86_64/cache.h new file mode 100644 index 0000000..ab5dd2f --- /dev/null +++ b/libcutils/arch-x86_64/cache.h @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + * Contributed by: Intel Corporation + */ + +#if defined(__slm__) +/* Values are optimized for Silvermont */ +#define SHARED_CACHE_SIZE (1024*1024) /* Silvermont L2 Cache */ +#define DATA_CACHE_SIZE (24*1024) /* Silvermont L1 Data Cache */ +#else +/* Values are optimized for Atom */ +#define SHARED_CACHE_SIZE (512*1024) /* Atom L2 Cache */ +#define DATA_CACHE_SIZE (24*1024) /* Atom L1 Data Cache */ +#endif + +#define SHARED_CACHE_SIZE_HALF (SHARED_CACHE_SIZE / 2) +#define DATA_CACHE_SIZE_HALF (DATA_CACHE_SIZE / 2) -- cgit v1.1