diff options
author | Hans Boehm <hboehm@google.com> | 2014-10-05 16:04:54 +0000 |
---|---|---|
committer | Hans Boehm <hboehm@google.com> | 2014-11-13 15:59:10 -0800 |
commit | c56648948c93ba6223770755609fafeab2c23cb5 (patch) | |
tree | bc17a0217cbc9a3dd0815f25159afcfe909a636f /include | |
parent | 06279285ae38b58988a9431c89b8689d8e16904e (diff) | |
download | system_core-c56648948c93ba6223770755609fafeab2c23cb5.zip system_core-c56648948c93ba6223770755609fafeab2c23cb5.tar.gz system_core-c56648948c93ba6223770755609fafeab2c23cb5.tar.bz2 |
Second attempt to "Rewrite atomic.h to use stdatomic.h."
This should not be committed until win_sdk and aarch64 builds are
fixed in the presence of this CL.
This reverts commit 2789faabfa9cdf16902a52a1de3fc4a7ae98efed.
We additionally remove uniprocessor support from the earlier CL,
thus avoiding a potential compiler code reordering issue.
Change-Id: I7207a5ca2efa907a6f757f172d7090a62b2311fe
Diffstat (limited to 'include')
-rw-r--r-- | include/cutils/atomic-arm.h | 167 | ||||
-rw-r--r-- | include/cutils/atomic-arm64.h | 156 | ||||
-rw-r--r-- | include/cutils/atomic-inline.h | 59 | ||||
-rw-r--r-- | include/cutils/atomic-mips.h | 174 | ||||
-rw-r--r-- | include/cutils/atomic-mips64.h | 166 | ||||
-rw-r--r-- | include/cutils/atomic-x86.h | 145 | ||||
-rw-r--r-- | include/cutils/atomic-x86_64.h | 151 | ||||
-rw-r--r-- | include/cutils/atomic.h | 133 |
8 files changed, 112 insertions, 1039 deletions
diff --git a/include/cutils/atomic-arm.h b/include/cutils/atomic-arm.h deleted file mode 100644 index 6b031b6..0000000 --- a/include/cutils/atomic-arm.h +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Copyright (C) 2010 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef ANDROID_CUTILS_ATOMIC_ARM_H -#define ANDROID_CUTILS_ATOMIC_ARM_H - -#include <stdint.h> - -#ifndef ANDROID_ATOMIC_INLINE -#define ANDROID_ATOMIC_INLINE inline __attribute__((always_inline)) -#endif - -extern ANDROID_ATOMIC_INLINE void android_compiler_barrier() -{ - __asm__ __volatile__ ("" : : : "memory"); -} - -extern ANDROID_ATOMIC_INLINE void android_memory_barrier() -{ -#if ANDROID_SMP == 0 - android_compiler_barrier(); -#else - __asm__ __volatile__ ("dmb" : : : "memory"); -#endif -} - -extern ANDROID_ATOMIC_INLINE -int32_t android_atomic_acquire_load(volatile const int32_t *ptr) -{ - int32_t value = *ptr; - android_memory_barrier(); - return value; -} - -extern ANDROID_ATOMIC_INLINE -int32_t android_atomic_release_load(volatile const int32_t *ptr) -{ - android_memory_barrier(); - return *ptr; -} - -extern ANDROID_ATOMIC_INLINE -void android_atomic_acquire_store(int32_t value, volatile int32_t *ptr) -{ - *ptr = value; - android_memory_barrier(); -} - -extern ANDROID_ATOMIC_INLINE -void android_atomic_release_store(int32_t value, volatile int32_t *ptr) -{ - android_memory_barrier(); - *ptr = value; -} - -extern ANDROID_ATOMIC_INLINE -int android_atomic_cas(int32_t old_value, int32_t new_value, - volatile int32_t *ptr) -{ - int32_t prev, status; - do { - __asm__ __volatile__ ("ldrex %0, [%3]\n" - "mov %1, #0\n" - "teq %0, %4\n" -#ifdef __thumb2__ - "it eq\n" -#endif - "strexeq %1, %5, [%3]" - : "=&r" (prev), "=&r" (status), "+m"(*ptr) - : "r" (ptr), "Ir" (old_value), "r" (new_value) - : "cc"); - } while (__builtin_expect(status != 0, 0)); - return prev != old_value; -} - -extern ANDROID_ATOMIC_INLINE -int android_atomic_acquire_cas(int32_t old_value, int32_t new_value, - volatile int32_t *ptr) -{ - int status = android_atomic_cas(old_value, new_value, ptr); - android_memory_barrier(); - return status; -} - -extern ANDROID_ATOMIC_INLINE -int android_atomic_release_cas(int32_t old_value, int32_t new_value, - volatile int32_t *ptr) -{ - android_memory_barrier(); - return android_atomic_cas(old_value, new_value, ptr); -} - -extern ANDROID_ATOMIC_INLINE -int32_t android_atomic_add(int32_t increment, volatile int32_t *ptr) -{ - int32_t prev, tmp, status; - android_memory_barrier(); - do { - __asm__ __volatile__ ("ldrex %0, [%4]\n" - "add %1, %0, %5\n" - "strex %2, %1, [%4]" - : "=&r" (prev), "=&r" (tmp), - "=&r" (status), "+m" (*ptr) - : "r" (ptr), "Ir" (increment) - : "cc"); - } while (__builtin_expect(status != 0, 0)); - return prev; -} - -extern ANDROID_ATOMIC_INLINE int32_t android_atomic_inc(volatile int32_t *addr) -{ - return android_atomic_add(1, addr); -} - -extern ANDROID_ATOMIC_INLINE int32_t android_atomic_dec(volatile int32_t *addr) -{ - return android_atomic_add(-1, addr); -} - -extern ANDROID_ATOMIC_INLINE -int32_t android_atomic_and(int32_t value, volatile int32_t *ptr) -{ - int32_t prev, tmp, status; - android_memory_barrier(); - do { - __asm__ __volatile__ ("ldrex %0, [%4]\n" - "and %1, %0, %5\n" - "strex %2, %1, [%4]" - : "=&r" (prev), "=&r" (tmp), - "=&r" (status), "+m" (*ptr) - : "r" (ptr), "Ir" (value) - : "cc"); - } while (__builtin_expect(status != 0, 0)); - return prev; -} - -extern ANDROID_ATOMIC_INLINE -int32_t android_atomic_or(int32_t value, volatile int32_t *ptr) -{ - int32_t prev, tmp, status; - android_memory_barrier(); - do { - __asm__ __volatile__ ("ldrex %0, [%4]\n" - "orr %1, %0, %5\n" - "strex %2, %1, [%4]" - : "=&r" (prev), "=&r" (tmp), - "=&r" (status), "+m" (*ptr) - : "r" (ptr), "Ir" (value) - : "cc"); - } while (__builtin_expect(status != 0, 0)); - return prev; -} - -#endif /* ANDROID_CUTILS_ATOMIC_ARM_H */ diff --git a/include/cutils/atomic-arm64.h b/include/cutils/atomic-arm64.h deleted file mode 100644 index 7ae47d7..0000000 --- a/include/cutils/atomic-arm64.h +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS - * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#ifndef ANDROID_CUTILS_ATOMIC_AARCH64_H -#define ANDROID_CUTILS_ATOMIC_AARCH64_H - -#include <stdint.h> - -#ifndef ANDROID_ATOMIC_INLINE -#define ANDROID_ATOMIC_INLINE inline __attribute__((always_inline)) -#endif - -/* - TODOAArch64: Revisit the below functions and check for potential - optimizations using assembly code or otherwise. -*/ - -extern ANDROID_ATOMIC_INLINE -void android_compiler_barrier(void) -{ - __asm__ __volatile__ ("" : : : "memory"); -} - -extern ANDROID_ATOMIC_INLINE -void android_memory_barrier(void) -{ - __asm__ __volatile__ ("dmb ish" : : : "memory"); -} - -extern ANDROID_ATOMIC_INLINE -int32_t android_atomic_acquire_load(volatile const int32_t *ptr) -{ - int32_t value = *ptr; - android_memory_barrier(); - return value; -} - -extern ANDROID_ATOMIC_INLINE -int32_t android_atomic_release_load(volatile const int32_t *ptr) -{ - android_memory_barrier(); - return *ptr; -} - -extern ANDROID_ATOMIC_INLINE -void android_atomic_acquire_store(int32_t value, volatile int32_t *ptr) -{ - *ptr = value; - android_memory_barrier(); -} - -extern ANDROID_ATOMIC_INLINE -void android_atomic_release_store(int32_t value, volatile int32_t *ptr) -{ - android_memory_barrier(); - *ptr = value; -} - -extern ANDROID_ATOMIC_INLINE -int android_atomic_cas(int32_t old_value, int32_t new_value, - volatile int32_t *ptr) -{ - return __sync_val_compare_and_swap(ptr, old_value, new_value) != old_value; -} - -extern ANDROID_ATOMIC_INLINE -int android_atomic_acquire_cas(int32_t old_value, int32_t new_value, - volatile int32_t *ptr) -{ - int status = android_atomic_cas(old_value, new_value, ptr); - android_memory_barrier(); - return status; -} - -extern ANDROID_ATOMIC_INLINE -int android_atomic_release_cas(int32_t old_value, int32_t new_value, - volatile int32_t *ptr) -{ - android_memory_barrier(); - return android_atomic_cas(old_value, new_value, ptr); -} - -extern ANDROID_ATOMIC_INLINE -int32_t android_atomic_add(int32_t increment, volatile int32_t *ptr) -{ - int32_t prev, status; - android_memory_barrier(); - do { - prev = *ptr; - status = android_atomic_cas(prev, prev + increment, ptr); - } while (__builtin_expect(status != 0, 0)); - return prev; -} - -extern ANDROID_ATOMIC_INLINE -int32_t android_atomic_inc(volatile int32_t *addr) -{ - return android_atomic_add(1, addr); -} - -extern ANDROID_ATOMIC_INLINE -int32_t android_atomic_dec(volatile int32_t *addr) -{ - return android_atomic_add(-1, addr); -} - -extern ANDROID_ATOMIC_INLINE -int32_t android_atomic_and(int32_t value, volatile int32_t *ptr) -{ - int32_t prev, status; - android_memory_barrier(); - do { - prev = *ptr; - status = android_atomic_cas(prev, prev & value, ptr); - } while (__builtin_expect(status != 0, 0)); - return prev; -} - -extern ANDROID_ATOMIC_INLINE -int32_t android_atomic_or(int32_t value, volatile int32_t *ptr) -{ - int32_t prev, status; - android_memory_barrier(); - do { - prev = *ptr; - status = android_atomic_cas(prev, prev | value, ptr); - } while (__builtin_expect(status != 0, 0)); - return prev; -} - -#endif /* ANDROID_CUTILS_ATOMIC_AARCH64_H */ diff --git a/include/cutils/atomic-inline.h b/include/cutils/atomic-inline.h index a31e913..103c5b0 100644 --- a/include/cutils/atomic-inline.h +++ b/include/cutils/atomic-inline.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010 The Android Open Source Project + * Copyright (C) 2014 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,59 +14,6 @@ * limitations under the License. */ -#ifndef ANDROID_CUTILS_ATOMIC_INLINE_H -#define ANDROID_CUTILS_ATOMIC_INLINE_H - -#ifdef __cplusplus -extern "C" { +#ifndef ANDROID_CUTILS_ATOMIC_H +#include <cutils/atomic.h> #endif - -/* - * Inline declarations and macros for some special-purpose atomic - * operations. These are intended for rare circumstances where a - * memory barrier needs to be issued inline rather than as a function - * call. - * - * Most code should not use these. - * - * Anything that does include this file must set ANDROID_SMP to either - * 0 or 1, indicating compilation for UP or SMP, respectively. - * - * Macros defined in this header: - * - * void ANDROID_MEMBAR_FULL(void) - * Full memory barrier. Provides a compiler reordering barrier, and - * on SMP systems emits an appropriate instruction. - */ - -#if !defined(ANDROID_SMP) -# error "Must define ANDROID_SMP before including atomic-inline.h" -#endif - -#if defined(__aarch64__) -#include <cutils/atomic-arm64.h> -#elif defined(__arm__) -#include <cutils/atomic-arm.h> -#elif defined(__i386__) -#include <cutils/atomic-x86.h> -#elif defined(__x86_64__) -#include <cutils/atomic-x86_64.h> -#elif defined(__mips64) -#include <cutils/atomic-mips64.h> -#elif defined(__mips__) -#include <cutils/atomic-mips.h> -#else -#error atomic operations are unsupported -#endif - -#if ANDROID_SMP == 0 -#define ANDROID_MEMBAR_FULL android_compiler_barrier -#else -#define ANDROID_MEMBAR_FULL android_memory_barrier -#endif - -#ifdef __cplusplus -} -#endif - -#endif /* ANDROID_CUTILS_ATOMIC_INLINE_H */ diff --git a/include/cutils/atomic-mips.h b/include/cutils/atomic-mips.h deleted file mode 100644 index 5d4f097..0000000 --- a/include/cutils/atomic-mips.h +++ /dev/null @@ -1,174 +0,0 @@ -/* - * Copyright (C) 2010 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef ANDROID_CUTILS_ATOMIC_MIPS_H -#define ANDROID_CUTILS_ATOMIC_MIPS_H - -#include <stdint.h> - -#ifndef ANDROID_ATOMIC_INLINE -#define ANDROID_ATOMIC_INLINE inline __attribute__((always_inline)) -#endif - -extern ANDROID_ATOMIC_INLINE void android_compiler_barrier(void) -{ - __asm__ __volatile__ ("" : : : "memory"); -} - -#if ANDROID_SMP == 0 -extern ANDROID_ATOMIC_INLINE void android_memory_barrier(void) -{ - android_compiler_barrier(); -} -#else -extern ANDROID_ATOMIC_INLINE void android_memory_barrier(void) -{ - __asm__ __volatile__ ("sync" : : : "memory"); -} -#endif - -extern ANDROID_ATOMIC_INLINE int32_t -android_atomic_acquire_load(volatile const int32_t *ptr) -{ - int32_t value = *ptr; - android_memory_barrier(); - return value; -} - -extern ANDROID_ATOMIC_INLINE int32_t -android_atomic_release_load(volatile const int32_t *ptr) -{ - android_memory_barrier(); - return *ptr; -} - -extern ANDROID_ATOMIC_INLINE void -android_atomic_acquire_store(int32_t value, volatile int32_t *ptr) -{ - *ptr = value; - android_memory_barrier(); -} - -extern ANDROID_ATOMIC_INLINE void -android_atomic_release_store(int32_t value, volatile int32_t *ptr) -{ - android_memory_barrier(); - *ptr = value; -} - -extern ANDROID_ATOMIC_INLINE int -android_atomic_cas(int32_t old_value, int32_t new_value, volatile int32_t *ptr) -{ - int32_t prev, status; - do { - __asm__ __volatile__ ( - " ll %[prev], (%[ptr])\n" - " li %[status], 1\n" - " bne %[prev], %[old], 9f\n" - " move %[status], %[new_value]\n" - " sc %[status], (%[ptr])\n" - "9:\n" - : [prev] "=&r" (prev), [status] "=&r" (status) - : [ptr] "r" (ptr), [old] "r" (old_value), [new_value] "r" (new_value) - ); - } while (__builtin_expect(status == 0, 0)); - return prev != old_value; -} - -extern ANDROID_ATOMIC_INLINE int -android_atomic_acquire_cas(int32_t old_value, - int32_t new_value, - volatile int32_t *ptr) -{ - int status = android_atomic_cas(old_value, new_value, ptr); - android_memory_barrier(); - return status; -} - -extern ANDROID_ATOMIC_INLINE int -android_atomic_release_cas(int32_t old_value, - int32_t new_value, - volatile int32_t *ptr) -{ - android_memory_barrier(); - return android_atomic_cas(old_value, new_value, ptr); -} - - -extern ANDROID_ATOMIC_INLINE int32_t -android_atomic_add(int32_t increment, volatile int32_t *ptr) -{ - int32_t prev, status; - android_memory_barrier(); - do { - __asm__ __volatile__ ( - " ll %[prev], (%[ptr])\n" - " addu %[status], %[prev], %[inc]\n" - " sc %[status], (%[ptr])\n" - : [status] "=&r" (status), [prev] "=&r" (prev) - : [ptr] "r" (ptr), [inc] "Ir" (increment) - ); - } while (__builtin_expect(status == 0, 0)); - return prev; -} - -extern ANDROID_ATOMIC_INLINE int32_t -android_atomic_inc(volatile int32_t *addr) -{ - return android_atomic_add(1, addr); -} - -extern ANDROID_ATOMIC_INLINE int32_t -android_atomic_dec(volatile int32_t *addr) -{ - return android_atomic_add(-1, addr); -} - -extern ANDROID_ATOMIC_INLINE int32_t -android_atomic_and(int32_t value, volatile int32_t *ptr) -{ - int32_t prev, status; - android_memory_barrier(); - do { - __asm__ __volatile__ ( - " ll %[prev], (%[ptr])\n" - " and %[status], %[prev], %[value]\n" - " sc %[status], (%[ptr])\n" - : [prev] "=&r" (prev), [status] "=&r" (status) - : [ptr] "r" (ptr), [value] "Ir" (value) - ); - } while (__builtin_expect(status == 0, 0)); - return prev; -} - -extern ANDROID_ATOMIC_INLINE int32_t -android_atomic_or(int32_t value, volatile int32_t *ptr) -{ - int32_t prev, status; - android_memory_barrier(); - do { - __asm__ __volatile__ ( - " ll %[prev], (%[ptr])\n" - " or %[status], %[prev], %[value]\n" - " sc %[status], (%[ptr])\n" - : [prev] "=&r" (prev), [status] "=&r" (status) - : [ptr] "r" (ptr), [value] "Ir" (value) - ); - } while (__builtin_expect(status == 0, 0)); - return prev; -} - -#endif /* ANDROID_CUTILS_ATOMIC_MIPS_H */ diff --git a/include/cutils/atomic-mips64.h b/include/cutils/atomic-mips64.h deleted file mode 100644 index 9d8f65e..0000000 --- a/include/cutils/atomic-mips64.h +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Copyright (C) 2010 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef ANDROID_CUTILS_ATOMIC_MIPS64_H -#define ANDROID_CUTILS_ATOMIC_MIPS64_H - -#include <stdint.h> - -#ifndef ANDROID_ATOMIC_INLINE -#define ANDROID_ATOMIC_INLINE inline __attribute__((always_inline)) -#endif - -extern ANDROID_ATOMIC_INLINE void android_compiler_barrier(void) -{ - __asm__ __volatile__ ("" : : : "memory"); -} - -extern ANDROID_ATOMIC_INLINE void android_memory_barrier(void) -{ - __asm__ __volatile__ ("sync" : : : "memory"); -} - -extern ANDROID_ATOMIC_INLINE -int32_t android_atomic_acquire_load(volatile const int32_t *ptr) -{ - int32_t value = *ptr; - android_memory_barrier(); - return value; -} - -extern ANDROID_ATOMIC_INLINE -int32_t android_atomic_release_load(volatile const int32_t *ptr) -{ - android_memory_barrier(); - return *ptr; -} - -extern ANDROID_ATOMIC_INLINE -void android_atomic_acquire_store(int32_t value, volatile int32_t *ptr) -{ - *ptr = value; - android_memory_barrier(); -} - -extern ANDROID_ATOMIC_INLINE -void android_atomic_release_store(int32_t value, volatile int32_t *ptr) -{ - android_memory_barrier(); - *ptr = value; -} - -extern ANDROID_ATOMIC_INLINE -int android_atomic_cas(int32_t old_value, int32_t new_value, volatile int32_t *ptr) -{ - int32_t prev, status; - do { - __asm__ __volatile__ ( - " ll %[prev], (%[ptr])\n" - " li %[status], 1\n" - " bne %[prev], %[old], 9f\n" - " move %[status], %[new_value]\n" - " sc %[status], (%[ptr])\n" - "9:\n" - : [prev] "=&r" (prev), [status] "=&r" (status) - : [ptr] "r" (ptr), [old] "r" (old_value), [new_value] "r" (new_value) - ); - } while (__builtin_expect(status == 0, 0)); - return prev != old_value; -} - -extern ANDROID_ATOMIC_INLINE -int android_atomic_acquire_cas(int32_t old_value, - int32_t new_value, - volatile int32_t *ptr) -{ - int status = android_atomic_cas(old_value, new_value, ptr); - android_memory_barrier(); - return status; -} - -extern ANDROID_ATOMIC_INLINE -int android_atomic_release_cas(int32_t old_value, - int32_t new_value, - volatile int32_t *ptr) -{ - android_memory_barrier(); - return android_atomic_cas(old_value, new_value, ptr); -} - -extern ANDROID_ATOMIC_INLINE -int32_t android_atomic_add(int32_t increment, volatile int32_t *ptr) -{ - int32_t prev, status; - android_memory_barrier(); - do { - __asm__ __volatile__ ( - " ll %[prev], (%[ptr])\n" - " addu %[status], %[prev], %[inc]\n" - " sc %[status], (%[ptr])\n" - : [status] "=&r" (status), [prev] "=&r" (prev) - : [ptr] "r" (ptr), [inc] "Ir" (increment) - ); - } while (__builtin_expect(status == 0, 0)); - return prev; -} - -extern ANDROID_ATOMIC_INLINE int32_t -android_atomic_inc(volatile int32_t *addr) -{ - return android_atomic_add(1, addr); -} - -extern ANDROID_ATOMIC_INLINE int32_t -android_atomic_dec(volatile int32_t *addr) -{ - return android_atomic_add(-1, addr); -} - -extern ANDROID_ATOMIC_INLINE int32_t -android_atomic_and(int32_t value, volatile int32_t *ptr) -{ - int32_t prev, status; - android_memory_barrier(); - do { - __asm__ __volatile__ ( - " ll %[prev], (%[ptr])\n" - " and %[status], %[prev], %[value]\n" - " sc %[status], (%[ptr])\n" - : [prev] "=&r" (prev), [status] "=&r" (status) - : [ptr] "r" (ptr), [value] "Ir" (value) - ); - } while (__builtin_expect(status == 0, 0)); - return prev; -} - -extern ANDROID_ATOMIC_INLINE int32_t -android_atomic_or(int32_t value, volatile int32_t *ptr) -{ - int32_t prev, status; - android_memory_barrier(); - do { - __asm__ __volatile__ ( - " ll %[prev], (%[ptr])\n" - " or %[status], %[prev], %[value]\n" - " sc %[status], (%[ptr])\n" - : [prev] "=&r" (prev), [status] "=&r" (status) - : [ptr] "r" (ptr), [value] "Ir" (value) - ); - } while (__builtin_expect(status == 0, 0)); - return prev; -} - -#endif /* ANDROID_CUTILS_ATOMIC_MIPS_H */ diff --git a/include/cutils/atomic-x86.h b/include/cutils/atomic-x86.h deleted file mode 100644 index 06bf1a3..0000000 --- a/include/cutils/atomic-x86.h +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Copyright (C) 2010 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef ANDROID_CUTILS_ATOMIC_X86_H -#define ANDROID_CUTILS_ATOMIC_X86_H - -#include <stdint.h> - -#ifndef ANDROID_ATOMIC_INLINE -#define ANDROID_ATOMIC_INLINE inline __attribute__((always_inline)) -#endif - -extern ANDROID_ATOMIC_INLINE void android_compiler_barrier(void) -{ - __asm__ __volatile__ ("" : : : "memory"); -} - -#if ANDROID_SMP == 0 -extern ANDROID_ATOMIC_INLINE void android_memory_barrier(void) -{ - android_compiler_barrier(); -} -#else -extern ANDROID_ATOMIC_INLINE void android_memory_barrier(void) -{ - __asm__ __volatile__ ("mfence" : : : "memory"); -} -#endif - -extern ANDROID_ATOMIC_INLINE int32_t -android_atomic_acquire_load(volatile const int32_t *ptr) -{ - int32_t value = *ptr; - android_compiler_barrier(); - return value; -} - -extern ANDROID_ATOMIC_INLINE int32_t -android_atomic_release_load(volatile const int32_t *ptr) -{ - android_memory_barrier(); - return *ptr; -} - -extern ANDROID_ATOMIC_INLINE void -android_atomic_acquire_store(int32_t value, volatile int32_t *ptr) -{ - *ptr = value; - android_memory_barrier(); -} - -extern ANDROID_ATOMIC_INLINE void -android_atomic_release_store(int32_t value, volatile int32_t *ptr) -{ - android_compiler_barrier(); - *ptr = value; -} - -extern ANDROID_ATOMIC_INLINE int -android_atomic_cas(int32_t old_value, int32_t new_value, volatile int32_t *ptr) -{ - int32_t prev; - __asm__ __volatile__ ("lock; cmpxchgl %1, %2" - : "=a" (prev) - : "q" (new_value), "m" (*ptr), "0" (old_value) - : "memory"); - return prev != old_value; -} - -extern ANDROID_ATOMIC_INLINE int -android_atomic_acquire_cas(int32_t old_value, - int32_t new_value, - volatile int32_t *ptr) -{ - /* Loads are not reordered with other loads. */ - return android_atomic_cas(old_value, new_value, ptr); -} - -extern ANDROID_ATOMIC_INLINE int -android_atomic_release_cas(int32_t old_value, - int32_t new_value, - volatile int32_t *ptr) -{ - /* Stores are not reordered with other stores. */ - return android_atomic_cas(old_value, new_value, ptr); -} - -extern ANDROID_ATOMIC_INLINE int32_t -android_atomic_add(int32_t increment, volatile int32_t *ptr) -{ - __asm__ __volatile__ ("lock; xaddl %0, %1" - : "+r" (increment), "+m" (*ptr) - : : "memory"); - /* increment now holds the old value of *ptr */ - return increment; -} - -extern ANDROID_ATOMIC_INLINE int32_t -android_atomic_inc(volatile int32_t *addr) -{ - return android_atomic_add(1, addr); -} - -extern ANDROID_ATOMIC_INLINE int32_t -android_atomic_dec(volatile int32_t *addr) -{ - return android_atomic_add(-1, addr); -} - -extern ANDROID_ATOMIC_INLINE int32_t -android_atomic_and(int32_t value, volatile int32_t *ptr) -{ - int32_t prev, status; - do { - prev = *ptr; - status = android_atomic_cas(prev, prev & value, ptr); - } while (__builtin_expect(status != 0, 0)); - return prev; -} - -extern ANDROID_ATOMIC_INLINE int32_t -android_atomic_or(int32_t value, volatile int32_t *ptr) -{ - int32_t prev, status; - do { - prev = *ptr; - status = android_atomic_cas(prev, prev | value, ptr); - } while (__builtin_expect(status != 0, 0)); - return prev; -} - -#endif /* ANDROID_CUTILS_ATOMIC_X86_H */ diff --git a/include/cutils/atomic-x86_64.h b/include/cutils/atomic-x86_64.h deleted file mode 100644 index 99cb070..0000000 --- a/include/cutils/atomic-x86_64.h +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS - * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#ifndef ANDROID_CUTILS_ATOMIC_X86_64_H -#define ANDROID_CUTILS_ATOMIC_X86_64_H - -#include <stdint.h> - -#ifndef ANDROID_ATOMIC_INLINE -#define ANDROID_ATOMIC_INLINE inline __attribute__((always_inline)) -#endif - -extern ANDROID_ATOMIC_INLINE -void android_compiler_barrier(void) -{ - __asm__ __volatile__ ("" : : : "memory"); -} - -extern ANDROID_ATOMIC_INLINE -void android_memory_barrier(void) -{ - __asm__ __volatile__ ("mfence" : : : "memory"); -} - -extern ANDROID_ATOMIC_INLINE -int32_t android_atomic_acquire_load(volatile const int32_t *ptr) -{ - int32_t value = *ptr; - android_compiler_barrier(); - return value; -} - -extern ANDROID_ATOMIC_INLINE -int32_t android_atomic_release_load(volatile const int32_t *ptr) -{ - android_memory_barrier(); - return *ptr; -} - -extern ANDROID_ATOMIC_INLINE -void android_atomic_acquire_store(int32_t value, volatile int32_t *ptr) -{ - *ptr = value; - android_memory_barrier(); -} - -extern ANDROID_ATOMIC_INLINE -void android_atomic_release_store(int32_t value, volatile int32_t *ptr) -{ - android_compiler_barrier(); - *ptr = value; -} - -extern ANDROID_ATOMIC_INLINE -int android_atomic_cas(int32_t old_value, int32_t new_value, - volatile int32_t *ptr) -{ - int32_t prev; - __asm__ __volatile__ ("lock; cmpxchgl %1, %2" - : "=a" (prev) - : "q" (new_value), "m" (*ptr), "0" (old_value) - : "memory"); - return prev != old_value; -} - -extern ANDROID_ATOMIC_INLINE -int android_atomic_acquire_cas(int32_t old_value, int32_t new_value, - volatile int32_t *ptr) -{ - /* Loads are not reordered with other loads. */ - return android_atomic_cas(old_value, new_value, ptr); -} - -extern ANDROID_ATOMIC_INLINE -int android_atomic_release_cas(int32_t old_value, int32_t new_value, - volatile int32_t *ptr) -{ - /* Stores are not reordered with other stores. */ - return android_atomic_cas(old_value, new_value, ptr); -} - -extern ANDROID_ATOMIC_INLINE -int32_t android_atomic_add(int32_t increment, volatile int32_t *ptr) -{ - __asm__ __volatile__ ("lock; xaddl %0, %1" - : "+r" (increment), "+m" (*ptr) - : : "memory"); - /* increment now holds the old value of *ptr */ - return increment; -} - -extern ANDROID_ATOMIC_INLINE -int32_t android_atomic_inc(volatile int32_t *addr) -{ - return android_atomic_add(1, addr); -} - -extern ANDROID_ATOMIC_INLINE -int32_t android_atomic_dec(volatile int32_t *addr) -{ - return android_atomic_add(-1, addr); -} - -extern ANDROID_ATOMIC_INLINE -int32_t android_atomic_and(int32_t value, volatile int32_t *ptr) -{ - int32_t prev, status; - do { - prev = *ptr; - status = android_atomic_cas(prev, prev & value, ptr); - } while (__builtin_expect(status != 0, 0)); - return prev; -} - -extern ANDROID_ATOMIC_INLINE -int32_t android_atomic_or(int32_t value, volatile int32_t *ptr) -{ - int32_t prev, status; - do { - prev = *ptr; - status = android_atomic_cas(prev, prev | value, ptr); - } while (__builtin_expect(status != 0, 0)); - return prev; -} - -#endif /* ANDROID_CUTILS_ATOMIC_X86_64_H */ diff --git a/include/cutils/atomic.h b/include/cutils/atomic.h index 79409a7..ded972a 100644 --- a/include/cutils/atomic.h +++ b/include/cutils/atomic.h @@ -19,9 +19,10 @@ #include <stdint.h> #include <sys/types.h> +#include <stdatomic.h> -#ifdef __cplusplus -extern "C" { +#ifndef ANDROID_ATOMIC_INLINE +#define ANDROID_ATOMIC_INLINE static inline #endif /* @@ -77,11 +78,41 @@ extern "C" { * These have the same characteristics (e.g. what happens on overflow) * as the equivalent non-atomic C operations. */ -int32_t android_atomic_inc(volatile int32_t* addr); -int32_t android_atomic_dec(volatile int32_t* addr); -int32_t android_atomic_add(int32_t value, volatile int32_t* addr); -int32_t android_atomic_and(int32_t value, volatile int32_t* addr); -int32_t android_atomic_or(int32_t value, volatile int32_t* addr); +ANDROID_ATOMIC_INLINE +int32_t android_atomic_inc(volatile int32_t* addr) +{ + volatile atomic_int_least32_t* a = (volatile atomic_int_least32_t*)addr; + /* Int32_t, if it exists, is the same as int_least32_t. */ + return atomic_fetch_add_explicit(a, 1, memory_order_release); +} + +ANDROID_ATOMIC_INLINE +int32_t android_atomic_dec(volatile int32_t* addr) +{ + volatile atomic_int_least32_t* a = (volatile atomic_int_least32_t*)addr; + return atomic_fetch_sub_explicit(a, 1, memory_order_release); +} + +ANDROID_ATOMIC_INLINE +int32_t android_atomic_add(int32_t value, volatile int32_t* addr) +{ + volatile atomic_int_least32_t* a = (volatile atomic_int_least32_t*)addr; + return atomic_fetch_add_explicit(a, value, memory_order_release); +} + +ANDROID_ATOMIC_INLINE +int32_t android_atomic_and(int32_t value, volatile int32_t* addr) +{ + volatile atomic_int_least32_t* a = (volatile atomic_int_least32_t*)addr; + return atomic_fetch_and_explicit(a, value, memory_order_release); +} + +ANDROID_ATOMIC_INLINE +int32_t android_atomic_or(int32_t value, volatile int32_t* addr) +{ + volatile atomic_int_least32_t* a = (volatile atomic_int_least32_t*)addr; + return atomic_fetch_or_explicit(a, value, memory_order_release); +} /* * Perform an atomic load with "acquire" or "release" ordering. @@ -96,29 +127,53 @@ int32_t android_atomic_or(int32_t value, volatile int32_t* addr); * this comment, you are in the vast majority, and should not be * using release loads or replacing them with anything other than * locks or default sequentially consistent atomics. - * - * This is only necessary if you need the memory barrier. A 32-bit read - * from a 32-bit aligned address is atomic on all supported platforms. */ -int32_t android_atomic_acquire_load(volatile const int32_t* addr); -int32_t android_atomic_release_load(volatile const int32_t* addr); +ANDROID_ATOMIC_INLINE +int32_t android_atomic_acquire_load(volatile const int32_t* addr) +{ + volatile atomic_int_least32_t* a = (volatile atomic_int_least32_t*)addr; + return atomic_load_explicit(a, memory_order_acquire); +} + +ANDROID_ATOMIC_INLINE +int32_t android_atomic_release_load(volatile const int32_t* addr) +{ + volatile atomic_int_least32_t* a = (volatile atomic_int_least32_t*)addr; + atomic_thread_fence(memory_order_seq_cst); + /* Any reasonable clients of this interface would probably prefer */ + /* something weaker. But some remaining clients seem to be */ + /* abusing this API in strange ways, e.g. by using it as a fence. */ + /* Thus we are conservative until we can get rid of remaining */ + /* clients (and this function). */ + return atomic_load_explicit(a, memory_order_relaxed); +} /* * Perform an atomic store with "acquire" or "release" ordering. * - * Note that the notion of a "acquire" ordering for a store does not + * Note that the notion of an "acquire" ordering for a store does not * really fit into the C11 or C++11 memory model. The extra ordering * is normally observable only by code using memory_order_relaxed * atomics, or data races. In the rare cases in which such ordering * is called for, use memory_order_relaxed atomics and a trailing * atomic_thread_fence (typically with memory_order_release, * not memory_order_acquire!) instead. - * - * This is only necessary if you need the memory barrier. A 32-bit write - * to a 32-bit aligned address is atomic on all supported platforms. */ -void android_atomic_acquire_store(int32_t value, volatile int32_t* addr); -void android_atomic_release_store(int32_t value, volatile int32_t* addr); +ANDROID_ATOMIC_INLINE +void android_atomic_acquire_store(int32_t value, volatile int32_t* addr) +{ + volatile atomic_int_least32_t* a = (volatile atomic_int_least32_t*)addr; + atomic_store_explicit(a, value, memory_order_relaxed); + atomic_thread_fence(memory_order_seq_cst); + /* Again overly conservative to accomodate weird clients. */ +} + +ANDROID_ATOMIC_INLINE +void android_atomic_release_store(int32_t value, volatile int32_t* addr) +{ + volatile atomic_int_least32_t* a = (volatile atomic_int_least32_t*)addr; + atomic_store_explicit(a, value, memory_order_release); +} /* * Compare-and-set operation with "acquire" or "release" ordering. @@ -132,10 +187,44 @@ void android_atomic_release_store(int32_t value, volatile int32_t* addr); * Implementations that use the release CAS in a loop may be less efficient * than possible, because we re-issue the memory barrier on each iteration. */ +ANDROID_ATOMIC_INLINE int android_atomic_acquire_cas(int32_t oldvalue, int32_t newvalue, - volatile int32_t* addr); + volatile int32_t* addr) +{ + volatile atomic_int_least32_t* a = (volatile atomic_int_least32_t*)addr; + return (int)(!atomic_compare_exchange_strong_explicit( + a, &oldvalue, newvalue, + memory_order_acquire, + memory_order_acquire)); +} + +ANDROID_ATOMIC_INLINE int android_atomic_release_cas(int32_t oldvalue, int32_t newvalue, - volatile int32_t* addr); + volatile int32_t* addr) +{ + volatile atomic_int_least32_t* a = (volatile atomic_int_least32_t*)addr; + return (int)(!atomic_compare_exchange_strong_explicit( + a, &oldvalue, newvalue, + memory_order_release, + memory_order_relaxed)); +} + +/* + * Fence primitives. + */ +ANDROID_ATOMIC_INLINE +void android_compiler_barrier(void) +{ + __asm__ __volatile__ ("" : : : "memory"); + /* Could probably also be: */ + /* atomic_signal_fence(memory_order_seq_cst); */ +} + +ANDROID_ATOMIC_INLINE +void android_memory_barrier(void) +{ + atomic_thread_fence(memory_order_seq_cst); +} /* * Aliases for code using an older version of this header. These are now @@ -145,8 +234,4 @@ int android_atomic_release_cas(int32_t oldvalue, int32_t newvalue, #define android_atomic_write android_atomic_release_store #define android_atomic_cmpxchg android_atomic_release_cas -#ifdef __cplusplus -} // extern "C" -#endif - #endif // ANDROID_CUTILS_ATOMIC_H |