From nobody Thu Oct 07 03:28:11 2021 X-Original-To: dev-commits-src-branches@mlmmj.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mlmmj.nyi.freebsd.org (Postfix) with ESMTP id E1EA412DDF2F; Thu, 7 Oct 2021 03:28:12 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256 client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "R3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 4HPxbw0WkNz4qj4; Thu, 7 Oct 2021 03:28:12 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from gitrepo.freebsd.org (gitrepo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:5]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id B308646AB; Thu, 7 Oct 2021 03:28:11 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from gitrepo.freebsd.org ([127.0.1.44]) by gitrepo.freebsd.org (8.16.1/8.16.1) with ESMTP id 1973SBfJ060779; Thu, 7 Oct 2021 03:28:11 GMT (envelope-from git@gitrepo.freebsd.org) Received: (from git@localhost) by gitrepo.freebsd.org (8.16.1/8.16.1/Submit) id 1973SBd3060778; Thu, 7 Oct 2021 03:28:11 GMT (envelope-from git) Date: Thu, 7 Oct 2021 03:28:11 GMT Message-Id: <202110070328.1973SBd3060778@gitrepo.freebsd.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-branches@FreeBSD.org From: Kyle Evans Subject: git: 168a6eee3a2a - stable/12 - powerpc: Implement atomic_(f)cmpset_ for short and char List-Id: Commits to the stable branches of the FreeBSD src repository List-Archive: https://lists.freebsd.org/archives/dev-commits-src-branches List-Help: List-Post: List-Subscribe: List-Unsubscribe: Sender: owner-dev-commits-src-branches@freebsd.org X-BeenThere: dev-commits-src-branches@freebsd.org MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit X-Git-Committer: kevans X-Git-Repository: src X-Git-Refname: refs/heads/stable/12 X-Git-Reftype: branch X-Git-Commit: 168a6eee3a2a6faa8e3eebafe8f1866b2f170727 Auto-Submitted: auto-generated X-ThisMailContainsUnwantedMimeParts: N The branch stable/12 has been updated by kevans: URL: https://cgit.FreeBSD.org/src/commit/?id=168a6eee3a2a6faa8e3eebafe8f1866b2f170727 commit 168a6eee3a2a6faa8e3eebafe8f1866b2f170727 Author: Justin Hibbits AuthorDate: 2019-10-08 01:36:34 +0000 Commit: Kyle Evans CommitDate: 2021-10-07 03:27:26 +0000 powerpc: Implement atomic_(f)cmpset_ for short and char This adds two implementations for each atomic_fcmpset_ and atomic_cmpset_ short and char functions, selectable at compile time for the target architecture. By default, it uses a generic shift-and-mask to perform atomic updates to sub-components of 32-bit words from . However, if ISA_206_ATOMICS is defined it uses the ll/sc instructions for halfword and bytes, introduced in PowerISA 2.06. These instructions are supported by all IBM processors from POWER7 on, as well as the Freescale/NXP e6500 core. Although the e5500 and e500mc both implement PowerISA 2.06 they do not implement these instructions. As part of this, clean up the atomic_(f)cmpset_acq and _rel wrappers, by using macros to reduce code duplication. ISA_206_ATOMICS requires clang or newer binutils (2.20 or later). (cherry picked from commit 84046d16eba76392ece41452b6ca3bd5808a1d19) --- sys/powerpc/include/atomic.h | 250 ++++++++++++++++++++++++++++++------------- 1 file changed, 178 insertions(+), 72 deletions(-) diff --git a/sys/powerpc/include/atomic.h b/sys/powerpc/include/atomic.h index e49a8473a747..c72639811e39 100644 --- a/sys/powerpc/include/atomic.h +++ b/sys/powerpc/include/atomic.h @@ -564,6 +564,56 @@ atomic_store_rel_long(volatile u_long *addr, u_long val) * two values are equal, update the value of *p with newval. Returns * zero if the compare failed, nonzero otherwise. */ +#ifdef ISA_206_ATOMICS +static __inline int +atomic_cmpset_char(volatile u_char *p, u_char cmpval, u_char newval) +{ + int ret; + + __asm __volatile ( + "1:\tlbarx %0, 0, %2\n\t" /* load old value */ + "cmplw %3, %0\n\t" /* compare */ + "bne- 2f\n\t" /* exit if not equal */ + "stbcx. %4, 0, %2\n\t" /* attempt to store */ + "bne- 1b\n\t" /* spin if failed */ + "li %0, 1\n\t" /* success - retval = 1 */ + "b 3f\n\t" /* we've succeeded */ + "2:\n\t" + "stbcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ + "li %0, 0\n\t" /* failure - retval = 0 */ + "3:\n\t" + : "=&r" (ret), "=m" (*p) + : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) + : "cr0", "memory"); + + return (ret); +} + +static __inline int +atomic_cmpset_short(volatile u_short *p, u_short cmpval, u_short newval) +{ + int ret; + + __asm __volatile ( + "1:\tlharx %0, 0, %2\n\t" /* load old value */ + "cmplw %3, %0\n\t" /* compare */ + "bne- 2f\n\t" /* exit if not equal */ + "sthcx. %4, 0, %2\n\t" /* attempt to store */ + "bne- 1b\n\t" /* spin if failed */ + "li %0, 1\n\t" /* success - retval = 1 */ + "b 3f\n\t" /* we've succeeded */ + "2:\n\t" + "sthcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ + "li %0, 0\n\t" /* failure - retval = 0 */ + "3:\n\t" + : "=&r" (ret), "=m" (*p) + : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) + : "cr0", "memory"); + + return (ret); +} +#endif + static __inline int atomic_cmpset_int(volatile u_int* p, u_int cmpval, u_int newval) { @@ -622,39 +672,36 @@ atomic_cmpset_long(volatile u_long* p, u_long cmpval, u_long newval) return (ret); } -static __inline int -atomic_cmpset_acq_int(volatile u_int *p, u_int cmpval, u_int newval) -{ - int retval; - - retval = atomic_cmpset_int(p, cmpval, newval); - __ATOMIC_ACQ(); - return (retval); -} - -static __inline int -atomic_cmpset_rel_int(volatile u_int *p, u_int cmpval, u_int newval) -{ - __ATOMIC_REL(); - return (atomic_cmpset_int(p, cmpval, newval)); -} - -static __inline int -atomic_cmpset_acq_long(volatile u_long *p, u_long cmpval, u_long newval) -{ - u_long retval; - - retval = atomic_cmpset_long(p, cmpval, newval); - __ATOMIC_ACQ(); - return (retval); -} - -static __inline int -atomic_cmpset_rel_long(volatile u_long *p, u_long cmpval, u_long newval) -{ - __ATOMIC_REL(); - return (atomic_cmpset_long(p, cmpval, newval)); -} +#define ATOMIC_CMPSET_ACQ_REL(type) \ + static __inline int \ + atomic_cmpset_acq_##type(volatile u_##type *p, \ + u_##type cmpval, u_##type newval)\ + {\ + u_##type retval; \ + retval = atomic_cmpset_##type(p, cmpval, newval);\ + __ATOMIC_ACQ();\ + return (retval);\ + }\ + static __inline int \ + atomic_cmpset_rel_##type(volatile u_##type *p, \ + u_##type cmpval, u_##type newval)\ + {\ + __ATOMIC_ACQ();\ + return (atomic_cmpset_##type(p, cmpval, newval));\ + }\ + struct hack + +ATOMIC_CMPSET_ACQ_REL(int); +ATOMIC_CMPSET_ACQ_REL(long); + + +#define atomic_cmpset_8 atomic_cmpset_char +#define atomic_cmpset_acq_8 atomic_cmpset_acq_char +#define atomic_cmpset_rel_8 atomic_cmpset_rel_char + +#define atomic_cmpset_16 atomic_cmpset_short +#define atomic_cmpset_acq_16 atomic_cmpset_acq_short +#define atomic_cmpset_rel_16 atomic_cmpset_rel_short #define atomic_cmpset_32 atomic_cmpset_int #define atomic_cmpset_acq_32 atomic_cmpset_acq_int @@ -680,13 +727,65 @@ atomic_cmpset_rel_long(volatile u_long *p, u_long cmpval, u_long newval) * zero if the compare failed and sets *cmpval to the read value from *p, * nonzero otherwise. */ +#ifdef ISA_206_ATOMICS +static __inline int +atomic_fcmpset_char(volatile u_char *p, u_char *cmpval, u_char newval) +{ + int ret; + + __asm __volatile ( + "lbarx %0, 0, %3\n\t" /* load old value */ + "cmplw %4, %0\n\t" /* compare */ + "bne- 1f\n\t" /* exit if not equal */ + "stbcx. %5, 0, %3\n\t" /* attempt to store */ + "bne- 1f\n\t" /* exit if failed */ + "li %0, 1\n\t" /* success - retval = 1 */ + "b 2f\n\t" /* we've succeeded */ + "1:\n\t" + "stbcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ + "stwx %0, 0, %7\n\t" + "li %0, 0\n\t" /* failure - retval = 0 */ + "2:\n\t" + : "=&r" (ret), "=m" (*p), "=m" (*cmpval) + : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) + : "cr0", "memory"); + + return (ret); +} + +static __inline int +atomic_fcmpset_short(volatile u_short *p, u_short *cmpval, u_short newval) +{ + int ret; + + __asm __volatile ( + "lharx %0, 0, %3\n\t" /* load old value */ + "cmplw %4, %0\n\t" /* compare */ + "bne- 1f\n\t" /* exit if not equal */ + "sthcx. %5, 0, %3\n\t" /* attempt to store */ + "bne- 1f\n\t" /* exit if failed */ + "li %0, 1\n\t" /* success - retval = 1 */ + "b 2f\n\t" /* we've succeeded */ + "1:\n\t" + "sthcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ + "stwx %0, 0, %7\n\t" + "li %0, 0\n\t" /* failure - retval = 0 */ + "2:\n\t" + : "=&r" (ret), "=m" (*p), "=m" (*cmpval) + : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) + : "cr0", "memory"); + + return (ret); +} +#endif /* ISA_206_ATOMICS */ + static __inline int atomic_fcmpset_int(volatile u_int *p, u_int *cmpval, u_int newval) { int ret; __asm __volatile ( - "lwarx %0, 0, %3\n\t" /* load old value */ + "lwarx %0, 0, %3\n\t" /* load old value */ "cmplw %4, %0\n\t" /* compare */ "bne 1f\n\t" /* exit if not equal */ "stwcx. %5, 0, %3\n\t" /* attempt to store */ @@ -711,12 +810,12 @@ atomic_fcmpset_long(volatile u_long *p, u_long *cmpval, u_long newval) __asm __volatile ( #ifdef __powerpc64__ - "ldarx %0, 0, %3\n\t" /* load old value */ + "ldarx %0, 0, %3\n\t" /* load old value */ "cmpld %4, %0\n\t" /* compare */ "bne 1f\n\t" /* exit if not equal */ "stdcx. %5, 0, %3\n\t" /* attempt to store */ #else - "lwarx %0, 0, %3\n\t" /* load old value */ + "lwarx %0, 0, %3\n\t" /* load old value */ "cmplw %4, %0\n\t" /* compare */ "bne 1f\n\t" /* exit if not equal */ "stwcx. %5, 0, %3\n\t" /* attempt to store */ @@ -741,39 +840,35 @@ atomic_fcmpset_long(volatile u_long *p, u_long *cmpval, u_long newval) return (ret); } -static __inline int -atomic_fcmpset_acq_int(volatile u_int *p, u_int *cmpval, u_int newval) -{ - int retval; - - retval = atomic_fcmpset_int(p, cmpval, newval); - __ATOMIC_ACQ(); - return (retval); -} - -static __inline int -atomic_fcmpset_rel_int(volatile u_int *p, u_int *cmpval, u_int newval) -{ - __ATOMIC_REL(); - return (atomic_fcmpset_int(p, cmpval, newval)); -} - -static __inline int -atomic_fcmpset_acq_long(volatile u_long *p, u_long *cmpval, u_long newval) -{ - u_long retval; - - retval = atomic_fcmpset_long(p, cmpval, newval); - __ATOMIC_ACQ(); - return (retval); -} - -static __inline int -atomic_fcmpset_rel_long(volatile u_long *p, u_long *cmpval, u_long newval) -{ - __ATOMIC_REL(); - return (atomic_fcmpset_long(p, cmpval, newval)); -} +#define ATOMIC_FCMPSET_ACQ_REL(type) \ + static __inline int \ + atomic_fcmpset_acq_##type(volatile u_##type *p, \ + u_##type *cmpval, u_##type newval)\ + {\ + u_##type retval; \ + retval = atomic_fcmpset_##type(p, cmpval, newval);\ + __ATOMIC_ACQ();\ + return (retval);\ + }\ + static __inline int \ + atomic_fcmpset_rel_##type(volatile u_##type *p, \ + u_##type *cmpval, u_##type newval)\ + {\ + __ATOMIC_REL();\ + return (atomic_fcmpset_##type(p, cmpval, newval));\ + }\ + struct hack + +ATOMIC_FCMPSET_ACQ_REL(int); +ATOMIC_FCMPSET_ACQ_REL(long); + +#define atomic_fcmpset_8 atomic_fcmpset_char +#define atomic_fcmpset_acq_8 atomic_fcmpset_acq_char +#define atomic_fcmpset_rel_8 atomic_fcmpset_rel_char + +#define atomic_fcmpset_16 atomic_fcmpset_short +#define atomic_fcmpset_acq_16 atomic_fcmpset_acq_short +#define atomic_fcmpset_rel_16 atomic_fcmpset_rel_short #define atomic_fcmpset_32 atomic_fcmpset_int #define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int @@ -861,9 +956,6 @@ atomic_swap_64(volatile u_long *p, u_long v) #define atomic_swap_ptr(p,v) atomic_swap_32((volatile u_int *)(p), v) #endif -#undef __ATOMIC_REL -#undef __ATOMIC_ACQ - static __inline void atomic_thread_fence_acq(void) { @@ -892,4 +984,18 @@ atomic_thread_fence_seq_cst(void) __asm __volatile("sync" : : : "memory"); } +#ifndef ISA_206_ATOMICS +#include +#endif + +/* These need sys/_atomic_subword.h on non-ISA-2.06-atomic platforms. */ +ATOMIC_CMPSET_ACQ_REL(char); +ATOMIC_CMPSET_ACQ_REL(short); + +ATOMIC_FCMPSET_ACQ_REL(char); +ATOMIC_FCMPSET_ACQ_REL(short); + +#undef __ATOMIC_REL +#undef __ATOMIC_ACQ + #endif /* ! _MACHINE_ATOMIC_H_ */