13 #include "kmp_atomic.h"
16 typedef unsigned char uchar;
17 typedef unsigned short ushort;
565 #ifndef KMP_GOMP_COMPAT
566 int __kmp_atomic_mode = 1;
568 int __kmp_atomic_mode = 2;
574 kmp_atomic_lock_t __kmp_atomic_lock;
576 kmp_atomic_lock_t __kmp_atomic_lock_1i;
578 kmp_atomic_lock_t __kmp_atomic_lock_2i;
580 kmp_atomic_lock_t __kmp_atomic_lock_4i;
582 kmp_atomic_lock_t __kmp_atomic_lock_4r;
584 kmp_atomic_lock_t __kmp_atomic_lock_8i;
586 kmp_atomic_lock_t __kmp_atomic_lock_8r;
588 kmp_atomic_lock_t __kmp_atomic_lock_8c;
590 kmp_atomic_lock_t __kmp_atomic_lock_10r;
592 kmp_atomic_lock_t __kmp_atomic_lock_16r;
594 kmp_atomic_lock_t __kmp_atomic_lock_16c;
596 kmp_atomic_lock_t __kmp_atomic_lock_20c;
598 kmp_atomic_lock_t __kmp_atomic_lock_32c;
605 #define KMP_ATOMIC_VOLATILE volatile
607 #if (KMP_ARCH_X86) && KMP_HAVE_QUAD
609 static inline Quad_a4_t operator+(Quad_a4_t &lhs, Quad_a4_t &rhs) {
610 return lhs.q + rhs.q;
612 static inline Quad_a4_t operator-(Quad_a4_t &lhs, Quad_a4_t &rhs) {
613 return lhs.q - rhs.q;
615 static inline Quad_a4_t operator*(Quad_a4_t &lhs, Quad_a4_t &rhs) {
616 return lhs.q * rhs.q;
618 static inline Quad_a4_t operator/(Quad_a4_t &lhs, Quad_a4_t &rhs) {
619 return lhs.q / rhs.q;
621 static inline bool operator<(Quad_a4_t &lhs, Quad_a4_t &rhs) {
622 return lhs.q < rhs.q;
624 static inline bool operator>(Quad_a4_t &lhs, Quad_a4_t &rhs) {
625 return lhs.q > rhs.q;
628 static inline Quad_a16_t operator+(Quad_a16_t &lhs, Quad_a16_t &rhs) {
629 return lhs.q + rhs.q;
631 static inline Quad_a16_t operator-(Quad_a16_t &lhs, Quad_a16_t &rhs) {
632 return lhs.q - rhs.q;
634 static inline Quad_a16_t operator*(Quad_a16_t &lhs, Quad_a16_t &rhs) {
635 return lhs.q * rhs.q;
637 static inline Quad_a16_t operator/(Quad_a16_t &lhs, Quad_a16_t &rhs) {
638 return lhs.q / rhs.q;
640 static inline bool operator<(Quad_a16_t &lhs, Quad_a16_t &rhs) {
641 return lhs.q < rhs.q;
643 static inline bool operator>(Quad_a16_t &lhs, Quad_a16_t &rhs) {
644 return lhs.q > rhs.q;
647 static inline kmp_cmplx128_a4_t operator+(kmp_cmplx128_a4_t &lhs,
648 kmp_cmplx128_a4_t &rhs) {
649 return lhs.q + rhs.q;
651 static inline kmp_cmplx128_a4_t operator-(kmp_cmplx128_a4_t &lhs,
652 kmp_cmplx128_a4_t &rhs) {
653 return lhs.q - rhs.q;
655 static inline kmp_cmplx128_a4_t operator*(kmp_cmplx128_a4_t &lhs,
656 kmp_cmplx128_a4_t &rhs) {
657 return lhs.q * rhs.q;
659 static inline kmp_cmplx128_a4_t operator/(kmp_cmplx128_a4_t &lhs,
660 kmp_cmplx128_a4_t &rhs) {
661 return lhs.q / rhs.q;
664 static inline kmp_cmplx128_a16_t operator+(kmp_cmplx128_a16_t &lhs,
665 kmp_cmplx128_a16_t &rhs) {
666 return lhs.q + rhs.q;
668 static inline kmp_cmplx128_a16_t operator-(kmp_cmplx128_a16_t &lhs,
669 kmp_cmplx128_a16_t &rhs) {
670 return lhs.q - rhs.q;
672 static inline kmp_cmplx128_a16_t operator*(kmp_cmplx128_a16_t &lhs,
673 kmp_cmplx128_a16_t &rhs) {
674 return lhs.q * rhs.q;
676 static inline kmp_cmplx128_a16_t operator/(kmp_cmplx128_a16_t &lhs,
677 kmp_cmplx128_a16_t &rhs) {
678 return lhs.q / rhs.q;
688 #define KMP_CHECK_GTID \
689 if (gtid == KMP_GTID_UNKNOWN) { \
690 gtid = __kmp_entry_gtid(); \
698 #define ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, RET_TYPE) \
699 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, \
700 TYPE *lhs, TYPE rhs) { \
701 KMP_DEBUG_ASSERT(__kmp_init_serial); \
702 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid));
706 #define ATOMIC_LOCK0 __kmp_atomic_lock
707 #define ATOMIC_LOCK1i __kmp_atomic_lock_1i
708 #define ATOMIC_LOCK2i __kmp_atomic_lock_2i
709 #define ATOMIC_LOCK4i __kmp_atomic_lock_4i
710 #define ATOMIC_LOCK4r __kmp_atomic_lock_4r
711 #define ATOMIC_LOCK8i __kmp_atomic_lock_8i
712 #define ATOMIC_LOCK8r __kmp_atomic_lock_8r
713 #define ATOMIC_LOCK8c __kmp_atomic_lock_8c
714 #define ATOMIC_LOCK10r __kmp_atomic_lock_10r
715 #define ATOMIC_LOCK16r __kmp_atomic_lock_16r
716 #define ATOMIC_LOCK16c __kmp_atomic_lock_16c
717 #define ATOMIC_LOCK20c __kmp_atomic_lock_20c
718 #define ATOMIC_LOCK32c __kmp_atomic_lock_32c
726 #define OP_CRITICAL(OP, LCK_ID) \
727 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
731 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
733 #define OP_UPDATE_CRITICAL(TYPE, OP, LCK_ID) \
734 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
735 (*lhs) = (TYPE)((*lhs)OP((TYPE)rhs)); \
736 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
760 #ifdef KMP_GOMP_COMPAT
761 #define OP_GOMP_CRITICAL(OP, FLAG) \
762 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
764 OP_CRITICAL(OP, 0); \
768 #define OP_UPDATE_GOMP_CRITICAL(TYPE, OP, FLAG) \
769 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
771 OP_UPDATE_CRITICAL(TYPE, OP, 0); \
775 #define OP_GOMP_CRITICAL(OP, FLAG)
776 #define OP_UPDATE_GOMP_CRITICAL(TYPE, OP, FLAG)
780 #define KMP_DO_PAUSE _mm_delay_32(1)
790 #define OP_CMPXCHG(TYPE, BITS, OP) \
792 TYPE old_value, new_value; \
793 old_value = *(TYPE volatile *)lhs; \
794 new_value = (TYPE)(old_value OP((TYPE)rhs)); \
795 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
796 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
797 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
800 old_value = *(TYPE volatile *)lhs; \
801 new_value = (TYPE)(old_value OP((TYPE)rhs)); \
812 #define OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \
816 kmp_int##BITS *vvv; \
818 struct _sss old_value, new_value; \
819 old_value.vvv = (kmp_int##BITS *)&old_value.cmp; \
820 new_value.vvv = (kmp_int##BITS *)&new_value.cmp; \
821 *old_value.vvv = *(volatile kmp_int##BITS *)lhs; \
822 new_value.cmp = (TYPE)(old_value.cmp OP rhs); \
823 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
824 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) old_value.vvv, \
825 *VOLATILE_CAST(kmp_int##BITS *) new_value.vvv)) { \
828 *old_value.vvv = *(volatile kmp_int##BITS *)lhs; \
829 new_value.cmp = (TYPE)(old_value.cmp OP rhs); \
835 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
839 #define ATOMIC_FIXED_ADD(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
841 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
842 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
844 KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \
847 #define ATOMIC_CMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
849 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
850 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
851 OP_CMPXCHG(TYPE, BITS, OP) \
856 #define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, \
858 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
859 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
860 OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \
868 #define ATOMIC_FIXED_ADD(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
870 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
871 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
872 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
874 KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \
877 OP_UPDATE_CRITICAL(TYPE, OP, \
882 #define ATOMIC_CMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
884 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
885 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
886 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
887 OP_CMPXCHG(TYPE, BITS, OP) \
890 OP_UPDATE_CRITICAL(TYPE, OP, \
897 #define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, \
899 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
900 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
901 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
902 OP_CMPXCHG(TYPE, BITS, OP) \
905 OP_UPDATE_CRITICAL(TYPE, OP, \
914 ATOMIC_FIXED_ADD(fixed4, add, kmp_int32, 32, +, 4i, 3,
916 ATOMIC_FIXED_ADD(fixed4, sub, kmp_int32, 32, -, 4i, 3,
919 ATOMIC_CMPXCHG(float4, add, kmp_real32, 32, +, 4r, 3,
921 ATOMIC_CMPXCHG(float4, sub, kmp_real32, 32, -, 4r, 3,
925 ATOMIC_FIXED_ADD(fixed8, add, kmp_int64, 64, +, 8i, 7,
927 ATOMIC_FIXED_ADD(fixed8, sub, kmp_int64, 64, -, 8i, 7,
930 ATOMIC_CMPXCHG(float8, add, kmp_real64, 64, +, 8r, 7,
932 ATOMIC_CMPXCHG(float8, sub, kmp_real64, 64, -, 8r, 7,
950 ATOMIC_CMPXCHG(fixed1, add, kmp_int8, 8, +, 1i, 0,
952 ATOMIC_CMPXCHG(fixed1, andb, kmp_int8, 8, &, 1i, 0,
954 ATOMIC_CMPXCHG(fixed1, div, kmp_int8, 8, /, 1i, 0,
956 ATOMIC_CMPXCHG(fixed1u, div, kmp_uint8, 8, /, 1i, 0,
958 ATOMIC_CMPXCHG(fixed1, mul, kmp_int8, 8, *, 1i, 0,
960 ATOMIC_CMPXCHG(fixed1, orb, kmp_int8, 8, |, 1i, 0,
962 ATOMIC_CMPXCHG(fixed1, shl, kmp_int8, 8, <<, 1i, 0,
964 ATOMIC_CMPXCHG(fixed1, shr, kmp_int8, 8, >>, 1i, 0,
966 ATOMIC_CMPXCHG(fixed1u, shr, kmp_uint8, 8, >>, 1i, 0,
968 ATOMIC_CMPXCHG(fixed1, sub, kmp_int8, 8, -, 1i, 0,
970 ATOMIC_CMPXCHG(fixed1, xor, kmp_int8, 8, ^, 1i, 0,
972 ATOMIC_CMPXCHG(fixed2, add, kmp_int16, 16, +, 2i, 1,
974 ATOMIC_CMPXCHG(fixed2, andb, kmp_int16, 16, &, 2i, 1,
976 ATOMIC_CMPXCHG(fixed2, div, kmp_int16, 16, /, 2i, 1,
978 ATOMIC_CMPXCHG(fixed2u, div, kmp_uint16, 16, /, 2i, 1,
980 ATOMIC_CMPXCHG(fixed2, mul, kmp_int16, 16, *, 2i, 1,
982 ATOMIC_CMPXCHG(fixed2, orb, kmp_int16, 16, |, 2i, 1,
984 ATOMIC_CMPXCHG(fixed2, shl, kmp_int16, 16, <<, 2i, 1,
986 ATOMIC_CMPXCHG(fixed2, shr, kmp_int16, 16, >>, 2i, 1,
988 ATOMIC_CMPXCHG(fixed2u, shr, kmp_uint16, 16, >>, 2i, 1,
990 ATOMIC_CMPXCHG(fixed2, sub, kmp_int16, 16, -, 2i, 1,
992 ATOMIC_CMPXCHG(fixed2, xor, kmp_int16, 16, ^, 2i, 1,
994 ATOMIC_CMPXCHG(fixed4, andb, kmp_int32, 32, &, 4i, 3,
996 ATOMIC_CMPXCHG(fixed4, div, kmp_int32, 32, /, 4i, 3,
998 ATOMIC_CMPXCHG(fixed4u, div, kmp_uint32, 32, /, 4i, 3,
1000 ATOMIC_CMPXCHG(fixed4, mul, kmp_int32, 32, *, 4i, 3,
1002 ATOMIC_CMPXCHG(fixed4, orb, kmp_int32, 32, |, 4i, 3,
1004 ATOMIC_CMPXCHG(fixed4, shl, kmp_int32, 32, <<, 4i, 3,
1006 ATOMIC_CMPXCHG(fixed4, shr, kmp_int32, 32, >>, 4i, 3,
1008 ATOMIC_CMPXCHG(fixed4u, shr, kmp_uint32, 32, >>, 4i, 3,
1010 ATOMIC_CMPXCHG(fixed4, xor, kmp_int32, 32, ^, 4i, 3,
1012 ATOMIC_CMPXCHG(fixed8, andb, kmp_int64, 64, &, 8i, 7,
1014 ATOMIC_CMPXCHG(fixed8, div, kmp_int64, 64, /, 8i, 7,
1016 ATOMIC_CMPXCHG(fixed8u, div, kmp_uint64, 64, /, 8i, 7,
1018 ATOMIC_CMPXCHG(fixed8, mul, kmp_int64, 64, *, 8i, 7,
1020 ATOMIC_CMPXCHG(fixed8, orb, kmp_int64, 64, |, 8i, 7,
1022 ATOMIC_CMPXCHG(fixed8, shl, kmp_int64, 64, <<, 8i, 7,
1024 ATOMIC_CMPXCHG(fixed8, shr, kmp_int64, 64, >>, 8i, 7,
1026 ATOMIC_CMPXCHG(fixed8u, shr, kmp_uint64, 64, >>, 8i, 7,
1028 ATOMIC_CMPXCHG(fixed8, xor, kmp_int64, 64, ^, 8i, 7,
1030 ATOMIC_CMPXCHG(float4, div, kmp_real32, 32, /, 4r, 3,
1032 ATOMIC_CMPXCHG(float4, mul, kmp_real32, 32, *, 4r, 3,
1034 ATOMIC_CMPXCHG(float8, div, kmp_real64, 64, /, 8r, 7,
1036 ATOMIC_CMPXCHG(float8, mul, kmp_real64, 64, *, 8r, 7,
1046 #define ATOMIC_CRIT_L(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
1047 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1048 OP_GOMP_CRITICAL(= *lhs OP, GOMP_FLAG) \
1049 OP_CRITICAL(= *lhs OP, LCK_ID) \
1052 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1056 #define ATOMIC_CMPX_L(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, GOMP_FLAG) \
1057 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1058 OP_GOMP_CRITICAL(= *lhs OP, GOMP_FLAG) \
1059 OP_CMPXCHG(TYPE, BITS, OP) \
1065 #define ATOMIC_CMPX_L(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, GOMP_FLAG) \
1066 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1067 OP_GOMP_CRITICAL(= *lhs OP, GOMP_FLAG) \
1068 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
1069 OP_CMPXCHG(TYPE, BITS, OP) \
1072 OP_CRITICAL(= *lhs OP, LCK_ID) \
1077 ATOMIC_CMPX_L(fixed1, andl,
char, 8, &&, 1i, 0,
1079 ATOMIC_CMPX_L(fixed1, orl,
char, 8, ||, 1i, 0,
1081 ATOMIC_CMPX_L(fixed2, andl,
short, 16, &&, 2i, 1,
1083 ATOMIC_CMPX_L(fixed2, orl,
short, 16, ||, 2i, 1,
1085 ATOMIC_CMPX_L(fixed4, andl, kmp_int32, 32, &&, 4i, 3,
1087 ATOMIC_CMPX_L(fixed4, orl, kmp_int32, 32, ||, 4i, 3,
1089 ATOMIC_CMPX_L(fixed8, andl, kmp_int64, 64, &&, 8i, 7,
1091 ATOMIC_CMPX_L(fixed8, orl, kmp_int64, 64, ||, 8i, 7,
1103 #define MIN_MAX_CRITSECT(OP, LCK_ID) \
1104 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
1106 if (*lhs OP rhs) { \
1109 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
1112 #ifdef KMP_GOMP_COMPAT
1113 #define GOMP_MIN_MAX_CRITSECT(OP, FLAG) \
1114 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
1116 MIN_MAX_CRITSECT(OP, 0); \
1120 #define GOMP_MIN_MAX_CRITSECT(OP, FLAG)
1124 #define MIN_MAX_CMPXCHG(TYPE, BITS, OP) \
1126 TYPE KMP_ATOMIC_VOLATILE temp_val; \
1129 old_value = temp_val; \
1130 while (old_value OP rhs && \
1131 !KMP_COMPARE_AND_STORE_ACQ##BITS( \
1132 (kmp_int##BITS *)lhs, \
1133 *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
1134 *VOLATILE_CAST(kmp_int##BITS *) & rhs)) { \
1136 old_value = temp_val; \
1142 #define MIN_MAX_CRITICAL(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
1143 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1144 if (*lhs OP rhs) { \
1145 GOMP_MIN_MAX_CRITSECT(OP, GOMP_FLAG) \
1146 MIN_MAX_CRITSECT(OP, LCK_ID) \
1150 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1154 #define MIN_MAX_COMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
1156 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1157 if (*lhs OP rhs) { \
1158 GOMP_MIN_MAX_CRITSECT(OP, GOMP_FLAG) \
1159 MIN_MAX_CMPXCHG(TYPE, BITS, OP) \
1166 #define MIN_MAX_COMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
1168 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1169 if (*lhs OP rhs) { \
1170 GOMP_MIN_MAX_CRITSECT(OP, GOMP_FLAG) \
1171 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
1172 MIN_MAX_CMPXCHG(TYPE, BITS, OP) \
1175 MIN_MAX_CRITSECT(OP, LCK_ID) \
1181 MIN_MAX_COMPXCHG(fixed1, max,
char, 8, <, 1i, 0,
1183 MIN_MAX_COMPXCHG(fixed1, min,
char, 8, >, 1i, 0,
1185 MIN_MAX_COMPXCHG(fixed2, max,
short, 16, <, 2i, 1,
1187 MIN_MAX_COMPXCHG(fixed2, min,
short, 16, >, 2i, 1,
1189 MIN_MAX_COMPXCHG(fixed4, max, kmp_int32, 32, <, 4i, 3,
1191 MIN_MAX_COMPXCHG(fixed4, min, kmp_int32, 32, >, 4i, 3,
1193 MIN_MAX_COMPXCHG(fixed8, max, kmp_int64, 64, <, 8i, 7,
1195 MIN_MAX_COMPXCHG(fixed8, min, kmp_int64, 64, >, 8i, 7,
1197 MIN_MAX_COMPXCHG(float4, max, kmp_real32, 32, <, 4r, 3,
1199 MIN_MAX_COMPXCHG(float4, min, kmp_real32, 32, >, 4r, 3,
1201 MIN_MAX_COMPXCHG(float8, max, kmp_real64, 64, <, 8r, 7,
1203 MIN_MAX_COMPXCHG(float8, min, kmp_real64, 64, >, 8r, 7,
1206 MIN_MAX_CRITICAL(float16, max, QUAD_LEGACY, <, 16r,
1208 MIN_MAX_CRITICAL(float16, min, QUAD_LEGACY, >, 16r,
1211 MIN_MAX_CRITICAL(float16, max_a16, Quad_a16_t, <, 16r,
1213 MIN_MAX_CRITICAL(float16, min_a16, Quad_a16_t, >, 16r,
1220 #define ATOMIC_CRIT_EQV(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
1221 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1222 OP_GOMP_CRITICAL(^= (TYPE) ~, GOMP_FLAG) \
1223 OP_CRITICAL(^= (TYPE) ~, LCK_ID) \
1227 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1230 #define ATOMIC_CMPX_EQV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
1232 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1233 OP_GOMP_CRITICAL(^= (TYPE) ~, GOMP_FLAG) \
1234 OP_CMPXCHG(TYPE, BITS, OP) \
1240 #define ATOMIC_CMPX_EQV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
1242 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1243 OP_GOMP_CRITICAL(^= (TYPE) ~, GOMP_FLAG) \
1244 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
1245 OP_CMPXCHG(TYPE, BITS, OP) \
1248 OP_CRITICAL(^= (TYPE) ~, LCK_ID) \
1253 ATOMIC_CMPXCHG(fixed1, neqv, kmp_int8, 8, ^, 1i, 0,
1255 ATOMIC_CMPXCHG(fixed2, neqv, kmp_int16, 16, ^, 2i, 1,
1257 ATOMIC_CMPXCHG(fixed4, neqv, kmp_int32, 32, ^, 4i, 3,
1259 ATOMIC_CMPXCHG(fixed8, neqv, kmp_int64, 64, ^, 8i, 7,
1261 ATOMIC_CMPX_EQV(fixed1, eqv, kmp_int8, 8, ^~, 1i, 0,
1263 ATOMIC_CMPX_EQV(fixed2, eqv, kmp_int16, 16, ^~, 2i, 1,
1265 ATOMIC_CMPX_EQV(fixed4, eqv, kmp_int32, 32, ^~, 4i, 3,
1267 ATOMIC_CMPX_EQV(fixed8, eqv, kmp_int64, 64, ^~, 8i, 7,
1276 #define ATOMIC_CRITICAL(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
1277 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1278 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
1279 OP_UPDATE_CRITICAL(TYPE, OP, LCK_ID) \
1284 ATOMIC_CRITICAL(float10, add,
long double, +, 10r,
1286 ATOMIC_CRITICAL(float10, sub,
long double, -, 10r,
1288 ATOMIC_CRITICAL(float10, mul,
long double, *, 10r,
1290 ATOMIC_CRITICAL(float10, div,
long double, /, 10r,
1294 ATOMIC_CRITICAL(float16, add, QUAD_LEGACY, +, 16r,
1296 ATOMIC_CRITICAL(float16, sub, QUAD_LEGACY, -, 16r,
1298 ATOMIC_CRITICAL(float16, mul, QUAD_LEGACY, *, 16r,
1300 ATOMIC_CRITICAL(float16, div, QUAD_LEGACY, /, 16r,
1303 ATOMIC_CRITICAL(float16, add_a16, Quad_a16_t, +, 16r,
1305 ATOMIC_CRITICAL(float16, sub_a16, Quad_a16_t, -, 16r,
1307 ATOMIC_CRITICAL(float16, mul_a16, Quad_a16_t, *, 16r,
1309 ATOMIC_CRITICAL(float16, div_a16, Quad_a16_t, /, 16r,
1317 ATOMIC_CMPXCHG_WORKAROUND(cmplx4, add, kmp_cmplx32, 64, +, 8c, 7,
1319 ATOMIC_CMPXCHG_WORKAROUND(cmplx4, sub, kmp_cmplx32, 64, -, 8c, 7,
1321 ATOMIC_CMPXCHG_WORKAROUND(cmplx4, mul, kmp_cmplx32, 64, *, 8c, 7,
1323 ATOMIC_CMPXCHG_WORKAROUND(cmplx4, div, kmp_cmplx32, 64, /, 8c, 7,
1327 ATOMIC_CRITICAL(cmplx4, add, kmp_cmplx32, +, 8c, 1)
1328 ATOMIC_CRITICAL(cmplx4, sub, kmp_cmplx32, -, 8c, 1)
1329 ATOMIC_CRITICAL(cmplx4, mul, kmp_cmplx32, *, 8c, 1)
1330 ATOMIC_CRITICAL(cmplx4, div, kmp_cmplx32, /, 8c, 1)
1333 ATOMIC_CRITICAL(cmplx8, add, kmp_cmplx64, +, 16c, 1)
1334 ATOMIC_CRITICAL(cmplx8, sub, kmp_cmplx64, -, 16c, 1)
1335 ATOMIC_CRITICAL(cmplx8, mul, kmp_cmplx64, *, 16c, 1)
1336 ATOMIC_CRITICAL(cmplx8, div, kmp_cmplx64, /, 16c, 1)
1337 ATOMIC_CRITICAL(cmplx10, add, kmp_cmplx80, +, 20c,
1339 ATOMIC_CRITICAL(cmplx10, sub, kmp_cmplx80, -, 20c,
1341 ATOMIC_CRITICAL(cmplx10, mul, kmp_cmplx80, *, 20c,
1343 ATOMIC_CRITICAL(cmplx10, div, kmp_cmplx80, /, 20c,
1346 ATOMIC_CRITICAL(cmplx16, add, CPLX128_LEG, +, 32c,
1348 ATOMIC_CRITICAL(cmplx16, sub, CPLX128_LEG, -, 32c,
1350 ATOMIC_CRITICAL(cmplx16, mul, CPLX128_LEG, *, 32c,
1352 ATOMIC_CRITICAL(cmplx16, div, CPLX128_LEG, /, 32c,
1355 ATOMIC_CRITICAL(cmplx16, add_a16, kmp_cmplx128_a16_t, +, 32c,
1357 ATOMIC_CRITICAL(cmplx16, sub_a16, kmp_cmplx128_a16_t, -, 32c,
1359 ATOMIC_CRITICAL(cmplx16, mul_a16, kmp_cmplx128_a16_t, *, 32c,
1361 ATOMIC_CRITICAL(cmplx16, div_a16, kmp_cmplx128_a16_t, /, 32c,
1368 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1376 #define OP_CRITICAL_REV(TYPE, OP, LCK_ID) \
1377 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
1379 (*lhs) = (TYPE)((rhs)OP(*lhs)); \
1381 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
1383 #ifdef KMP_GOMP_COMPAT
1384 #define OP_GOMP_CRITICAL_REV(TYPE, OP, FLAG) \
1385 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
1387 OP_CRITICAL_REV(TYPE, OP, 0); \
1392 #define OP_GOMP_CRITICAL_REV(TYPE, OP, FLAG)
1400 #define ATOMIC_BEGIN_REV(TYPE_ID, OP_ID, TYPE, RET_TYPE) \
1401 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID##_rev(ident_t *id_ref, int gtid, \
1402 TYPE *lhs, TYPE rhs) { \
1403 KMP_DEBUG_ASSERT(__kmp_init_serial); \
1404 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_rev: T#%d\n", gtid));
1413 #define OP_CMPXCHG_REV(TYPE, BITS, OP) \
1415 TYPE KMP_ATOMIC_VOLATILE temp_val; \
1416 TYPE old_value, new_value; \
1418 old_value = temp_val; \
1419 new_value = (TYPE)(rhs OP old_value); \
1420 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
1421 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
1422 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
1426 old_value = temp_val; \
1427 new_value = (TYPE)(rhs OP old_value); \
1432 #define ATOMIC_CMPXCHG_REV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, GOMP_FLAG) \
1433 ATOMIC_BEGIN_REV(TYPE_ID, OP_ID, TYPE, void) \
1434 OP_GOMP_CRITICAL_REV(TYPE, OP, GOMP_FLAG) \
1435 OP_CMPXCHG_REV(TYPE, BITS, OP) \
1452 ATOMIC_CMPXCHG_REV(fixed1, div, kmp_int8, 8, /, 1i,
1454 ATOMIC_CMPXCHG_REV(fixed1u, div, kmp_uint8, 8, /, 1i,
1456 ATOMIC_CMPXCHG_REV(fixed1, shl, kmp_int8, 8, <<, 1i,
1458 ATOMIC_CMPXCHG_REV(fixed1, shr, kmp_int8, 8, >>, 1i,
1460 ATOMIC_CMPXCHG_REV(fixed1u, shr, kmp_uint8, 8, >>, 1i,
1462 ATOMIC_CMPXCHG_REV(fixed1, sub, kmp_int8, 8, -, 1i,
1465 ATOMIC_CMPXCHG_REV(fixed2, div, kmp_int16, 16, /, 2i,
1467 ATOMIC_CMPXCHG_REV(fixed2u, div, kmp_uint16, 16, /, 2i,
1469 ATOMIC_CMPXCHG_REV(fixed2, shl, kmp_int16, 16, <<, 2i,
1471 ATOMIC_CMPXCHG_REV(fixed2, shr, kmp_int16, 16, >>, 2i,
1473 ATOMIC_CMPXCHG_REV(fixed2u, shr, kmp_uint16, 16, >>, 2i,
1475 ATOMIC_CMPXCHG_REV(fixed2, sub, kmp_int16, 16, -, 2i,
1478 ATOMIC_CMPXCHG_REV(fixed4, div, kmp_int32, 32, /, 4i,
1480 ATOMIC_CMPXCHG_REV(fixed4u, div, kmp_uint32, 32, /, 4i,
1482 ATOMIC_CMPXCHG_REV(fixed4, shl, kmp_int32, 32, <<, 4i,
1484 ATOMIC_CMPXCHG_REV(fixed4, shr, kmp_int32, 32, >>, 4i,
1486 ATOMIC_CMPXCHG_REV(fixed4u, shr, kmp_uint32, 32, >>, 4i,
1488 ATOMIC_CMPXCHG_REV(fixed4, sub, kmp_int32, 32, -, 4i,
1491 ATOMIC_CMPXCHG_REV(fixed8, div, kmp_int64, 64, /, 8i,
1493 ATOMIC_CMPXCHG_REV(fixed8u, div, kmp_uint64, 64, /, 8i,
1495 ATOMIC_CMPXCHG_REV(fixed8, shl, kmp_int64, 64, <<, 8i,
1497 ATOMIC_CMPXCHG_REV(fixed8, shr, kmp_int64, 64, >>, 8i,
1499 ATOMIC_CMPXCHG_REV(fixed8u, shr, kmp_uint64, 64, >>, 8i,
1501 ATOMIC_CMPXCHG_REV(fixed8, sub, kmp_int64, 64, -, 8i,
1504 ATOMIC_CMPXCHG_REV(float4, div, kmp_real32, 32, /, 4r,
1506 ATOMIC_CMPXCHG_REV(float4, sub, kmp_real32, 32, -, 4r,
1509 ATOMIC_CMPXCHG_REV(float8, div, kmp_real64, 64, /, 8r,
1511 ATOMIC_CMPXCHG_REV(float8, sub, kmp_real64, 64, -, 8r,
1521 #define ATOMIC_CRITICAL_REV(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
1522 ATOMIC_BEGIN_REV(TYPE_ID, OP_ID, TYPE, void) \
1523 OP_GOMP_CRITICAL_REV(TYPE, OP, GOMP_FLAG) \
1524 OP_CRITICAL_REV(TYPE, OP, LCK_ID) \
1529 ATOMIC_CRITICAL_REV(float10, sub,
long double, -, 10r,
1531 ATOMIC_CRITICAL_REV(float10, div,
long double, /, 10r,
1535 ATOMIC_CRITICAL_REV(float16, sub, QUAD_LEGACY, -, 16r,
1537 ATOMIC_CRITICAL_REV(float16, div, QUAD_LEGACY, /, 16r,
1540 ATOMIC_CRITICAL_REV(float16, sub_a16, Quad_a16_t, -, 16r,
1542 ATOMIC_CRITICAL_REV(float16, div_a16, Quad_a16_t, /, 16r,
1548 ATOMIC_CRITICAL_REV(cmplx4, sub, kmp_cmplx32, -, 8c,
1550 ATOMIC_CRITICAL_REV(cmplx4, div, kmp_cmplx32, /, 8c,
1552 ATOMIC_CRITICAL_REV(cmplx8, sub, kmp_cmplx64, -, 16c,
1554 ATOMIC_CRITICAL_REV(cmplx8, div, kmp_cmplx64, /, 16c,
1556 ATOMIC_CRITICAL_REV(cmplx10, sub, kmp_cmplx80, -, 20c,
1558 ATOMIC_CRITICAL_REV(cmplx10, div, kmp_cmplx80, /, 20c,
1561 ATOMIC_CRITICAL_REV(cmplx16, sub, CPLX128_LEG, -, 32c,
1563 ATOMIC_CRITICAL_REV(cmplx16, div, CPLX128_LEG, /, 32c,
1566 ATOMIC_CRITICAL_REV(cmplx16, sub_a16, kmp_cmplx128_a16_t, -, 32c,
1568 ATOMIC_CRITICAL_REV(cmplx16, div_a16, kmp_cmplx128_a16_t, /, 32c,
1587 #define ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1588 void __kmpc_atomic_##TYPE_ID##_##OP_ID##_##RTYPE_ID( \
1589 ident_t *id_ref, int gtid, TYPE *lhs, RTYPE rhs) { \
1590 KMP_DEBUG_ASSERT(__kmp_init_serial); \
1592 ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_" #RTYPE_ID ": T#%d\n", \
1596 #define ATOMIC_CRITICAL_FP(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, LCK_ID, \
1598 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1599 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
1600 OP_UPDATE_CRITICAL(TYPE, OP, LCK_ID) \
1604 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1607 #define ATOMIC_CMPXCHG_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \
1608 LCK_ID, MASK, GOMP_FLAG) \
1609 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1610 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
1611 OP_CMPXCHG(TYPE, BITS, OP) \
1617 #define ATOMIC_CMPXCHG_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \
1618 LCK_ID, MASK, GOMP_FLAG) \
1619 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1620 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
1621 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
1622 OP_CMPXCHG(TYPE, BITS, OP) \
1625 OP_UPDATE_CRITICAL(TYPE, OP, \
1632 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1634 #define ATOMIC_CMPXCHG_REV_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, \
1635 RTYPE, LCK_ID, MASK, GOMP_FLAG) \
1636 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1637 OP_GOMP_CRITICAL_REV(TYPE, OP, GOMP_FLAG) \
1638 OP_CMPXCHG_REV(TYPE, BITS, OP) \
1640 #define ATOMIC_CRITICAL_REV_FP(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, \
1641 LCK_ID, GOMP_FLAG) \
1642 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1643 OP_GOMP_CRITICAL_REV(TYPE, OP, GOMP_FLAG) \
1644 OP_CRITICAL_REV(TYPE, OP, LCK_ID) \
1649 ATOMIC_CMPXCHG_MIX(fixed1,
char, mul, 8, *, float8, kmp_real64, 1i, 0,
1651 ATOMIC_CMPXCHG_MIX(fixed1,
char, div, 8, /, float8, kmp_real64, 1i, 0,
1653 ATOMIC_CMPXCHG_MIX(fixed2,
short, mul, 16, *, float8, kmp_real64, 2i, 1,
1655 ATOMIC_CMPXCHG_MIX(fixed2,
short, div, 16, /, float8, kmp_real64, 2i, 1,
1657 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, mul, 32, *, float8, kmp_real64, 4i, 3,
1659 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, div, 32, /, float8, kmp_real64, 4i, 3,
1661 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, mul, 64, *, float8, kmp_real64, 8i, 7,
1663 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, div, 64, /, float8, kmp_real64, 8i, 7,
1665 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, add, 32, +, float8, kmp_real64, 4r, 3,
1667 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, sub, 32, -, float8, kmp_real64, 4r, 3,
1669 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, mul, 32, *, float8, kmp_real64, 4r, 3,
1671 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, div, 32, /, float8, kmp_real64, 4r, 3,
1677 ATOMIC_CMPXCHG_MIX(fixed1,
char, add, 8, +, fp, _Quad, 1i, 0,
1679 ATOMIC_CMPXCHG_MIX(fixed1u, uchar, add, 8, +, fp, _Quad, 1i, 0,
1681 ATOMIC_CMPXCHG_MIX(fixed1,
char, sub, 8, -, fp, _Quad, 1i, 0,
1683 ATOMIC_CMPXCHG_MIX(fixed1u, uchar, sub, 8, -, fp, _Quad, 1i, 0,
1685 ATOMIC_CMPXCHG_MIX(fixed1,
char, mul, 8, *, fp, _Quad, 1i, 0,
1687 ATOMIC_CMPXCHG_MIX(fixed1u, uchar, mul, 8, *, fp, _Quad, 1i, 0,
1689 ATOMIC_CMPXCHG_MIX(fixed1,
char, div, 8, /, fp, _Quad, 1i, 0,
1691 ATOMIC_CMPXCHG_MIX(fixed1u, uchar, div, 8, /, fp, _Quad, 1i, 0,
1694 ATOMIC_CMPXCHG_MIX(fixed2,
short, add, 16, +, fp, _Quad, 2i, 1,
1696 ATOMIC_CMPXCHG_MIX(fixed2u, ushort, add, 16, +, fp, _Quad, 2i, 1,
1698 ATOMIC_CMPXCHG_MIX(fixed2,
short, sub, 16, -, fp, _Quad, 2i, 1,
1700 ATOMIC_CMPXCHG_MIX(fixed2u, ushort, sub, 16, -, fp, _Quad, 2i, 1,
1702 ATOMIC_CMPXCHG_MIX(fixed2,
short, mul, 16, *, fp, _Quad, 2i, 1,
1704 ATOMIC_CMPXCHG_MIX(fixed2u, ushort, mul, 16, *, fp, _Quad, 2i, 1,
1706 ATOMIC_CMPXCHG_MIX(fixed2,
short, div, 16, /, fp, _Quad, 2i, 1,
1708 ATOMIC_CMPXCHG_MIX(fixed2u, ushort, div, 16, /, fp, _Quad, 2i, 1,
1711 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, add, 32, +, fp, _Quad, 4i, 3,
1713 ATOMIC_CMPXCHG_MIX(fixed4u, kmp_uint32, add, 32, +, fp, _Quad, 4i, 3,
1715 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, sub, 32, -, fp, _Quad, 4i, 3,
1717 ATOMIC_CMPXCHG_MIX(fixed4u, kmp_uint32, sub, 32, -, fp, _Quad, 4i, 3,
1719 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, mul, 32, *, fp, _Quad, 4i, 3,
1721 ATOMIC_CMPXCHG_MIX(fixed4u, kmp_uint32, mul, 32, *, fp, _Quad, 4i, 3,
1723 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, div, 32, /, fp, _Quad, 4i, 3,
1725 ATOMIC_CMPXCHG_MIX(fixed4u, kmp_uint32, div, 32, /, fp, _Quad, 4i, 3,
1728 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, add, 64, +, fp, _Quad, 8i, 7,
1730 ATOMIC_CMPXCHG_MIX(fixed8u, kmp_uint64, add, 64, +, fp, _Quad, 8i, 7,
1732 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, sub, 64, -, fp, _Quad, 8i, 7,
1734 ATOMIC_CMPXCHG_MIX(fixed8u, kmp_uint64, sub, 64, -, fp, _Quad, 8i, 7,
1736 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, mul, 64, *, fp, _Quad, 8i, 7,
1738 ATOMIC_CMPXCHG_MIX(fixed8u, kmp_uint64, mul, 64, *, fp, _Quad, 8i, 7,
1740 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, div, 64, /, fp, _Quad, 8i, 7,
1742 ATOMIC_CMPXCHG_MIX(fixed8u, kmp_uint64, div, 64, /, fp, _Quad, 8i, 7,
1745 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, add, 32, +, fp, _Quad, 4r, 3,
1747 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, sub, 32, -, fp, _Quad, 4r, 3,
1749 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, mul, 32, *, fp, _Quad, 4r, 3,
1751 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, div, 32, /, fp, _Quad, 4r, 3,
1754 ATOMIC_CMPXCHG_MIX(float8, kmp_real64, add, 64, +, fp, _Quad, 8r, 7,
1756 ATOMIC_CMPXCHG_MIX(float8, kmp_real64, sub, 64, -, fp, _Quad, 8r, 7,
1758 ATOMIC_CMPXCHG_MIX(float8, kmp_real64, mul, 64, *, fp, _Quad, 8r, 7,
1760 ATOMIC_CMPXCHG_MIX(float8, kmp_real64, div, 64, /, fp, _Quad, 8r, 7,
1763 ATOMIC_CRITICAL_FP(float10,
long double, add, +, fp, _Quad, 10r,
1765 ATOMIC_CRITICAL_FP(float10,
long double, sub, -, fp, _Quad, 10r,
1767 ATOMIC_CRITICAL_FP(float10,
long double, mul, *, fp, _Quad, 10r,
1769 ATOMIC_CRITICAL_FP(float10,
long double, div, /, fp, _Quad, 10r,
1772 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1774 ATOMIC_CMPXCHG_REV_MIX(fixed1,
char, sub_rev, 8, -, fp, _Quad, 1i, 0,
1776 ATOMIC_CMPXCHG_REV_MIX(fixed1u, uchar, sub_rev, 8, -, fp, _Quad, 1i, 0,
1778 ATOMIC_CMPXCHG_REV_MIX(fixed1,
char, div_rev, 8, /, fp, _Quad, 1i, 0,
1780 ATOMIC_CMPXCHG_REV_MIX(fixed1u, uchar, div_rev, 8, /, fp, _Quad, 1i, 0,
1783 ATOMIC_CMPXCHG_REV_MIX(fixed2,
short, sub_rev, 16, -, fp, _Quad, 2i, 1,
1785 ATOMIC_CMPXCHG_REV_MIX(fixed2u, ushort, sub_rev, 16, -, fp, _Quad, 2i, 1,
1787 ATOMIC_CMPXCHG_REV_MIX(fixed2,
short, div_rev, 16, /, fp, _Quad, 2i, 1,
1789 ATOMIC_CMPXCHG_REV_MIX(fixed2u, ushort, div_rev, 16, /, fp, _Quad, 2i, 1,
1792 ATOMIC_CMPXCHG_REV_MIX(fixed4, kmp_int32, sub_rev, 32, -, fp, _Quad, 4i, 3,
1794 ATOMIC_CMPXCHG_REV_MIX(fixed4u, kmp_uint32, sub_rev, 32, -, fp, _Quad, 4i, 3,
1796 ATOMIC_CMPXCHG_REV_MIX(fixed4, kmp_int32, div_rev, 32, /, fp, _Quad, 4i, 3,
1798 ATOMIC_CMPXCHG_REV_MIX(fixed4u, kmp_uint32, div_rev, 32, /, fp, _Quad, 4i, 3,
1801 ATOMIC_CMPXCHG_REV_MIX(fixed8, kmp_int64, sub_rev, 64, -, fp, _Quad, 8i, 7,
1803 ATOMIC_CMPXCHG_REV_MIX(fixed8u, kmp_uint64, sub_rev, 64, -, fp, _Quad, 8i, 7,
1805 ATOMIC_CMPXCHG_REV_MIX(fixed8, kmp_int64, div_rev, 64, /, fp, _Quad, 8i, 7,
1807 ATOMIC_CMPXCHG_REV_MIX(fixed8u, kmp_uint64, div_rev, 64, /, fp, _Quad, 8i, 7,
1810 ATOMIC_CMPXCHG_REV_MIX(float4, kmp_real32, sub_rev, 32, -, fp, _Quad, 4r, 3,
1812 ATOMIC_CMPXCHG_REV_MIX(float4, kmp_real32, div_rev, 32, /, fp, _Quad, 4r, 3,
1815 ATOMIC_CMPXCHG_REV_MIX(float8, kmp_real64, sub_rev, 64, -, fp, _Quad, 8r, 7,
1817 ATOMIC_CMPXCHG_REV_MIX(float8, kmp_real64, div_rev, 64, /, fp, _Quad, 8r, 7,
1820 ATOMIC_CRITICAL_REV_FP(float10,
long double, sub_rev, -, fp, _Quad, 10r,
1822 ATOMIC_CRITICAL_REV_FP(float10,
long double, div_rev, /, fp, _Quad, 10r,
1828 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1833 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \
1834 LCK_ID, MASK, GOMP_FLAG) \
1835 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1836 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
1837 OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \
1841 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \
1842 LCK_ID, MASK, GOMP_FLAG) \
1843 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1844 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
1845 OP_CMPXCHG(TYPE, BITS, OP) \
1851 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \
1852 LCK_ID, MASK, GOMP_FLAG) \
1853 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1854 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
1855 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
1856 OP_CMPXCHG(TYPE, BITS, OP) \
1859 OP_UPDATE_CRITICAL(TYPE, OP, \
1865 ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, add, 64, +, cmplx8, kmp_cmplx64, 8c,
1867 ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, sub, 64, -, cmplx8, kmp_cmplx64, 8c,
1869 ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, mul, 64, *, cmplx8, kmp_cmplx64, 8c,
1871 ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, div, 64, /, cmplx8, kmp_cmplx64, 8c,
1875 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1886 #define ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, RET_TYPE) \
1887 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, \
1889 KMP_DEBUG_ASSERT(__kmp_init_serial); \
1890 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid));
1901 #define OP_CMPXCHG_READ(TYPE, BITS, OP) \
1903 TYPE KMP_ATOMIC_VOLATILE temp_val; \
1906 kmp_int##BITS i_val; \
1908 union f_i_union old_value; \
1910 old_value.f_val = temp_val; \
1911 old_value.i_val = KMP_COMPARE_AND_STORE_RET##BITS( \
1912 (kmp_int##BITS *)loc, \
1913 *VOLATILE_CAST(kmp_int##BITS *) & old_value.i_val, \
1914 *VOLATILE_CAST(kmp_int##BITS *) & old_value.i_val); \
1915 new_value = old_value.f_val; \
1925 #define OP_CRITICAL_READ(OP, LCK_ID) \
1926 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
1928 new_value = (*loc); \
1930 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
1933 #ifdef KMP_GOMP_COMPAT
1934 #define OP_GOMP_CRITICAL_READ(OP, FLAG) \
1935 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
1937 OP_CRITICAL_READ(OP, 0); \
1941 #define OP_GOMP_CRITICAL_READ(OP, FLAG)
1945 #define ATOMIC_FIXED_READ(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
1946 ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, TYPE) \
1948 OP_GOMP_CRITICAL_READ(OP## =, GOMP_FLAG) \
1949 new_value = KMP_TEST_THEN_ADD##BITS(loc, OP 0); \
1953 #define ATOMIC_CMPXCHG_READ(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
1954 ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, TYPE) \
1956 OP_GOMP_CRITICAL_READ(OP## =, GOMP_FLAG) \
1957 OP_CMPXCHG_READ(TYPE, BITS, OP) \
1965 #define ATOMIC_CRITICAL_READ(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
1966 ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, TYPE) \
1968 OP_GOMP_CRITICAL_READ(OP## =, GOMP_FLAG) \
1969 OP_CRITICAL_READ(OP, LCK_ID) \
1977 #if (KMP_OS_WINDOWS)
1979 #define OP_CRITICAL_READ_WRK(OP, LCK_ID) \
1980 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
1984 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
1986 #ifdef KMP_GOMP_COMPAT
1987 #define OP_GOMP_CRITICAL_READ_WRK(OP, FLAG) \
1988 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
1990 OP_CRITICAL_READ_WRK(OP, 0); \
1993 #define OP_GOMP_CRITICAL_READ_WRK(OP, FLAG)
1996 #define ATOMIC_BEGIN_READ_WRK(TYPE_ID, OP_ID, TYPE) \
1997 void __kmpc_atomic_##TYPE_ID##_##OP_ID(TYPE *out, ident_t *id_ref, int gtid, \
1999 KMP_DEBUG_ASSERT(__kmp_init_serial); \
2000 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid));
2003 #define ATOMIC_CRITICAL_READ_WRK(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
2004 ATOMIC_BEGIN_READ_WRK(TYPE_ID, OP_ID, TYPE) \
2005 OP_GOMP_CRITICAL_READ_WRK(OP## =, GOMP_FLAG) \
2006 OP_CRITICAL_READ_WRK(OP, LCK_ID) \
2013 ATOMIC_FIXED_READ(fixed4, rd, kmp_int32, 32, +, 0)
2014 ATOMIC_FIXED_READ(fixed8, rd, kmp_int64, 64, +,
2016 ATOMIC_CMPXCHG_READ(float4, rd, kmp_real32, 32, +,
2018 ATOMIC_CMPXCHG_READ(float8, rd, kmp_real64, 64, +,
2022 ATOMIC_CMPXCHG_READ(fixed1, rd, kmp_int8, 8, +,
2024 ATOMIC_CMPXCHG_READ(fixed2, rd, kmp_int16, 16, +,
2027 ATOMIC_CRITICAL_READ(float10, rd,
long double, +, 10r,
2030 ATOMIC_CRITICAL_READ(float16, rd, QUAD_LEGACY, +, 16r,
2035 #if (KMP_OS_WINDOWS)
2036 ATOMIC_CRITICAL_READ_WRK(cmplx4, rd, kmp_cmplx32, +, 8c,
2039 ATOMIC_CRITICAL_READ(cmplx4, rd, kmp_cmplx32, +, 8c,
2042 ATOMIC_CRITICAL_READ(cmplx8, rd, kmp_cmplx64, +, 16c,
2044 ATOMIC_CRITICAL_READ(cmplx10, rd, kmp_cmplx80, +, 20c,
2047 ATOMIC_CRITICAL_READ(cmplx16, rd, CPLX128_LEG, +, 32c,
2050 ATOMIC_CRITICAL_READ(float16, a16_rd, Quad_a16_t, +, 16r,
2052 ATOMIC_CRITICAL_READ(cmplx16, a16_rd, kmp_cmplx128_a16_t, +, 32c,
2060 #define ATOMIC_XCHG_WR(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2061 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
2062 OP_GOMP_CRITICAL(OP, GOMP_FLAG) \
2063 KMP_XCHG_FIXED##BITS(lhs, rhs); \
2066 #define ATOMIC_XCHG_FLOAT_WR(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2067 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
2068 OP_GOMP_CRITICAL(OP, GOMP_FLAG) \
2069 KMP_XCHG_REAL##BITS(lhs, rhs); \
2079 #define OP_CMPXCHG_WR(TYPE, BITS, OP) \
2081 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2082 TYPE old_value, new_value; \
2084 old_value = temp_val; \
2086 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
2087 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
2088 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
2090 old_value = temp_val; \
2096 #define ATOMIC_CMPXCHG_WR(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2097 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
2098 OP_GOMP_CRITICAL(OP, GOMP_FLAG) \
2099 OP_CMPXCHG_WR(TYPE, BITS, OP) \
2108 #define ATOMIC_CRITICAL_WR(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
2109 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
2110 OP_GOMP_CRITICAL(OP, GOMP_FLAG) \
2111 OP_CRITICAL(OP, LCK_ID) \
2115 ATOMIC_XCHG_WR(fixed1, wr, kmp_int8, 8, =,
2117 ATOMIC_XCHG_WR(fixed2, wr, kmp_int16, 16, =,
2119 ATOMIC_XCHG_WR(fixed4, wr, kmp_int32, 32, =,
2122 ATOMIC_CMPXCHG_WR(fixed8, wr, kmp_int64, 64, =,
2125 ATOMIC_XCHG_WR(fixed8, wr, kmp_int64, 64, =,
2129 ATOMIC_XCHG_FLOAT_WR(float4, wr, kmp_real32, 32, =,
2132 ATOMIC_CMPXCHG_WR(float8, wr, kmp_real64, 64, =,
2135 ATOMIC_XCHG_FLOAT_WR(float8, wr, kmp_real64, 64, =,
2139 ATOMIC_CRITICAL_WR(float10, wr,
long double, =, 10r,
2142 ATOMIC_CRITICAL_WR(float16, wr, QUAD_LEGACY, =, 16r,
2145 ATOMIC_CRITICAL_WR(cmplx4, wr, kmp_cmplx32, =, 8c, 1)
2146 ATOMIC_CRITICAL_WR(cmplx8, wr, kmp_cmplx64, =, 16c,
2148 ATOMIC_CRITICAL_WR(cmplx10, wr, kmp_cmplx80, =, 20c,
2151 ATOMIC_CRITICAL_WR(cmplx16, wr, CPLX128_LEG, =, 32c,
2154 ATOMIC_CRITICAL_WR(float16, a16_wr, Quad_a16_t, =, 16r,
2156 ATOMIC_CRITICAL_WR(cmplx16, a16_wr, kmp_cmplx128_a16_t, =, 32c,
2169 #define ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, RET_TYPE) \
2170 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, \
2171 TYPE *lhs, TYPE rhs, int flag) { \
2172 KMP_DEBUG_ASSERT(__kmp_init_serial); \
2173 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid));
2181 #define OP_CRITICAL_CPT(OP, LCK_ID) \
2182 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2186 new_value = (*lhs); \
2188 new_value = (*lhs); \
2192 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2195 #define OP_UPDATE_CRITICAL_CPT(TYPE, OP, LCK_ID) \
2196 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2199 (*lhs) = (TYPE)((*lhs)OP rhs); \
2200 new_value = (*lhs); \
2202 new_value = (*lhs); \
2203 (*lhs) = (TYPE)((*lhs)OP rhs); \
2206 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2210 #ifdef KMP_GOMP_COMPAT
2211 #define OP_GOMP_CRITICAL_CPT(TYPE, OP, FLAG) \
2212 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
2214 OP_UPDATE_CRITICAL_CPT(TYPE, OP, 0); \
2217 #define OP_GOMP_CRITICAL_CPT(TYPE, OP, FLAG)
2227 #define OP_CMPXCHG_CPT(TYPE, BITS, OP) \
2229 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2230 TYPE old_value, new_value; \
2232 old_value = temp_val; \
2233 new_value = (TYPE)(old_value OP rhs); \
2234 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
2235 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
2236 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
2238 old_value = temp_val; \
2239 new_value = (TYPE)(old_value OP rhs); \
2248 #define ATOMIC_CMPXCHG_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2249 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2252 OP_GOMP_CRITICAL_CPT(TYPE, OP, GOMP_FLAG) \
2253 OP_CMPXCHG_CPT(TYPE, BITS, OP) \
2257 #define ATOMIC_FIXED_ADD_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2258 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2259 TYPE old_value, new_value; \
2261 OP_GOMP_CRITICAL_CPT(TYPE, OP, GOMP_FLAG) \
2263 old_value = KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \
2265 return old_value OP rhs; \
2271 ATOMIC_FIXED_ADD_CPT(fixed4, add_cpt, kmp_int32, 32, +,
2273 ATOMIC_FIXED_ADD_CPT(fixed4, sub_cpt, kmp_int32, 32, -,
2275 ATOMIC_FIXED_ADD_CPT(fixed8, add_cpt, kmp_int64, 64, +,
2277 ATOMIC_FIXED_ADD_CPT(fixed8, sub_cpt, kmp_int64, 64, -,
2280 ATOMIC_CMPXCHG_CPT(float4, add_cpt, kmp_real32, 32, +,
2282 ATOMIC_CMPXCHG_CPT(float4, sub_cpt, kmp_real32, 32, -,
2284 ATOMIC_CMPXCHG_CPT(float8, add_cpt, kmp_real64, 64, +,
2286 ATOMIC_CMPXCHG_CPT(float8, sub_cpt, kmp_real64, 64, -,
2301 ATOMIC_CMPXCHG_CPT(fixed1, add_cpt, kmp_int8, 8, +,
2303 ATOMIC_CMPXCHG_CPT(fixed1, andb_cpt, kmp_int8, 8, &,
2305 ATOMIC_CMPXCHG_CPT(fixed1, div_cpt, kmp_int8, 8, /,
2307 ATOMIC_CMPXCHG_CPT(fixed1u, div_cpt, kmp_uint8, 8, /,
2309 ATOMIC_CMPXCHG_CPT(fixed1, mul_cpt, kmp_int8, 8, *,
2311 ATOMIC_CMPXCHG_CPT(fixed1, orb_cpt, kmp_int8, 8, |,
2313 ATOMIC_CMPXCHG_CPT(fixed1, shl_cpt, kmp_int8, 8, <<,
2315 ATOMIC_CMPXCHG_CPT(fixed1, shr_cpt, kmp_int8, 8, >>,
2317 ATOMIC_CMPXCHG_CPT(fixed1u, shr_cpt, kmp_uint8, 8, >>,
2319 ATOMIC_CMPXCHG_CPT(fixed1, sub_cpt, kmp_int8, 8, -,
2321 ATOMIC_CMPXCHG_CPT(fixed1, xor_cpt, kmp_int8, 8, ^,
2323 ATOMIC_CMPXCHG_CPT(fixed2, add_cpt, kmp_int16, 16, +,
2325 ATOMIC_CMPXCHG_CPT(fixed2, andb_cpt, kmp_int16, 16, &,
2327 ATOMIC_CMPXCHG_CPT(fixed2, div_cpt, kmp_int16, 16, /,
2329 ATOMIC_CMPXCHG_CPT(fixed2u, div_cpt, kmp_uint16, 16, /,
2331 ATOMIC_CMPXCHG_CPT(fixed2, mul_cpt, kmp_int16, 16, *,
2333 ATOMIC_CMPXCHG_CPT(fixed2, orb_cpt, kmp_int16, 16, |,
2335 ATOMIC_CMPXCHG_CPT(fixed2, shl_cpt, kmp_int16, 16, <<,
2337 ATOMIC_CMPXCHG_CPT(fixed2, shr_cpt, kmp_int16, 16, >>,
2339 ATOMIC_CMPXCHG_CPT(fixed2u, shr_cpt, kmp_uint16, 16, >>,
2341 ATOMIC_CMPXCHG_CPT(fixed2, sub_cpt, kmp_int16, 16, -,
2343 ATOMIC_CMPXCHG_CPT(fixed2, xor_cpt, kmp_int16, 16, ^,
2345 ATOMIC_CMPXCHG_CPT(fixed4, andb_cpt, kmp_int32, 32, &,
2347 ATOMIC_CMPXCHG_CPT(fixed4, div_cpt, kmp_int32, 32, /,
2349 ATOMIC_CMPXCHG_CPT(fixed4u, div_cpt, kmp_uint32, 32, /,
2351 ATOMIC_CMPXCHG_CPT(fixed4, mul_cpt, kmp_int32, 32, *,
2353 ATOMIC_CMPXCHG_CPT(fixed4, orb_cpt, kmp_int32, 32, |,
2355 ATOMIC_CMPXCHG_CPT(fixed4, shl_cpt, kmp_int32, 32, <<,
2357 ATOMIC_CMPXCHG_CPT(fixed4, shr_cpt, kmp_int32, 32, >>,
2359 ATOMIC_CMPXCHG_CPT(fixed4u, shr_cpt, kmp_uint32, 32, >>,
2361 ATOMIC_CMPXCHG_CPT(fixed4, xor_cpt, kmp_int32, 32, ^,
2363 ATOMIC_CMPXCHG_CPT(fixed8, andb_cpt, kmp_int64, 64, &,
2365 ATOMIC_CMPXCHG_CPT(fixed8, div_cpt, kmp_int64, 64, /,
2367 ATOMIC_CMPXCHG_CPT(fixed8u, div_cpt, kmp_uint64, 64, /,
2369 ATOMIC_CMPXCHG_CPT(fixed8, mul_cpt, kmp_int64, 64, *,
2371 ATOMIC_CMPXCHG_CPT(fixed8, orb_cpt, kmp_int64, 64, |,
2373 ATOMIC_CMPXCHG_CPT(fixed8, shl_cpt, kmp_int64, 64, <<,
2375 ATOMIC_CMPXCHG_CPT(fixed8, shr_cpt, kmp_int64, 64, >>,
2377 ATOMIC_CMPXCHG_CPT(fixed8u, shr_cpt, kmp_uint64, 64, >>,
2379 ATOMIC_CMPXCHG_CPT(fixed8, xor_cpt, kmp_int64, 64, ^,
2381 ATOMIC_CMPXCHG_CPT(float4, div_cpt, kmp_real32, 32, /,
2383 ATOMIC_CMPXCHG_CPT(float4, mul_cpt, kmp_real32, 32, *,
2385 ATOMIC_CMPXCHG_CPT(float8, div_cpt, kmp_real64, 64, /,
2387 ATOMIC_CMPXCHG_CPT(float8, mul_cpt, kmp_real64, 64, *,
2399 #define ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \
2400 TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID##_##RTYPE_ID( \
2401 ident_t *id_ref, int gtid, TYPE *lhs, RTYPE rhs, int flag) { \
2402 KMP_DEBUG_ASSERT(__kmp_init_serial); \
2404 ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_" #RTYPE_ID ": T#%d\n", \
2408 #define ATOMIC_CMPXCHG_CPT_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, \
2409 RTYPE, LCK_ID, MASK, GOMP_FLAG) \
2410 ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \
2412 OP_GOMP_CRITICAL_CPT(TYPE, OP, GOMP_FLAG) \
2413 OP_CMPXCHG_CPT(TYPE, BITS, OP) \
2417 #define ATOMIC_CRITICAL_CPT_MIX(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, \
2418 LCK_ID, GOMP_FLAG) \
2419 ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \
2421 OP_GOMP_CRITICAL_CPT(TYPE, OP, GOMP_FLAG) \
2422 OP_UPDATE_CRITICAL_CPT(TYPE, OP, LCK_ID) \
2425 ATOMIC_CMPXCHG_CPT_MIX(fixed1,
char, add_cpt, 8, +, fp, _Quad, 1i, 0,
2427 ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, add_cpt, 8, +, fp, _Quad, 1i, 0,
2429 ATOMIC_CMPXCHG_CPT_MIX(fixed1,
char, sub_cpt, 8, -, fp, _Quad, 1i, 0,
2431 ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, sub_cpt, 8, -, fp, _Quad, 1i, 0,
2433 ATOMIC_CMPXCHG_CPT_MIX(fixed1,
char, mul_cpt, 8, *, fp, _Quad, 1i, 0,
2435 ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, mul_cpt, 8, *, fp, _Quad, 1i, 0,
2437 ATOMIC_CMPXCHG_CPT_MIX(fixed1,
char, div_cpt, 8, /, fp, _Quad, 1i, 0,
2439 ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, div_cpt, 8, /, fp, _Quad, 1i, 0,
2442 ATOMIC_CMPXCHG_CPT_MIX(fixed2,
short, add_cpt, 16, +, fp, _Quad, 2i, 1,
2444 ATOMIC_CMPXCHG_CPT_MIX(fixed2u, ushort, add_cpt, 16, +, fp, _Quad, 2i, 1,
2446 ATOMIC_CMPXCHG_CPT_MIX(fixed2,
short, sub_cpt, 16, -, fp, _Quad, 2i, 1,
2448 ATOMIC_CMPXCHG_CPT_MIX(fixed2u, ushort, sub_cpt, 16, -, fp, _Quad, 2i, 1,
2450 ATOMIC_CMPXCHG_CPT_MIX(fixed2,
short, mul_cpt, 16, *, fp, _Quad, 2i, 1,
2452 ATOMIC_CMPXCHG_CPT_MIX(fixed2u, ushort, mul_cpt, 16, *, fp, _Quad, 2i, 1,
2454 ATOMIC_CMPXCHG_CPT_MIX(fixed2,
short, div_cpt, 16, /, fp, _Quad, 2i, 1,
2456 ATOMIC_CMPXCHG_CPT_MIX(fixed2u, ushort, div_cpt, 16, /, fp, _Quad, 2i, 1,
2459 ATOMIC_CMPXCHG_CPT_MIX(fixed4, kmp_int32, add_cpt, 32, +, fp, _Quad, 4i, 3,
2461 ATOMIC_CMPXCHG_CPT_MIX(fixed4u, kmp_uint32, add_cpt, 32, +, fp, _Quad, 4i, 3,
2463 ATOMIC_CMPXCHG_CPT_MIX(fixed4, kmp_int32, sub_cpt, 32, -, fp, _Quad, 4i, 3,
2465 ATOMIC_CMPXCHG_CPT_MIX(fixed4u, kmp_uint32, sub_cpt, 32, -, fp, _Quad, 4i, 3,
2467 ATOMIC_CMPXCHG_CPT_MIX(fixed4, kmp_int32, mul_cpt, 32, *, fp, _Quad, 4i, 3,
2469 ATOMIC_CMPXCHG_CPT_MIX(fixed4u, kmp_uint32, mul_cpt, 32, *, fp, _Quad, 4i, 3,
2471 ATOMIC_CMPXCHG_CPT_MIX(fixed4, kmp_int32, div_cpt, 32, /, fp, _Quad, 4i, 3,
2473 ATOMIC_CMPXCHG_CPT_MIX(fixed4u, kmp_uint32, div_cpt, 32, /, fp, _Quad, 4i, 3,
2476 ATOMIC_CMPXCHG_CPT_MIX(fixed8, kmp_int64, add_cpt, 64, +, fp, _Quad, 8i, 7,
2478 ATOMIC_CMPXCHG_CPT_MIX(fixed8u, kmp_uint64, add_cpt, 64, +, fp, _Quad, 8i, 7,
2480 ATOMIC_CMPXCHG_CPT_MIX(fixed8, kmp_int64, sub_cpt, 64, -, fp, _Quad, 8i, 7,
2482 ATOMIC_CMPXCHG_CPT_MIX(fixed8u, kmp_uint64, sub_cpt, 64, -, fp, _Quad, 8i, 7,
2484 ATOMIC_CMPXCHG_CPT_MIX(fixed8, kmp_int64, mul_cpt, 64, *, fp, _Quad, 8i, 7,
2486 ATOMIC_CMPXCHG_CPT_MIX(fixed8u, kmp_uint64, mul_cpt, 64, *, fp, _Quad, 8i, 7,
2488 ATOMIC_CMPXCHG_CPT_MIX(fixed8, kmp_int64, div_cpt, 64, /, fp, _Quad, 8i, 7,
2490 ATOMIC_CMPXCHG_CPT_MIX(fixed8u, kmp_uint64, div_cpt, 64, /, fp, _Quad, 8i, 7,
2493 ATOMIC_CMPXCHG_CPT_MIX(float4, kmp_real32, add_cpt, 32, +, fp, _Quad, 4r, 3,
2495 ATOMIC_CMPXCHG_CPT_MIX(float4, kmp_real32, sub_cpt, 32, -, fp, _Quad, 4r, 3,
2497 ATOMIC_CMPXCHG_CPT_MIX(float4, kmp_real32, mul_cpt, 32, *, fp, _Quad, 4r, 3,
2499 ATOMIC_CMPXCHG_CPT_MIX(float4, kmp_real32, div_cpt, 32, /, fp, _Quad, 4r, 3,
2502 ATOMIC_CMPXCHG_CPT_MIX(float8, kmp_real64, add_cpt, 64, +, fp, _Quad, 8r, 7,
2504 ATOMIC_CMPXCHG_CPT_MIX(float8, kmp_real64, sub_cpt, 64, -, fp, _Quad, 8r, 7,
2506 ATOMIC_CMPXCHG_CPT_MIX(float8, kmp_real64, mul_cpt, 64, *, fp, _Quad, 8r, 7,
2508 ATOMIC_CMPXCHG_CPT_MIX(float8, kmp_real64, div_cpt, 64, /, fp, _Quad, 8r, 7,
2511 ATOMIC_CRITICAL_CPT_MIX(float10,
long double, add_cpt, +, fp, _Quad, 10r,
2513 ATOMIC_CRITICAL_CPT_MIX(float10,
long double, sub_cpt, -, fp, _Quad, 10r,
2515 ATOMIC_CRITICAL_CPT_MIX(float10,
long double, mul_cpt, *, fp, _Quad, 10r,
2517 ATOMIC_CRITICAL_CPT_MIX(float10,
long double, div_cpt, /, fp, _Quad, 10r,
2531 #define OP_CRITICAL_L_CPT(OP, LCK_ID) \
2532 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2536 (*lhs) = new_value; \
2538 new_value = (*lhs); \
2542 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
2545 #ifdef KMP_GOMP_COMPAT
2546 #define OP_GOMP_CRITICAL_L_CPT(OP, FLAG) \
2547 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
2549 OP_CRITICAL_L_CPT(OP, 0); \
2553 #define OP_GOMP_CRITICAL_L_CPT(OP, FLAG)
2558 #define ATOMIC_CMPX_L_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2559 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2562 OP_GOMP_CRITICAL_L_CPT(= *lhs OP, GOMP_FLAG) \
2563 OP_CMPXCHG_CPT(TYPE, BITS, OP) \
2566 ATOMIC_CMPX_L_CPT(fixed1, andl_cpt,
char, 8, &&,
2568 ATOMIC_CMPX_L_CPT(fixed1, orl_cpt,
char, 8, ||,
2570 ATOMIC_CMPX_L_CPT(fixed2, andl_cpt,
short, 16, &&,
2572 ATOMIC_CMPX_L_CPT(fixed2, orl_cpt,
short, 16, ||,
2574 ATOMIC_CMPX_L_CPT(fixed4, andl_cpt, kmp_int32, 32, &&,
2576 ATOMIC_CMPX_L_CPT(fixed4, orl_cpt, kmp_int32, 32, ||,
2578 ATOMIC_CMPX_L_CPT(fixed8, andl_cpt, kmp_int64, 64, &&,
2580 ATOMIC_CMPX_L_CPT(fixed8, orl_cpt, kmp_int64, 64, ||,
2592 #define MIN_MAX_CRITSECT_CPT(OP, LCK_ID) \
2593 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2595 if (*lhs OP rhs) { \
2601 new_value = old_value; \
2605 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2609 #ifdef KMP_GOMP_COMPAT
2610 #define GOMP_MIN_MAX_CRITSECT_CPT(OP, FLAG) \
2611 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
2613 MIN_MAX_CRITSECT_CPT(OP, 0); \
2616 #define GOMP_MIN_MAX_CRITSECT_CPT(OP, FLAG)
2620 #define MIN_MAX_CMPXCHG_CPT(TYPE, BITS, OP) \
2622 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2625 old_value = temp_val; \
2626 while (old_value OP rhs && \
2627 !KMP_COMPARE_AND_STORE_ACQ##BITS( \
2628 (kmp_int##BITS *)lhs, \
2629 *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
2630 *VOLATILE_CAST(kmp_int##BITS *) & rhs)) { \
2632 old_value = temp_val; \
2642 #define MIN_MAX_CRITICAL_CPT(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
2643 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2644 TYPE new_value, old_value; \
2645 if (*lhs OP rhs) { \
2646 GOMP_MIN_MAX_CRITSECT_CPT(OP, GOMP_FLAG) \
2647 MIN_MAX_CRITSECT_CPT(OP, LCK_ID) \
2652 #define MIN_MAX_COMPXCHG_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2653 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2654 TYPE new_value, old_value; \
2656 if (*lhs OP rhs) { \
2657 GOMP_MIN_MAX_CRITSECT_CPT(OP, GOMP_FLAG) \
2658 MIN_MAX_CMPXCHG_CPT(TYPE, BITS, OP) \
2663 MIN_MAX_COMPXCHG_CPT(fixed1, max_cpt,
char, 8, <,
2665 MIN_MAX_COMPXCHG_CPT(fixed1, min_cpt,
char, 8, >,
2667 MIN_MAX_COMPXCHG_CPT(fixed2, max_cpt,
short, 16, <,
2669 MIN_MAX_COMPXCHG_CPT(fixed2, min_cpt,
short, 16, >,
2671 MIN_MAX_COMPXCHG_CPT(fixed4, max_cpt, kmp_int32, 32, <,
2673 MIN_MAX_COMPXCHG_CPT(fixed4, min_cpt, kmp_int32, 32, >,
2675 MIN_MAX_COMPXCHG_CPT(fixed8, max_cpt, kmp_int64, 64, <,
2677 MIN_MAX_COMPXCHG_CPT(fixed8, min_cpt, kmp_int64, 64, >,
2679 MIN_MAX_COMPXCHG_CPT(float4, max_cpt, kmp_real32, 32, <,
2681 MIN_MAX_COMPXCHG_CPT(float4, min_cpt, kmp_real32, 32, >,
2683 MIN_MAX_COMPXCHG_CPT(float8, max_cpt, kmp_real64, 64, <,
2685 MIN_MAX_COMPXCHG_CPT(float8, min_cpt, kmp_real64, 64, >,
2688 MIN_MAX_CRITICAL_CPT(float16, max_cpt, QUAD_LEGACY, <, 16r,
2690 MIN_MAX_CRITICAL_CPT(float16, min_cpt, QUAD_LEGACY, >, 16r,
2693 MIN_MAX_CRITICAL_CPT(float16, max_a16_cpt, Quad_a16_t, <, 16r,
2695 MIN_MAX_CRITICAL_CPT(float16, min_a16_cpt, Quad_a16_t, >, 16r,
2701 #ifdef KMP_GOMP_COMPAT
2702 #define OP_GOMP_CRITICAL_EQV_CPT(OP, FLAG) \
2703 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
2705 OP_CRITICAL_CPT(OP, 0); \
2708 #define OP_GOMP_CRITICAL_EQV_CPT(OP, FLAG)
2711 #define ATOMIC_CMPX_EQV_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2712 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2715 OP_GOMP_CRITICAL_EQV_CPT(^= (TYPE) ~, GOMP_FLAG) \
2716 OP_CMPXCHG_CPT(TYPE, BITS, OP) \
2721 ATOMIC_CMPXCHG_CPT(fixed1, neqv_cpt, kmp_int8, 8, ^,
2723 ATOMIC_CMPXCHG_CPT(fixed2, neqv_cpt, kmp_int16, 16, ^,
2725 ATOMIC_CMPXCHG_CPT(fixed4, neqv_cpt, kmp_int32, 32, ^,
2727 ATOMIC_CMPXCHG_CPT(fixed8, neqv_cpt, kmp_int64, 64, ^,
2729 ATOMIC_CMPX_EQV_CPT(fixed1, eqv_cpt, kmp_int8, 8, ^~,
2731 ATOMIC_CMPX_EQV_CPT(fixed2, eqv_cpt, kmp_int16, 16, ^~,
2733 ATOMIC_CMPX_EQV_CPT(fixed4, eqv_cpt, kmp_int32, 32, ^~,
2735 ATOMIC_CMPX_EQV_CPT(fixed8, eqv_cpt, kmp_int64, 64, ^~,
2744 #define ATOMIC_CRITICAL_CPT(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
2745 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2747 OP_GOMP_CRITICAL_CPT(TYPE, OP, GOMP_FLAG) \
2748 OP_UPDATE_CRITICAL_CPT(TYPE, OP, LCK_ID) \
2754 #define OP_CRITICAL_CPT_WRK(OP, LCK_ID) \
2755 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2765 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2769 #ifdef KMP_GOMP_COMPAT
2770 #define OP_GOMP_CRITICAL_CPT_WRK(OP, FLAG) \
2771 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
2773 OP_CRITICAL_CPT_WRK(OP## =, 0); \
2776 #define OP_GOMP_CRITICAL_CPT_WRK(OP, FLAG)
2780 #define ATOMIC_BEGIN_WRK(TYPE_ID, OP_ID, TYPE) \
2781 void __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, TYPE *lhs, \
2782 TYPE rhs, TYPE *out, int flag) { \
2783 KMP_DEBUG_ASSERT(__kmp_init_serial); \
2784 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid));
2787 #define ATOMIC_CRITICAL_CPT_WRK(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
2788 ATOMIC_BEGIN_WRK(TYPE_ID, OP_ID, TYPE) \
2789 OP_GOMP_CRITICAL_CPT_WRK(OP, GOMP_FLAG) \
2790 OP_CRITICAL_CPT_WRK(OP## =, LCK_ID) \
2796 ATOMIC_CRITICAL_CPT(float10, add_cpt,
long double, +, 10r,
2798 ATOMIC_CRITICAL_CPT(float10, sub_cpt,
long double, -, 10r,
2800 ATOMIC_CRITICAL_CPT(float10, mul_cpt,
long double, *, 10r,
2802 ATOMIC_CRITICAL_CPT(float10, div_cpt,
long double, /, 10r,
2806 ATOMIC_CRITICAL_CPT(float16, add_cpt, QUAD_LEGACY, +, 16r,
2808 ATOMIC_CRITICAL_CPT(float16, sub_cpt, QUAD_LEGACY, -, 16r,
2810 ATOMIC_CRITICAL_CPT(float16, mul_cpt, QUAD_LEGACY, *, 16r,
2812 ATOMIC_CRITICAL_CPT(float16, div_cpt, QUAD_LEGACY, /, 16r,
2815 ATOMIC_CRITICAL_CPT(float16, add_a16_cpt, Quad_a16_t, +, 16r,
2817 ATOMIC_CRITICAL_CPT(float16, sub_a16_cpt, Quad_a16_t, -, 16r,
2819 ATOMIC_CRITICAL_CPT(float16, mul_a16_cpt, Quad_a16_t, *, 16r,
2821 ATOMIC_CRITICAL_CPT(float16, div_a16_cpt, Quad_a16_t, /, 16r,
2829 ATOMIC_CRITICAL_CPT_WRK(cmplx4, add_cpt, kmp_cmplx32, +, 8c,
2831 ATOMIC_CRITICAL_CPT_WRK(cmplx4, sub_cpt, kmp_cmplx32, -, 8c,
2833 ATOMIC_CRITICAL_CPT_WRK(cmplx4, mul_cpt, kmp_cmplx32, *, 8c,
2835 ATOMIC_CRITICAL_CPT_WRK(cmplx4, div_cpt, kmp_cmplx32, /, 8c,
2838 ATOMIC_CRITICAL_CPT(cmplx8, add_cpt, kmp_cmplx64, +, 16c,
2840 ATOMIC_CRITICAL_CPT(cmplx8, sub_cpt, kmp_cmplx64, -, 16c,
2842 ATOMIC_CRITICAL_CPT(cmplx8, mul_cpt, kmp_cmplx64, *, 16c,
2844 ATOMIC_CRITICAL_CPT(cmplx8, div_cpt, kmp_cmplx64, /, 16c,
2846 ATOMIC_CRITICAL_CPT(cmplx10, add_cpt, kmp_cmplx80, +, 20c,
2848 ATOMIC_CRITICAL_CPT(cmplx10, sub_cpt, kmp_cmplx80, -, 20c,
2850 ATOMIC_CRITICAL_CPT(cmplx10, mul_cpt, kmp_cmplx80, *, 20c,
2852 ATOMIC_CRITICAL_CPT(cmplx10, div_cpt, kmp_cmplx80, /, 20c,
2855 ATOMIC_CRITICAL_CPT(cmplx16, add_cpt, CPLX128_LEG, +, 32c,
2857 ATOMIC_CRITICAL_CPT(cmplx16, sub_cpt, CPLX128_LEG, -, 32c,
2859 ATOMIC_CRITICAL_CPT(cmplx16, mul_cpt, CPLX128_LEG, *, 32c,
2861 ATOMIC_CRITICAL_CPT(cmplx16, div_cpt, CPLX128_LEG, /, 32c,
2864 ATOMIC_CRITICAL_CPT(cmplx16, add_a16_cpt, kmp_cmplx128_a16_t, +, 32c,
2866 ATOMIC_CRITICAL_CPT(cmplx16, sub_a16_cpt, kmp_cmplx128_a16_t, -, 32c,
2868 ATOMIC_CRITICAL_CPT(cmplx16, mul_a16_cpt, kmp_cmplx128_a16_t, *, 32c,
2870 ATOMIC_CRITICAL_CPT(cmplx16, div_a16_cpt, kmp_cmplx128_a16_t, /, 32c,
2885 #define OP_CRITICAL_CPT_REV(TYPE, OP, LCK_ID) \
2886 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2890 (*lhs) = (TYPE)((rhs)OP(*lhs)); \
2891 new_value = (*lhs); \
2893 new_value = (*lhs); \
2894 (*lhs) = (TYPE)((rhs)OP(*lhs)); \
2896 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2900 #ifdef KMP_GOMP_COMPAT
2901 #define OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, FLAG) \
2902 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
2904 OP_CRITICAL_CPT_REV(TYPE, OP, 0); \
2907 #define OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, FLAG)
2917 #define OP_CMPXCHG_CPT_REV(TYPE, BITS, OP) \
2919 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2920 TYPE old_value, new_value; \
2922 old_value = temp_val; \
2923 new_value = (TYPE)(rhs OP old_value); \
2924 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
2925 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
2926 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
2928 old_value = temp_val; \
2929 new_value = (TYPE)(rhs OP old_value); \
2938 #define ATOMIC_CMPXCHG_CPT_REV(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2939 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2942 OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, GOMP_FLAG) \
2943 OP_CMPXCHG_CPT_REV(TYPE, BITS, OP) \
2946 ATOMIC_CMPXCHG_CPT_REV(fixed1, div_cpt_rev, kmp_int8, 8, /,
2948 ATOMIC_CMPXCHG_CPT_REV(fixed1u, div_cpt_rev, kmp_uint8, 8, /,
2950 ATOMIC_CMPXCHG_CPT_REV(fixed1, shl_cpt_rev, kmp_int8, 8, <<,
2952 ATOMIC_CMPXCHG_CPT_REV(fixed1, shr_cpt_rev, kmp_int8, 8, >>,
2954 ATOMIC_CMPXCHG_CPT_REV(fixed1u, shr_cpt_rev, kmp_uint8, 8, >>,
2956 ATOMIC_CMPXCHG_CPT_REV(fixed1, sub_cpt_rev, kmp_int8, 8, -,
2958 ATOMIC_CMPXCHG_CPT_REV(fixed2, div_cpt_rev, kmp_int16, 16, /,
2960 ATOMIC_CMPXCHG_CPT_REV(fixed2u, div_cpt_rev, kmp_uint16, 16, /,
2962 ATOMIC_CMPXCHG_CPT_REV(fixed2, shl_cpt_rev, kmp_int16, 16, <<,
2964 ATOMIC_CMPXCHG_CPT_REV(fixed2, shr_cpt_rev, kmp_int16, 16, >>,
2966 ATOMIC_CMPXCHG_CPT_REV(fixed2u, shr_cpt_rev, kmp_uint16, 16, >>,
2968 ATOMIC_CMPXCHG_CPT_REV(fixed2, sub_cpt_rev, kmp_int16, 16, -,
2970 ATOMIC_CMPXCHG_CPT_REV(fixed4, div_cpt_rev, kmp_int32, 32, /,
2972 ATOMIC_CMPXCHG_CPT_REV(fixed4u, div_cpt_rev, kmp_uint32, 32, /,
2974 ATOMIC_CMPXCHG_CPT_REV(fixed4, shl_cpt_rev, kmp_int32, 32, <<,
2976 ATOMIC_CMPXCHG_CPT_REV(fixed4, shr_cpt_rev, kmp_int32, 32, >>,
2978 ATOMIC_CMPXCHG_CPT_REV(fixed4u, shr_cpt_rev, kmp_uint32, 32, >>,
2980 ATOMIC_CMPXCHG_CPT_REV(fixed4, sub_cpt_rev, kmp_int32, 32, -,
2982 ATOMIC_CMPXCHG_CPT_REV(fixed8, div_cpt_rev, kmp_int64, 64, /,
2984 ATOMIC_CMPXCHG_CPT_REV(fixed8u, div_cpt_rev, kmp_uint64, 64, /,
2986 ATOMIC_CMPXCHG_CPT_REV(fixed8, shl_cpt_rev, kmp_int64, 64, <<,
2988 ATOMIC_CMPXCHG_CPT_REV(fixed8, shr_cpt_rev, kmp_int64, 64, >>,
2990 ATOMIC_CMPXCHG_CPT_REV(fixed8u, shr_cpt_rev, kmp_uint64, 64, >>,
2992 ATOMIC_CMPXCHG_CPT_REV(fixed8, sub_cpt_rev, kmp_int64, 64, -,
2994 ATOMIC_CMPXCHG_CPT_REV(float4, div_cpt_rev, kmp_real32, 32, /,
2996 ATOMIC_CMPXCHG_CPT_REV(float4, sub_cpt_rev, kmp_real32, 32, -,
2998 ATOMIC_CMPXCHG_CPT_REV(float8, div_cpt_rev, kmp_real64, 64, /,
3000 ATOMIC_CMPXCHG_CPT_REV(float8, sub_cpt_rev, kmp_real64, 64, -,
3010 #define ATOMIC_CRITICAL_CPT_REV(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
3011 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
3014 OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, GOMP_FLAG) \
3015 OP_CRITICAL_CPT_REV(TYPE, OP, LCK_ID) \
3020 ATOMIC_CRITICAL_CPT_REV(float10, sub_cpt_rev,
long double, -, 10r,
3022 ATOMIC_CRITICAL_CPT_REV(float10, div_cpt_rev,
long double, /, 10r,
3026 ATOMIC_CRITICAL_CPT_REV(float16, sub_cpt_rev, QUAD_LEGACY, -, 16r,
3028 ATOMIC_CRITICAL_CPT_REV(float16, div_cpt_rev, QUAD_LEGACY, /, 16r,
3031 ATOMIC_CRITICAL_CPT_REV(float16, sub_a16_cpt_rev, Quad_a16_t, -, 16r,
3033 ATOMIC_CRITICAL_CPT_REV(float16, div_a16_cpt_rev, Quad_a16_t, /, 16r,
3043 #define OP_CRITICAL_CPT_REV_WRK(OP, LCK_ID) \
3044 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
3047 (*lhs) = (rhs)OP(*lhs); \
3051 (*lhs) = (rhs)OP(*lhs); \
3054 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
3058 #ifdef KMP_GOMP_COMPAT
3059 #define OP_GOMP_CRITICAL_CPT_REV_WRK(OP, FLAG) \
3060 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
3062 OP_CRITICAL_CPT_REV_WRK(OP, 0); \
3065 #define OP_GOMP_CRITICAL_CPT_REV_WRK(OP, FLAG)
3069 #define ATOMIC_CRITICAL_CPT_REV_WRK(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, \
3071 ATOMIC_BEGIN_WRK(TYPE_ID, OP_ID, TYPE) \
3072 OP_GOMP_CRITICAL_CPT_REV_WRK(OP, GOMP_FLAG) \
3073 OP_CRITICAL_CPT_REV_WRK(OP, LCK_ID) \
3079 ATOMIC_CRITICAL_CPT_REV_WRK(cmplx4, sub_cpt_rev, kmp_cmplx32, -, 8c,
3081 ATOMIC_CRITICAL_CPT_REV_WRK(cmplx4, div_cpt_rev, kmp_cmplx32, /, 8c,
3084 ATOMIC_CRITICAL_CPT_REV(cmplx8, sub_cpt_rev, kmp_cmplx64, -, 16c,
3086 ATOMIC_CRITICAL_CPT_REV(cmplx8, div_cpt_rev, kmp_cmplx64, /, 16c,
3088 ATOMIC_CRITICAL_CPT_REV(cmplx10, sub_cpt_rev, kmp_cmplx80, -, 20c,
3090 ATOMIC_CRITICAL_CPT_REV(cmplx10, div_cpt_rev, kmp_cmplx80, /, 20c,
3093 ATOMIC_CRITICAL_CPT_REV(cmplx16, sub_cpt_rev, CPLX128_LEG, -, 32c,
3095 ATOMIC_CRITICAL_CPT_REV(cmplx16, div_cpt_rev, CPLX128_LEG, /, 32c,
3098 ATOMIC_CRITICAL_CPT_REV(cmplx16, sub_a16_cpt_rev, kmp_cmplx128_a16_t, -, 32c,
3100 ATOMIC_CRITICAL_CPT_REV(cmplx16, div_a16_cpt_rev, kmp_cmplx128_a16_t, /, 32c,
3114 #define ATOMIC_CMPXCHG_CPT_REV_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, \
3115 RTYPE, LCK_ID, MASK, GOMP_FLAG) \
3116 ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \
3118 OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, GOMP_FLAG) \
3119 OP_CMPXCHG_CPT_REV(TYPE, BITS, OP) \
3123 #define ATOMIC_CRITICAL_CPT_REV_MIX(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, \
3124 LCK_ID, GOMP_FLAG) \
3125 ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \
3127 OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, GOMP_FLAG) \
3128 OP_CRITICAL_CPT_REV(TYPE, OP, LCK_ID) \
3131 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1,
char, sub_cpt_rev, 8, -, fp, _Quad, 1i, 0,
3133 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1u, uchar, sub_cpt_rev, 8, -, fp, _Quad, 1i, 0,
3135 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1,
char, div_cpt_rev, 8, /, fp, _Quad, 1i, 0,
3137 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1u, uchar, div_cpt_rev, 8, /, fp, _Quad, 1i, 0,
3140 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2,
short, sub_cpt_rev, 16, -, fp, _Quad, 2i, 1,
3142 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2u, ushort, sub_cpt_rev, 16, -, fp, _Quad, 2i,
3145 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2,
short, div_cpt_rev, 16, /, fp, _Quad, 2i, 1,
3147 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2u, ushort, div_cpt_rev, 16, /, fp, _Quad, 2i,
3151 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4, kmp_int32, sub_cpt_rev, 32, -, fp, _Quad, 4i,
3153 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4u, kmp_uint32, sub_cpt_rev, 32, -, fp, _Quad,
3155 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4, kmp_int32, div_cpt_rev, 32, /, fp, _Quad, 4i,
3157 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4u, kmp_uint32, div_cpt_rev, 32, /, fp, _Quad,
3160 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8, kmp_int64, sub_cpt_rev, 64, -, fp, _Quad, 8i,
3163 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8u, kmp_uint64, sub_cpt_rev, 64, -, fp, _Quad,
3166 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8, kmp_int64, div_cpt_rev, 64, /, fp, _Quad, 8i,
3169 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8u, kmp_uint64, div_cpt_rev, 64, /, fp, _Quad,
3173 ATOMIC_CMPXCHG_CPT_REV_MIX(float4, kmp_real32, sub_cpt_rev, 32, -, fp, _Quad,
3176 ATOMIC_CMPXCHG_CPT_REV_MIX(float4, kmp_real32, div_cpt_rev, 32, /, fp, _Quad,
3180 ATOMIC_CMPXCHG_CPT_REV_MIX(float8, kmp_real64, sub_cpt_rev, 64, -, fp, _Quad,
3183 ATOMIC_CMPXCHG_CPT_REV_MIX(float8, kmp_real64, div_cpt_rev, 64, /, fp, _Quad,
3187 ATOMIC_CRITICAL_CPT_REV_MIX(float10,
long double, sub_cpt_rev, -, fp, _Quad,
3189 ATOMIC_CRITICAL_CPT_REV_MIX(float10,
long double, div_cpt_rev, /, fp, _Quad,
3196 #define ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \
3197 TYPE __kmpc_atomic_##TYPE_ID##_swp(ident_t *id_ref, int gtid, TYPE *lhs, \
3199 KMP_DEBUG_ASSERT(__kmp_init_serial); \
3200 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_swp: T#%d\n", gtid));
3202 #define CRITICAL_SWP(LCK_ID) \
3203 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
3205 old_value = (*lhs); \
3208 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
3212 #ifdef KMP_GOMP_COMPAT
3213 #define GOMP_CRITICAL_SWP(FLAG) \
3214 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
3219 #define GOMP_CRITICAL_SWP(FLAG)
3222 #define ATOMIC_XCHG_SWP(TYPE_ID, TYPE, BITS, GOMP_FLAG) \
3223 ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \
3225 GOMP_CRITICAL_SWP(GOMP_FLAG) \
3226 old_value = KMP_XCHG_FIXED##BITS(lhs, rhs); \
3230 #define ATOMIC_XCHG_FLOAT_SWP(TYPE_ID, TYPE, BITS, GOMP_FLAG) \
3231 ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \
3233 GOMP_CRITICAL_SWP(GOMP_FLAG) \
3234 old_value = KMP_XCHG_REAL##BITS(lhs, rhs); \
3239 #define CMPXCHG_SWP(TYPE, BITS) \
3241 TYPE KMP_ATOMIC_VOLATILE temp_val; \
3242 TYPE old_value, new_value; \
3244 old_value = temp_val; \
3246 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
3247 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
3248 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
3250 old_value = temp_val; \
3257 #define ATOMIC_CMPXCHG_SWP(TYPE_ID, TYPE, BITS, GOMP_FLAG) \
3258 ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \
3261 GOMP_CRITICAL_SWP(GOMP_FLAG) \
3262 CMPXCHG_SWP(TYPE, BITS) \
3265 ATOMIC_XCHG_SWP(fixed1, kmp_int8, 8, KMP_ARCH_X86)
3266 ATOMIC_XCHG_SWP(fixed2, kmp_int16, 16, KMP_ARCH_X86)
3267 ATOMIC_XCHG_SWP(fixed4, kmp_int32, 32, KMP_ARCH_X86)
3269 ATOMIC_XCHG_FLOAT_SWP(float4, kmp_real32, 32,
3273 ATOMIC_CMPXCHG_SWP(fixed8, kmp_int64, 64,
3275 ATOMIC_CMPXCHG_SWP(float8, kmp_real64, 64,
3278 ATOMIC_XCHG_SWP(fixed8, kmp_int64, 64, KMP_ARCH_X86)
3279 ATOMIC_XCHG_FLOAT_SWP(float8, kmp_real64, 64,
3286 #define ATOMIC_CRITICAL_SWP(TYPE_ID, TYPE, LCK_ID, GOMP_FLAG) \
3287 ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \
3289 GOMP_CRITICAL_SWP(GOMP_FLAG) \
3290 CRITICAL_SWP(LCK_ID) \
3298 #define ATOMIC_BEGIN_SWP_WRK(TYPE_ID, TYPE) \
3299 void __kmpc_atomic_##TYPE_ID##_swp(ident_t *id_ref, int gtid, TYPE *lhs, \
3300 TYPE rhs, TYPE *out) { \
3301 KMP_DEBUG_ASSERT(__kmp_init_serial); \
3302 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_swp: T#%d\n", gtid));
3304 #define CRITICAL_SWP_WRK(LCK_ID) \
3305 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
3310 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
3314 #ifdef KMP_GOMP_COMPAT
3315 #define GOMP_CRITICAL_SWP_WRK(FLAG) \
3316 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
3318 CRITICAL_SWP_WRK(0); \
3321 #define GOMP_CRITICAL_SWP_WRK(FLAG)
3325 #define ATOMIC_CRITICAL_SWP_WRK(TYPE_ID, TYPE, LCK_ID, GOMP_FLAG) \
3326 ATOMIC_BEGIN_SWP_WRK(TYPE_ID, TYPE) \
3328 GOMP_CRITICAL_SWP_WRK(GOMP_FLAG) \
3329 CRITICAL_SWP_WRK(LCK_ID) \
3333 ATOMIC_CRITICAL_SWP(float10,
long double, 10r, 1)
3335 ATOMIC_CRITICAL_SWP(float16, QUAD_LEGACY, 16r, 1)
3338 ATOMIC_CRITICAL_SWP_WRK(cmplx4, kmp_cmplx32, 8c, 1)
3343 ATOMIC_CRITICAL_SWP(cmplx8, kmp_cmplx64, 16c, 1)
3344 ATOMIC_CRITICAL_SWP(cmplx10, kmp_cmplx80, 20c, 1)
3346 ATOMIC_CRITICAL_SWP(cmplx16, CPLX128_LEG, 32c, 1)
3348 ATOMIC_CRITICAL_SWP(float16_a16, Quad_a16_t, 16r,
3350 ATOMIC_CRITICAL_SWP(cmplx16_a16, kmp_cmplx128_a16_t, 32c,
3364 void __kmpc_atomic_1(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3365 void (*f)(
void *,
void *,
void *)) {
3366 KMP_DEBUG_ASSERT(__kmp_init_serial);
3369 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
3375 kmp_int8 old_value, new_value;
3377 old_value = *(kmp_int8 *)lhs;
3378 (*f)(&new_value, &old_value, rhs);
3381 while (!KMP_COMPARE_AND_STORE_ACQ8((kmp_int8 *)lhs, *(kmp_int8 *)&old_value,
3382 *(kmp_int8 *)&new_value)) {
3385 old_value = *(kmp_int8 *)lhs;
3386 (*f)(&new_value, &old_value, rhs);
3393 #ifdef KMP_GOMP_COMPAT
3394 if (__kmp_atomic_mode == 2) {
3395 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3398 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_1i, gtid);
3400 (*f)(lhs, lhs, rhs);
3402 #ifdef KMP_GOMP_COMPAT
3403 if (__kmp_atomic_mode == 2) {
3404 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3407 __kmp_release_atomic_lock(&__kmp_atomic_lock_1i, gtid);
3411 void __kmpc_atomic_2(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3412 void (*f)(
void *,
void *,
void *)) {
3414 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
3416 #elif KMP_ARCH_X86 || KMP_ARCH_X86_64
3419 !((kmp_uintptr_t)lhs & 0x1)
3422 kmp_int16 old_value, new_value;
3424 old_value = *(kmp_int16 *)lhs;
3425 (*f)(&new_value, &old_value, rhs);
3428 while (!KMP_COMPARE_AND_STORE_ACQ16(
3429 (kmp_int16 *)lhs, *(kmp_int16 *)&old_value, *(kmp_int16 *)&new_value)) {
3432 old_value = *(kmp_int16 *)lhs;
3433 (*f)(&new_value, &old_value, rhs);
3440 #ifdef KMP_GOMP_COMPAT
3441 if (__kmp_atomic_mode == 2) {
3442 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3445 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_2i, gtid);
3447 (*f)(lhs, lhs, rhs);
3449 #ifdef KMP_GOMP_COMPAT
3450 if (__kmp_atomic_mode == 2) {
3451 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3454 __kmp_release_atomic_lock(&__kmp_atomic_lock_2i, gtid);
3458 void __kmpc_atomic_4(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3459 void (*f)(
void *,
void *,
void *)) {
3460 KMP_DEBUG_ASSERT(__kmp_init_serial);
3465 #
if KMP_ARCH_X86 || KMP_ARCH_X86_64
3468 !((kmp_uintptr_t)lhs & 0x3)
3471 kmp_int32 old_value, new_value;
3473 old_value = *(kmp_int32 *)lhs;
3474 (*f)(&new_value, &old_value, rhs);
3477 while (!KMP_COMPARE_AND_STORE_ACQ32(
3478 (kmp_int32 *)lhs, *(kmp_int32 *)&old_value, *(kmp_int32 *)&new_value)) {
3481 old_value = *(kmp_int32 *)lhs;
3482 (*f)(&new_value, &old_value, rhs);
3490 #ifdef KMP_GOMP_COMPAT
3491 if (__kmp_atomic_mode == 2) {
3492 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3495 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_4i, gtid);
3497 (*f)(lhs, lhs, rhs);
3499 #ifdef KMP_GOMP_COMPAT
3500 if (__kmp_atomic_mode == 2) {
3501 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3504 __kmp_release_atomic_lock(&__kmp_atomic_lock_4i, gtid);
3508 void __kmpc_atomic_8(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3509 void (*f)(
void *,
void *,
void *)) {
3510 KMP_DEBUG_ASSERT(__kmp_init_serial);
3513 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
3515 #elif KMP_ARCH_X86 || KMP_ARCH_X86_64
3518 !((kmp_uintptr_t)lhs & 0x7)
3521 kmp_int64 old_value, new_value;
3523 old_value = *(kmp_int64 *)lhs;
3524 (*f)(&new_value, &old_value, rhs);
3526 while (!KMP_COMPARE_AND_STORE_ACQ64(
3527 (kmp_int64 *)lhs, *(kmp_int64 *)&old_value, *(kmp_int64 *)&new_value)) {
3530 old_value = *(kmp_int64 *)lhs;
3531 (*f)(&new_value, &old_value, rhs);
3539 #ifdef KMP_GOMP_COMPAT
3540 if (__kmp_atomic_mode == 2) {
3541 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3544 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_8i, gtid);
3546 (*f)(lhs, lhs, rhs);
3548 #ifdef KMP_GOMP_COMPAT
3549 if (__kmp_atomic_mode == 2) {
3550 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3553 __kmp_release_atomic_lock(&__kmp_atomic_lock_8i, gtid);
3557 void __kmpc_atomic_10(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3558 void (*f)(
void *,
void *,
void *)) {
3559 KMP_DEBUG_ASSERT(__kmp_init_serial);
3561 #ifdef KMP_GOMP_COMPAT
3562 if (__kmp_atomic_mode == 2) {
3563 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3566 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_10r, gtid);
3568 (*f)(lhs, lhs, rhs);
3570 #ifdef KMP_GOMP_COMPAT
3571 if (__kmp_atomic_mode == 2) {
3572 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3575 __kmp_release_atomic_lock(&__kmp_atomic_lock_10r, gtid);
3578 void __kmpc_atomic_16(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3579 void (*f)(
void *,
void *,
void *)) {
3580 KMP_DEBUG_ASSERT(__kmp_init_serial);
3582 #ifdef KMP_GOMP_COMPAT
3583 if (__kmp_atomic_mode == 2) {
3584 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3587 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_16c, gtid);
3589 (*f)(lhs, lhs, rhs);
3591 #ifdef KMP_GOMP_COMPAT
3592 if (__kmp_atomic_mode == 2) {
3593 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3596 __kmp_release_atomic_lock(&__kmp_atomic_lock_16c, gtid);
3599 void __kmpc_atomic_20(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3600 void (*f)(
void *,
void *,
void *)) {
3601 KMP_DEBUG_ASSERT(__kmp_init_serial);
3603 #ifdef KMP_GOMP_COMPAT
3604 if (__kmp_atomic_mode == 2) {
3605 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3608 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_20c, gtid);
3610 (*f)(lhs, lhs, rhs);
3612 #ifdef KMP_GOMP_COMPAT
3613 if (__kmp_atomic_mode == 2) {
3614 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3617 __kmp_release_atomic_lock(&__kmp_atomic_lock_20c, gtid);
3620 void __kmpc_atomic_32(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3621 void (*f)(
void *,
void *,
void *)) {
3622 KMP_DEBUG_ASSERT(__kmp_init_serial);
3624 #ifdef KMP_GOMP_COMPAT
3625 if (__kmp_atomic_mode == 2) {
3626 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3629 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_32c, gtid);
3631 (*f)(lhs, lhs, rhs);
3633 #ifdef KMP_GOMP_COMPAT
3634 if (__kmp_atomic_mode == 2) {
3635 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3638 __kmp_release_atomic_lock(&__kmp_atomic_lock_32c, gtid);
3644 void __kmpc_atomic_start(
void) {
3645 int gtid = __kmp_entry_gtid();
3646 KA_TRACE(20, (
"__kmpc_atomic_start: T#%d\n", gtid));
3647 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3650 void __kmpc_atomic_end(
void) {
3651 int gtid = __kmp_get_gtid();
3652 KA_TRACE(20, (
"__kmpc_atomic_end: T#%d\n", gtid));
3653 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);