[RFC PATCH v1 1/5] locking/atomic: Implement atomic_fetch_and_or

[RFC PATCH v1 1/5] locking/atomic: Implement atomic_fetch_and_or

[Rui Wang]

From: wangrui <wangrui@loongson.cn>

This patch introduce a new atomic primitive 'and_or', It may be have three
types of implemeations:

 * The generic implementation is based on arch_cmpxchg.
 * The hardware supports atomic 'and_or' of single instruction.
 * The hardware supports LL/SC style atomic operations:

   1:  ll  v1, mem
       and t1, v1, arg1
       or  t1, t1, arg2
       sc  t1, mem
       beq t1, 0, 1b

Now that all the architectures have implemented it.

Signed-by-off: Rui Wang <wangrui@loongson.cn>
Signed-by-off: hev <r@hev.cc>
---
 arch/alpha/include/asm/atomic.h           | 27 ++++++++++++
 arch/arc/include/asm/atomic.h             | 52 +++++++++++++++++++++++
 arch/arm/include/asm/atomic.h             | 44 +++++++++++++++++++
 arch/arm64/include/asm/atomic.h           | 16 +++++++
 arch/arm64/include/asm/atomic_ll_sc.h     | 33 ++++++++++++++
 arch/hexagon/include/asm/atomic.h         | 24 +++++++++++
 arch/ia64/include/asm/atomic.h            | 18 ++++++++
 arch/m68k/include/asm/atomic.h            | 36 ++++++++++++++++
 arch/mips/include/asm/atomic.h            | 41 ++++++++++++++++++
 arch/openrisc/include/asm/atomic.h        | 22 ++++++++++
 arch/parisc/include/asm/atomic.h          | 20 +++++++++
 arch/powerpc/include/asm/atomic.h         | 26 ++++++++++++
 arch/riscv/include/asm/atomic.h           | 25 +++++++++++
 arch/s390/include/asm/atomic.h            |  2 +
 arch/s390/include/asm/atomic_ops.h        | 25 +++++++++++
 arch/sparc/include/asm/atomic_32.h        |  2 +
 arch/sparc/lib/atomic32.c                 | 17 ++++++++
 arch/x86/include/asm/atomic.h             | 10 +++++
 arch/xtensa/include/asm/atomic.h          | 49 +++++++++++++++++++++
 include/asm-generic/atomic-instrumented.h | 28 ++++++++++++
 include/asm-generic/atomic.h              | 29 +++++++++++++
 include/linux/atomic-arch-fallback.h      | 42 ++++++++++++++++++
 22 files changed, 588 insertions(+)

diff --git a/arch/alpha/include/asm/atomic.h b/arch/alpha/include/asm/atomic.h
index f2861a43a61e..deb05ac292b8 100644
--- a/arch/alpha/include/asm/atomic.h
+++ b/arch/alpha/include/asm/atomic.h
@@ -91,6 +91,25 @@ static inline int arch_atomic_fetch_##op##_relaxed(int i, atomic_t *v)	\
 	return result;							\
 }
 
+#define ATOMIC_FETCH_OP2(op, asm_op1, asm_op2)					\
+static inline int arch_atomic_fetch_##op##_relaxed(int i, int j, atomic_t *v)	\
+{									\
+	long temp, result;						\
+	__asm__ __volatile__(						\
+	"1:	ldl_l %2,%1\n"						\
+	"	" #asm_op1 " %2,%3,%0\n"				\
+	"	" #asm_op2 " %0,%4,%0\n"				\
+	"	stl_c %0,%1\n"						\
+	"	beq %0,2f\n"						\
+	".subsection 2\n"						\
+	"2:	br 1b\n"						\
+	".previous"							\
+	:"=&r" (temp), "=m" (v->counter), "=&r" (result)		\
+	:"Ir" (i), "Ir" (j), "m" (v->counter) : "memory");		\
+	smp_mb();							\
+	return result;							\
+}
+
 #define ATOMIC64_OP(op, asm_op)						\
 static __inline__ void arch_atomic64_##op(s64 i, atomic64_t * v)	\
 {									\
@@ -182,10 +201,17 @@ ATOMIC_OPS(andnot, bic)
 ATOMIC_OPS(or, bis)
 ATOMIC_OPS(xor, xor)
 
+#undef ATOMIC_OPS
+#define ATOMIC_OPS(op, asm_op1, asm_op2)			\
+	ATOMIC_FETCH_OP2(op, asm_op1, asm_op2)			\
+
+ATOMIC_OPS(and_or, and, bis)
+
 #define arch_atomic_fetch_and_relaxed		arch_atomic_fetch_and_relaxed
 #define arch_atomic_fetch_andnot_relaxed	arch_atomic_fetch_andnot_relaxed
 #define arch_atomic_fetch_or_relaxed		arch_atomic_fetch_or_relaxed
 #define arch_atomic_fetch_xor_relaxed		arch_atomic_fetch_xor_relaxed
+#define arch_atomic_fetch_and_or_relaxed	arch_atomic_fetch_and_or_relaxed
 
 #define arch_atomic64_fetch_and_relaxed		arch_atomic64_fetch_and_relaxed
 #define arch_atomic64_fetch_andnot_relaxed	arch_atomic64_fetch_andnot_relaxed
@@ -197,6 +223,7 @@ ATOMIC_OPS(xor, xor)
 #undef ATOMIC64_OP_RETURN
 #undef ATOMIC64_OP
 #undef ATOMIC_FETCH_OP
+#undef ATOMIC_FETCH_OP2
 #undef ATOMIC_OP_RETURN
 #undef ATOMIC_OP
 
diff --git a/arch/arc/include/asm/atomic.h b/arch/arc/include/asm/atomic.h
index 7a36d79b5b2f..1aa9e0f396d7 100644
--- a/arch/arc/include/asm/atomic.h
+++ b/arch/arc/include/asm/atomic.h
@@ -89,6 +89,35 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t *v)		\
 	return orig;							\
 }
 
+#define ATOMIC_FETCH_OP2(op, c_op1, c_op2, asm_op1, asm_op2)		\
+static inline int arch_atomic_fetch_##op(int i, int j, atomic_t *v)	\
+{									\
+	unsigned int val, orig;						\
+									\
+	/*								\
+	 * Explicit full memory barrier needed before/after as		\
+	 * LLOCK/SCOND themselves don't provide any such semantics	\
+	 */								\
+	smp_mb();							\
+									\
+	__asm__ __volatile__(						\
+	"1:	llock   %[orig], [%[ctr]]		\n"		\
+	"	" #asm_op1 " %[val], %[orig], %[i]	\n"		\
+	"	" #asm_op2 " %[val], %[val], %[j]	\n"		\
+	"	scond   %[val], [%[ctr]]		\n"		\
+	"	bnz     1b				\n"		\
+	: [val]	"=&r"	(val),						\
+	  [orig] "=&r" (orig)						\
+	: [ctr]	"r"	(&v->counter),					\
+	  [i]	"ir"	(i),						\
+	  [j]	"ir"	(j),						\
+	: "cc");							\
+									\
+	smp_mb();							\
+									\
+	return orig;							\
+}
+
 #else	/* !CONFIG_ARC_HAS_LLSC */
 
 #ifndef CONFIG_SMP
@@ -170,6 +199,23 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t *v)		\
 	return orig;							\
 }
 
+#define ATOMIC_FETCH_OP2(op, c_op1, c_op2, asm_op1, asm_op2)		\
+static inline int arch_atomic_fetch_##op(int i, int j, atomic_t *v)	\
+{									\
+	unsigned long flags;						\
+	unsigned long orig;						\
+									\
+	/*								\
+	 * spin lock/unlock provides the needed smp_mb() before/after	\
+	 */								\
+	atomic_ops_lock(flags);						\
+	orig = v->counter;						\
+	v->counter = (orig c_op1 i) c_op2 j;				\
+	atomic_ops_unlock(flags);					\
+									\
+	return orig;							\
+}
+
 #endif /* !CONFIG_ARC_HAS_LLSC */
 
 #define ATOMIC_OPS(op, c_op, asm_op)					\
@@ -190,6 +236,12 @@ ATOMIC_OPS(andnot, &= ~, bic)
 ATOMIC_OPS(or, |=, or)
 ATOMIC_OPS(xor, ^=, xor)
 
+#undef ATOMIC_OPS
+#define ATOMIC_OPS(op, c_op1, c_op2, asm_op1, asm_op2)			\
+	ATOMIC_FETCH_OP2(op, c_op1, c_op2, asm_op1, asm_op2)
+
+ATOMIC_OPS(and_or, &, |, and, or)
+
 #define arch_atomic_andnot		arch_atomic_andnot
 #define arch_atomic_fetch_andnot	arch_atomic_fetch_andnot
 
diff --git a/arch/arm/include/asm/atomic.h b/arch/arm/include/asm/atomic.h
index db8512d9a918..faddbc183ced 100644
--- a/arch/arm/include/asm/atomic.h
+++ b/arch/arm/include/asm/atomic.h
@@ -93,6 +93,28 @@ static inline int arch_atomic_fetch_##op##_relaxed(int i, atomic_t *v)	\
 	return result;							\
 }
 
+#define ATOMIC_FETCH_OP2(op, c_op1, c_op2, asm_op1, asm_op2)			\
+static inline int arch_atomic_fetch_##op##_relaxed(int i, int j, atomic_t *v)	\
+{									\
+	unsigned long tmp;						\
+	int result, val;						\
+									\
+	prefetchw(&v->counter);						\
+									\
+	__asm__ __volatile__("@ atomic_fetch_" #op "\n"			\
+"1:	ldrex	%0, [%4]\n"						\
+"	" #asm_op1 "	%1, %0, %5\n"					\
+"	" #asm_op2 "	%1, %1, %6\n"					\
+"	strex	%2, %1, [%4]\n"						\
+"	teq	%2, #0\n"						\
+"	bne	1b"							\
+	: "=&r" (result), "=&r" (val), "=&r" (tmp), "+Qo" (v->counter)	\
+	: "r" (&v->counter), "Ir" (i), "Ir" (j)				\
+	: "cc");							\
+									\
+	return result;							\
+}
+
 #define arch_atomic_add_return_relaxed		arch_atomic_add_return_relaxed
 #define arch_atomic_sub_return_relaxed		arch_atomic_sub_return_relaxed
 #define arch_atomic_fetch_add_relaxed		arch_atomic_fetch_add_relaxed
@@ -102,6 +124,7 @@ static inline int arch_atomic_fetch_##op##_relaxed(int i, atomic_t *v)	\
 #define arch_atomic_fetch_andnot_relaxed	arch_atomic_fetch_andnot_relaxed
 #define arch_atomic_fetch_or_relaxed		arch_atomic_fetch_or_relaxed
 #define arch_atomic_fetch_xor_relaxed		arch_atomic_fetch_xor_relaxed
+#define arch_atomic_fetch_and_or_relaxed	arch_atomic_fetch_and_or_relaxed
 
 static inline int arch_atomic_cmpxchg_relaxed(atomic_t *ptr, int old, int new)
 {
@@ -197,6 +220,20 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t *v)		\
 	return val;							\
 }
 
+#define ATOMIC_FETCH_OP2(op, c_op1, c_op2, asm_op1, asm_op2)		\
+static inline int arch_atomic_fetch_##op(int i, int j, atomic_t *v)	\
+{									\
+	unsigned long flags;						\
+	int val;							\
+									\
+	raw_local_irq_save(flags);					\
+	val = v->counter;						\
+	v->counter = (val c_op1 i) c_op2 j;				\
+	raw_local_irq_restore(flags);					\
+									\
+	return val;							\
+}
+
 static inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new)
 {
 	int ret;
@@ -235,8 +272,15 @@ ATOMIC_OPS(andnot, &= ~, bic)
 ATOMIC_OPS(or,  |=, orr)
 ATOMIC_OPS(xor, ^=, eor)
 
+#undef ATOMIC_OPS
+#define ATOMIC_OPS(op, c_op1, c_op2, asm_op1, asm_op2)		\
+	ATOMIC_FETCH_OP2(op, c_op1, c_op2, asm_op1, asm_op2)
+
+ATOMIC_OPS(and_or, &, |, and, orr)
+
 #undef ATOMIC_OPS
 #undef ATOMIC_FETCH_OP
+#undef ATOMIC_FETCH_OP2
 #undef ATOMIC_OP_RETURN
 #undef ATOMIC_OP
 
diff --git a/arch/arm64/include/asm/atomic.h b/arch/arm64/include/asm/atomic.h
index c9979273d389..3f1cdd3e2ef9 100644
--- a/arch/arm64/include/asm/atomic.h
+++ b/arch/arm64/include/asm/atomic.h
@@ -43,6 +43,12 @@ static __always_inline int arch_##op##name(int i, atomic_t *v)		\
 	ATOMIC_FETCH_OP(_release, op)					\
 	ATOMIC_FETCH_OP(        , op)
 
+#define ATOMIC_FETCH_OP2(name, op)					\
+static __always_inline int arch_##op##name(int i, int j, atomic_t *v)	\
+{									\
+	return __ll_sc_##op##name(i, j, v);				\
+}
+
 ATOMIC_FETCH_OPS(atomic_fetch_andnot)
 ATOMIC_FETCH_OPS(atomic_fetch_or)
 ATOMIC_FETCH_OPS(atomic_fetch_xor)
@@ -52,7 +58,17 @@ ATOMIC_FETCH_OPS(atomic_fetch_sub)
 ATOMIC_FETCH_OPS(atomic_add_return)
 ATOMIC_FETCH_OPS(atomic_sub_return)
 
+#undef ATOMIC_FETCH_OPS
+#define ATOMIC_FETCH_OPS(op)						\
+	ATOMIC_FETCH_OP2(_relaxed, op)					\
+	ATOMIC_FETCH_OP2(_acquire, op)					\
+	ATOMIC_FETCH_OP2(_release, op)					\
+	ATOMIC_FETCH_OP2(        , op)
+
+ATOMIC_FETCH_OPS(atomic_fetch_and_or)
+
 #undef ATOMIC_FETCH_OP
+#undef ATOMIC_FETCH_OP2
 #undef ATOMIC_FETCH_OPS
 
 #define ATOMIC64_OP(op)							\
diff --git a/arch/arm64/include/asm/atomic_ll_sc.h b/arch/arm64/include/asm/atomic_ll_sc.h
index 13869b76b58c..90289c536ed6 100644
--- a/arch/arm64/include/asm/atomic_ll_sc.h
+++ b/arch/arm64/include/asm/atomic_ll_sc.h
@@ -97,6 +97,29 @@ __ll_sc_atomic_fetch_##op##name(int i, atomic_t *v)			\
 	return result;							\
 }
 
+#define ATOMIC_FETCH_OP2(name, mb, acq, rel, cl, op, asm_op1, asm_op2, cstr) \
+static inline int							\
+__ll_sc_atomic_fetch_##op##name(int i, int j, atomic_t *v)		\
+{									\
+	unsigned long tmp;						\
+	int val, result;						\
+									\
+	asm volatile("// atomic_fetch_" #op #name "\n"			\
+	__LL_SC_FALLBACK(						\
+"	prfm	pstl1strm, %3\n"					\
+"1:	ld" #acq "xr	%w0, %3\n"					\
+"	" #asm_op1 "	%w1, %w0, %w4\n"				\
+"	" #asm_op2 "	%w1, %w1, %w5\n"				\
+"	st" #rel "xr	%w2, %w1, %3\n"					\
+"	cbnz	%w2, 1b\n"						\
+"	" #mb )								\
+	: "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter)	\
+	: __stringify(cstr) "r" (i), __stringify(cstr) "r" (j)		\
+	: cl);								\
+									\
+	return result;							\
+}
+
 #define ATOMIC_OPS(...)							\
 	ATOMIC_OP(__VA_ARGS__)						\
 	ATOMIC_OP_RETURN(        , dmb ish,  , l, "memory", __VA_ARGS__)\
@@ -129,8 +152,18 @@ ATOMIC_OPS(xor, eor, K)
  */
 ATOMIC_OPS(andnot, bic, )
 
+#undef ATOMIC_OPS
+#define ATOMIC_OPS(...)							\
+	ATOMIC_FETCH_OP2 (        , dmb ish,  , l, "memory", __VA_ARGS__)\
+	ATOMIC_FETCH_OP2 (_relaxed,        ,  ,  ,         , __VA_ARGS__)\
+	ATOMIC_FETCH_OP2 (_acquire,        , a,  , "memory", __VA_ARGS__)\
+	ATOMIC_FETCH_OP2 (_release,        ,  , l, "memory", __VA_ARGS__)
+
+ATOMIC_OPS(and_or, and, orr, K)
+
 #undef ATOMIC_OPS
 #undef ATOMIC_FETCH_OP
+#undef ATOMIC_FETCH_OP2
 #undef ATOMIC_OP_RETURN
 #undef ATOMIC_OP
 
diff --git a/arch/hexagon/include/asm/atomic.h b/arch/hexagon/include/asm/atomic.h
index 6e94f8d04146..d944e210085a 100644
--- a/arch/hexagon/include/asm/atomic.h
+++ b/arch/hexagon/include/asm/atomic.h
@@ -130,6 +130,24 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t *v)		\
 	return output;							\
 }
 
+#define ATOMIC_FETCH_OP2(op1, op2)					\
+static inline int arch_atomic_fetch_##op1##_##op2(int i, int j, atomic_t *v)\
+{									\
+	int output, val;						\
+									\
+	__asm__ __volatile__ (						\
+		"1:	%0 = memw_locked(%2);\n"			\
+		"	%1 = "#op1 "(%0,%3);\n"				\
+		"	%1 = "#op2 "(%1,%4);\n"				\
+		"	memw_locked(%2,P3)=%1;\n"			\
+		"	if (!P3) jump 1b;\n"				\
+		: "=&r" (output), "=&r" (val)				\
+		: "r" (&v->counter), "r" (i), "r" (j)			\
+		: "memory", "p3"					\
+	);								\
+	return output;							\
+}
+
 #define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op) ATOMIC_FETCH_OP(op)
 
 ATOMIC_OPS(add)
@@ -142,8 +160,14 @@ ATOMIC_OPS(and)
 ATOMIC_OPS(or)
 ATOMIC_OPS(xor)
 
+#undef ATOMIC_OPS
+#define ATOMIC_OPS(op1, op2) ATOMIC_FETCH_OP2(op1, op2)
+
+ATOMIC_OPS(and, or)
+
 #undef ATOMIC_OPS
 #undef ATOMIC_FETCH_OP
+#undef ATOMIC_FETCH_OP2
 #undef ATOMIC_OP_RETURN
 #undef ATOMIC_OP
 
diff --git a/arch/ia64/include/asm/atomic.h b/arch/ia64/include/asm/atomic.h
index 266c429b9137..6190108dcd53 100644
--- a/arch/ia64/include/asm/atomic.h
+++ b/arch/ia64/include/asm/atomic.h
@@ -57,6 +57,21 @@ ia64_atomic_fetch_##op (int i, atomic_t *v)				\
 	return old;							\
 }
 
+#define ATOMIC_FETCH_OP2(op, c_op1, c_op2)				\
+static __inline__ int							\
+ia64_atomic_fetch_##op (int i, int j, atomic_t *v)			\
+{									\
+	__s32 old, new;							\
+	CMPXCHG_BUGCHECK_DECL						\
+									\
+	do {								\
+		CMPXCHG_BUGCHECK(v);					\
+		old = arch_atomic_read(v);				\
+		new = (old c_op1 i) c_op2 j;				\
+	} while (ia64_cmpxchg(acq, v, old, new, sizeof(atomic_t)) != old); \
+	return old;							\
+}
+
 #define ATOMIC_OPS(op, c_op)						\
 	ATOMIC_OP(op, c_op)						\
 	ATOMIC_FETCH_OP(op, c_op)
@@ -109,6 +124,7 @@ ATOMIC_OPS(sub, -)
 ATOMIC_FETCH_OP(and, &)
 ATOMIC_FETCH_OP(or, |)
 ATOMIC_FETCH_OP(xor, ^)
+ATOMIC_FETCH_OP2(and_or, &, |)
 
 #define arch_atomic_and(i,v)	(void)ia64_atomic_fetch_and(i,v)
 #define arch_atomic_or(i,v)	(void)ia64_atomic_fetch_or(i,v)
@@ -117,9 +133,11 @@ ATOMIC_FETCH_OP(xor, ^)
 #define arch_atomic_fetch_and(i,v)	ia64_atomic_fetch_and(i,v)
 #define arch_atomic_fetch_or(i,v)	ia64_atomic_fetch_or(i,v)
 #define arch_atomic_fetch_xor(i,v)	ia64_atomic_fetch_xor(i,v)
+#define arch_atomic_fetch_and_or(i,j,v)	ia64_atomic_fetch_and_or(i,j,v)
 
 #undef ATOMIC_OPS
 #undef ATOMIC_FETCH_OP
+#undef ATOMIC_FETCH_OP2
 #undef ATOMIC_OP
 
 #define ATOMIC64_OP(op, c_op)						\
diff --git a/arch/m68k/include/asm/atomic.h b/arch/m68k/include/asm/atomic.h
index 8637bf8a2f65..480ecb6534a3 100644
--- a/arch/m68k/include/asm/atomic.h
+++ b/arch/m68k/include/asm/atomic.h
@@ -67,6 +67,22 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t *v)		\
 	return tmp;							\
 }
 
+#define ATOMIC_FETCH_OP2(op, c_op1, c_op2, asm_op1, asm_op2)		\
+static inline int arch_atomic_fetch_##op(int i, int j, atomic_t *v)	\
+{									\
+	int t, tmp;							\
+									\
+	__asm__ __volatile__(						\
+			"1:	movel %2,%1\n"				\
+			"	" #asm_op1 "l %3,%1\n"			\
+			"	" #asm_op2 "l %4,%1\n"			\
+			"	casl %2,%1,%0\n"			\
+			"	jne 1b"					\
+			: "+m" (*v), "=&d" (t), "=&d" (tmp)		\
+			: "g" (i), "g" (j), "2" (arch_atomic_read(v)));	\
+	return tmp;							\
+}
+
 #else
 
 #define ATOMIC_OP_RETURN(op, c_op, asm_op)				\
@@ -96,6 +112,20 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t * v)		\
 	return t;							\
 }
 
+#define ATOMIC_FETCH_OP2(op, c_op1, c_op2, asm_op1, asm_op2)		\
+static inline int arch_atomic_fetch_##op(int i, int j, atomic_t * v)	\
+{									\
+	unsigned long flags;						\
+	int t;								\
+									\
+	local_irq_save(flags);						\
+	t = v->counter;							\
+	v->counter = (t c_op1 i) c_op2 j;				\
+	local_irq_restore(flags);					\
+									\
+	return t;							\
+}
+
 #endif /* CONFIG_RMW_INSNS */
 
 #define ATOMIC_OPS(op, c_op, asm_op)					\
@@ -115,6 +145,12 @@ ATOMIC_OPS(and, &=, and)
 ATOMIC_OPS(or, |=, or)
 ATOMIC_OPS(xor, ^=, eor)
 
+#undef ATOMIC_OPS
+#define ATOMIC_OPS(op, c_op1, c_op2, asm_op1, asm_op2)			\
+	ATOMIC_FETCH_OP2(op, c_op1, c_op2, asm_op1, asm_op2)
+
+ATOMIC_OPS(and_or, &, |, and, or)
+
 #undef ATOMIC_OPS
 #undef ATOMIC_FETCH_OP
 #undef ATOMIC_OP_RETURN
diff --git a/arch/mips/include/asm/atomic.h b/arch/mips/include/asm/atomic.h
index 95e1f7f3597f..84319b1ab9b6 100644
--- a/arch/mips/include/asm/atomic.h
+++ b/arch/mips/include/asm/atomic.h
@@ -147,6 +147,39 @@ arch_##pfx##_fetch_##op##_relaxed(type i, pfx##_t * v)			\
 	return result;							\
 }
 
+#define ATOMIC_FETCH_OP2(pfx, op, type, c_op1, c_op2, asm_op1, asm_op2, ll, sc)\
+static __inline__ type							\
+arch_##pfx##_fetch_##op##_relaxed(type i, type j, pfx##_t * v)		\
+{									\
+	int temp, result;						\
+									\
+	if (!kernel_uses_llsc) {					\
+		unsigned long flags;					\
+									\
+		raw_local_irq_save(flags);				\
+		result = v->counter;					\
+		v->counter = (result c_op1 i) c_op2 j;			\
+		raw_local_irq_restore(flags);				\
+		return result;						\
+	}								\
+									\
+	__asm__ __volatile__(						\
+	"	.set	push					\n"	\
+	"	.set	" MIPS_ISA_LEVEL "			\n"	\
+	"	" __SYNC(full, loongson3_war) "			\n"	\
+	"1:	" #ll "	%0, %2		# " #pfx "_fetch_" #op "\n"	\
+	"	" #asm_op1 " %1, %0, %3				\n"	\
+	"	" #asm_op2 " %1, %1, %4				\n"	\
+	"	" #sc "	%1, %2					\n"	\
+	"\t" __SC_BEQZ "%1, 1b					\n"	\
+	"	.set	pop					\n"	\
+	: "=&r" (result), "=&r" (temp),					\
+	  "+" GCC_OFF_SMALL_ASM() (v->counter)				\
+	: "Ir" (i), "Ir" (j) : __LLSC_CLOBBER);				\
+									\
+	return result;							\
+}
+
 #undef ATOMIC_OPS
 #define ATOMIC_OPS(pfx, op, type, c_op, asm_op, ll, sc)			\
 	ATOMIC_OP(pfx, op, type, c_op, asm_op, ll, sc)			\
@@ -179,9 +212,16 @@ ATOMIC_OPS(atomic, and, int, &=, and, ll, sc)
 ATOMIC_OPS(atomic, or, int, |=, or, ll, sc)
 ATOMIC_OPS(atomic, xor, int, ^=, xor, ll, sc)
 
+#undef ATOMIC_OPS
+#define ATOMIC_OPS(pfx, op, type, c_op1, c_op2, asm_op1, asm_op2, ll, sc)	\
+	ATOMIC_FETCH_OP2(pfx, op, type, c_op1, c_op2, asm_op1, asm_op2, ll, sc)
+
+ATOMIC_OPS(atomic, and_or, int, &, |, and, or, ll, sc)
+
 #define arch_atomic_fetch_and_relaxed	arch_atomic_fetch_and_relaxed
 #define arch_atomic_fetch_or_relaxed	arch_atomic_fetch_or_relaxed
 #define arch_atomic_fetch_xor_relaxed	arch_atomic_fetch_xor_relaxed
+#define arch_atomic_fetch_and_or_relaxed	arch_atomic_fetch_and_or_relaxed
 
 #ifdef CONFIG_64BIT
 ATOMIC_OPS(atomic64, and, s64, &=, and, lld, scd)
@@ -194,6 +234,7 @@ ATOMIC_OPS(atomic64, xor, s64, ^=, xor, lld, scd)
 
 #undef ATOMIC_OPS
 #undef ATOMIC_FETCH_OP
+#undef ATOMIC_FETCH_OP2
 #undef ATOMIC_OP_RETURN
 #undef ATOMIC_OP
 
diff --git a/arch/openrisc/include/asm/atomic.h b/arch/openrisc/include/asm/atomic.h
index 326167e4783a..04598ef16977 100644
--- a/arch/openrisc/include/asm/atomic.h
+++ b/arch/openrisc/include/asm/atomic.h
@@ -66,6 +66,25 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t *v)		\
 	return old;							\
 }
 
+#define ATOMIC_FETCH_OP2(op1, op2)					\
+static inline int arch_atomic_fetch_##op1##_##op2(int i, int j, atomic_t *v)	\
+{									\
+	int tmp, old;							\
+									\
+	__asm__ __volatile__(						\
+		"1:	l.lwa	%0,0(%2)	\n"			\
+		"	l." #op1 " %1,%0,%3	\n"			\
+		"	l." #op2 " %1,%1,%4	\n"			\
+		"	l.swa	0(%2),%1	\n"			\
+		"	l.bnf	1b		\n"			\
+		"	 l.nop			\n"			\
+		: "=&r"(old), "=&r"(tmp)				\
+		: "r"(&v->counter), "r"(i), "r"(j)			\
+		: "cc", "memory");					\
+									\
+	return old;							\
+}
+
 ATOMIC_OP_RETURN(add)
 ATOMIC_OP_RETURN(sub)
 
@@ -74,6 +93,7 @@ ATOMIC_FETCH_OP(sub)
 ATOMIC_FETCH_OP(and)
 ATOMIC_FETCH_OP(or)
 ATOMIC_FETCH_OP(xor)
+ATOMIC_FETCH_OP2(and, or)
 
 ATOMIC_OP(add)
 ATOMIC_OP(sub)
@@ -82,6 +102,7 @@ ATOMIC_OP(or)
 ATOMIC_OP(xor)
 
 #undef ATOMIC_FETCH_OP
+#undef ATOMIC_FETCH_OP2
 #undef ATOMIC_OP_RETURN
 #undef ATOMIC_OP
 
@@ -92,6 +113,7 @@ ATOMIC_OP(xor)
 #define arch_atomic_fetch_and	arch_atomic_fetch_and
 #define arch_atomic_fetch_or	arch_atomic_fetch_or
 #define arch_atomic_fetch_xor	arch_atomic_fetch_xor
+#define arch_atomic_fetch_and_or	arch_atomic_fetch_and_or
 #define arch_atomic_add		arch_atomic_add
 #define arch_atomic_sub		arch_atomic_sub
 #define arch_atomic_and		arch_atomic_and
diff --git a/arch/parisc/include/asm/atomic.h b/arch/parisc/include/asm/atomic.h
index dd5a299ada69..59b9685ed2b1 100644
--- a/arch/parisc/include/asm/atomic.h
+++ b/arch/parisc/include/asm/atomic.h
@@ -114,6 +114,20 @@ static __inline__ int arch_atomic_fetch_##op(int i, atomic_t *v)	\
 	return ret;							\
 }
 
+#define ATOMIC_FETCH_OP2(op, c_op1, c_op2)				\
+static __inline__ int arch_atomic_fetch_##op(int i, int j, atomic_t *v)\
+{									\
+	unsigned long flags;						\
+	int ret;							\
+									\
+	_atomic_spin_lock_irqsave(v, flags);				\
+	ret = v->counter;						\
+	v->counter = (ret c_op1 i) c_op2 j;				\
+	_atomic_spin_unlock_irqrestore(v, flags);			\
+									\
+	return ret;							\
+}
+
 #define ATOMIC_OPS(op, c_op)						\
 	ATOMIC_OP(op, c_op)						\
 	ATOMIC_OP_RETURN(op, c_op)					\
@@ -131,6 +145,12 @@ ATOMIC_OPS(and, &=)
 ATOMIC_OPS(or, |=)
 ATOMIC_OPS(xor, ^=)
 
+#undef ATOMIC_OPS
+#define ATOMIC_OPS(op, c_op1, c_op2)					\
+	ATOMIC_FETCH_OP2(op, c_op1, c_op2)
+
+ATOMIC_OPS(and_or, &, |)
+
 #undef ATOMIC_OPS
 #undef ATOMIC_FETCH_OP
 #undef ATOMIC_OP_RETURN
diff --git a/arch/powerpc/include/asm/atomic.h b/arch/powerpc/include/asm/atomic.h
index a1732a79e92a..c2e966ab4b81 100644
--- a/arch/powerpc/include/asm/atomic.h
+++ b/arch/powerpc/include/asm/atomic.h
@@ -86,6 +86,24 @@ static inline int arch_atomic_fetch_##op##_relaxed(int a, atomic_t *v)	\
 	return res;							\
 }
 
+#define ATOMIC_FETCH_OP2_RELAXED(op, asm_op1, asm_op2)			\
+static inline int arch_atomic_fetch_##op##_relaxed(int a, int b, atomic_t *v)\
+{									\
+	int res, t;							\
+									\
+	__asm__ __volatile__(						\
+"1:	lwarx	%0,0,%5		# atomic_fetch_" #op "_relaxed\n"	\
+	#asm_op1 " %1,%3,%0\n"						\
+	#asm_op2 " %1,%4,%1\n"						\
+"	stwcx.	%1,0,%5\n"						\
+"	bne-	1b\n"							\
+	: "=&r" (res), "=&r" (t), "+m" (v->counter)			\
+	: "r" (a), "r" (b), "r" (&v->counter)				\
+	: "cc");							\
+									\
+	return res;							\
+}
+
 #define ATOMIC_OPS(op, asm_op)						\
 	ATOMIC_OP(op, asm_op)						\
 	ATOMIC_OP_RETURN_RELAXED(op, asm_op)				\
@@ -109,12 +127,20 @@ ATOMIC_OPS(and, and)
 ATOMIC_OPS(or, or)
 ATOMIC_OPS(xor, xor)
 
+#undef ATOMIC_OPS
+#define ATOMIC_OPS(op, asm_op1, asm_op2)			\
+	ATOMIC_FETCH_OP2_RELAXED(op, asm_op1, asm_op2)
+
+ATOMIC_OPS(and_or, and, or)
+
 #define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed
 #define arch_atomic_fetch_or_relaxed  arch_atomic_fetch_or_relaxed
 #define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed
+#define arch_atomic_fetch_and_or_relaxed arch_atomic_fetch_and_or_relaxed
 
 #undef ATOMIC_OPS
 #undef ATOMIC_FETCH_OP_RELAXED
+#undef ATOMIC_FETCH_OP2_RELAXED
 #undef ATOMIC_OP_RETURN_RELAXED
 #undef ATOMIC_OP
 
diff --git a/arch/riscv/include/asm/atomic.h b/arch/riscv/include/asm/atomic.h
index ac9bdf4fc404..572ca0ae2e76 100644
--- a/arch/riscv/include/asm/atomic.h
+++ b/arch/riscv/include/asm/atomic.h
@@ -110,6 +110,24 @@ c_type arch_atomic##prefix##_fetch_##op(c_type i, atomic##prefix##_t *v)	\
 	return ret;							\
 }
 
+#define ATOMIC_FETCH_OP2(op, asm_op1, asm_op2, asm_type, c_type, prefix)	\
+static __always_inline								\
+c_type arch_atomic##prefix##_fetch_##op##_relaxed(c_type i,	c_type j,	\
+					     atomic##prefix##_t *v)	\
+{									\
+	register c_type ret, tmp;					\
+	__asm__ __volatile__ (						\
+		"0:	lr." #asm_type " %0,  %2\n"			\
+		"	" #asm_op1 "%1, %0, %3\n"			\
+		"	" #asm_op2 "%1, %1, %4\n"			\
+		"	sc." #asm_type " %1, %1, %2\n"			\
+		"	bnez     %1, 0b\n"				\
+		: "=r" (ret), "=&r" (tmp), "+A" (v->counter)		\
+		: "r" (i), "r" (j)					\
+		: "memory");						\
+	return ret;							\
+}
+
 #define ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_type, c_type, prefix)	\
 static __always_inline							\
 c_type arch_atomic##prefix##_##op##_return_relaxed(c_type i,		\
@@ -175,9 +193,15 @@ ATOMIC_OPS(and, and, i)
 ATOMIC_OPS( or,  or, i)
 ATOMIC_OPS(xor, xor, i)
 
+#define ATOMIC_OPS(op, asm_op1, asm_op2, I)			\
+        ATOMIC_FETCH_OP2(op, asm_op1, asm_op2, I, w, int,)
+
+ATOMIC_OPS(and_or, and, or, w)
+
 #define arch_atomic_fetch_and_relaxed	arch_atomic_fetch_and_relaxed
 #define arch_atomic_fetch_or_relaxed	arch_atomic_fetch_or_relaxed
 #define arch_atomic_fetch_xor_relaxed	arch_atomic_fetch_xor_relaxed
+#define arch_atomic_fetch_and_or_relaxed	arch_atomic_fetch_and_or_relaxed
 #define arch_atomic_fetch_and		arch_atomic_fetch_and
 #define arch_atomic_fetch_or		arch_atomic_fetch_or
 #define arch_atomic_fetch_xor		arch_atomic_fetch_xor
@@ -194,6 +218,7 @@ ATOMIC_OPS(xor, xor, i)
 #undef ATOMIC_OPS
 
 #undef ATOMIC_FETCH_OP
+#undef ATOMIC_FETCH_OP2
 #undef ATOMIC_OP_RETURN
 
 /* This is required to provide a full barrier on success. */
diff --git a/arch/s390/include/asm/atomic.h b/arch/s390/include/asm/atomic.h
index 7138d189cc42..abebd658c1fa 100644
--- a/arch/s390/include/asm/atomic.h
+++ b/arch/s390/include/asm/atomic.h
@@ -62,6 +62,7 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t *v)		\
 ATOMIC_OPS(and)
 ATOMIC_OPS(or)
 ATOMIC_OPS(xor)
+ATOMIC_OPS(and_or)
 
 #undef ATOMIC_OPS
 
@@ -71,6 +72,7 @@ ATOMIC_OPS(xor)
 #define arch_atomic_fetch_and		arch_atomic_fetch_and
 #define arch_atomic_fetch_or		arch_atomic_fetch_or
 #define arch_atomic_fetch_xor		arch_atomic_fetch_xor
+#define arch_atomic_fetch_and_or	arch_atomic_fetch_and_or
 
 #define arch_atomic_xchg(v, new)	(arch_xchg(&((v)->counter), new))
 
diff --git a/arch/s390/include/asm/atomic_ops.h b/arch/s390/include/asm/atomic_ops.h
index 50510e08b893..d396f2e2eb9a 100644
--- a/arch/s390/include/asm/atomic_ops.h
+++ b/arch/s390/include/asm/atomic_ops.h
@@ -154,6 +154,31 @@ __ATOMIC64_OPS(__atomic64_xor, "xgr")
 
 #endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */
 
+#define __ATOMIC_OP2(op_name, op1, op2)				\
+static inline int op_name(int i, int j, int *ptr)			\
+{									\
+	int old, new;							\
+									\
+	asm volatile(							\
+		"0:	lr	%[new],%[old]\n"			\
+		op1 "	%[new],%[i]\n"					\
+		op2 "	%[new],%[j]\n"					\
+		"	cs	%[old],%[new],%[ptr]\n"			\
+		"	jl	0b"					\
+		: [old] "=d" (old), [new] "=&d" (new), [ptr] "+Q" (*ptr)\
+		: [i] "d" (i), [j] "d" (j), "0" (*ptr) : "cc", "memory");\
+	return old;							\
+}
+
+#define __ATOMIC_OPS(op_name, op1_string, op2_string)			\
+	__ATOMIC_OP2(op_name, op1_string, op2_string)			\
+	__ATOMIC_OP2(op_name##_barrier, op1_string, op2_string)
+
+__ATOMIC_OPS(__atomic_and_or, "ngr", "ogr")
+
+#undef __ATOMIC_OPS
+#undef __ATOMIC_OP2
+
 static inline int __atomic_cmpxchg(int *ptr, int old, int new)
 {
 	asm volatile(
diff --git a/arch/sparc/include/asm/atomic_32.h b/arch/sparc/include/asm/atomic_32.h
index d775daa83d12..d062b20eb64c 100644
--- a/arch/sparc/include/asm/atomic_32.h
+++ b/arch/sparc/include/asm/atomic_32.h
@@ -23,6 +23,7 @@ int arch_atomic_fetch_add(int, atomic_t *);
 int arch_atomic_fetch_and(int, atomic_t *);
 int arch_atomic_fetch_or(int, atomic_t *);
 int arch_atomic_fetch_xor(int, atomic_t *);
+int arch_atomic_fetch_and_or(int, int, atomic_t *);
 int arch_atomic_cmpxchg(atomic_t *, int, int);
 int arch_atomic_xchg(atomic_t *, int);
 int arch_atomic_fetch_add_unless(atomic_t *, int, int);
@@ -40,6 +41,7 @@ void arch_atomic_set(atomic_t *, int);
 #define arch_atomic_and(i, v)	((void)arch_atomic_fetch_and((i), (v)))
 #define arch_atomic_or(i, v)	((void)arch_atomic_fetch_or((i), (v)))
 #define arch_atomic_xor(i, v)	((void)arch_atomic_fetch_xor((i), (v)))
+#define arch_atomic_and_or(i, j, v)	((void)arch_atomic_fetch_and_or((i), (j), (v)))
 
 #define arch_atomic_sub_return(i, v)	(arch_atomic_add_return(-(int)(i), (v)))
 #define arch_atomic_fetch_sub(i, v)	(arch_atomic_fetch_add (-(int)(i), (v)))
diff --git a/arch/sparc/lib/atomic32.c b/arch/sparc/lib/atomic32.c
index 8b81d0f00c97..aefb6d91985e 100644
--- a/arch/sparc/lib/atomic32.c
+++ b/arch/sparc/lib/atomic32.c
@@ -43,6 +43,21 @@ int arch_atomic_fetch_##op(int i, atomic_t *v)				\
 }									\
 EXPORT_SYMBOL(arch_atomic_fetch_##op);
 
+#define ATOMIC_FETCH_OP2(op, c_op1, c_op2)				\
+int arch_atomic_fetch_##op(int i, int j, atomic_t *v)			\
+{									\
+	int ret;							\
+	unsigned long flags;						\
+	spin_lock_irqsave(ATOMIC_HASH(v), flags);			\
+									\
+	ret = v->counter;						\
+	v->counter = (ret c_op1 i) c_op2 j;				\
+									\
+	spin_unlock_irqrestore(ATOMIC_HASH(v), flags);			\
+	return ret;							\
+}									\
+EXPORT_SYMBOL(arch_atomic_fetch_##op);
+
 #define ATOMIC_OP_RETURN(op, c_op)					\
 int arch_atomic_##op##_return(int i, atomic_t *v)			\
 {									\
@@ -63,8 +78,10 @@ ATOMIC_FETCH_OP(add, +=)
 ATOMIC_FETCH_OP(and, &=)
 ATOMIC_FETCH_OP(or, |=)
 ATOMIC_FETCH_OP(xor, ^=)
+ATOMIC_FETCH_OP2(and_or, &, |)
 
 #undef ATOMIC_FETCH_OP
+#undef ATOMIC_FETCH_OP2
 #undef ATOMIC_OP_RETURN
 
 int arch_atomic_xchg(atomic_t *v, int new)
diff --git a/arch/x86/include/asm/atomic.h b/arch/x86/include/asm/atomic.h
index 5e754e895767..145dce45d02a 100644
--- a/arch/x86/include/asm/atomic.h
+++ b/arch/x86/include/asm/atomic.h
@@ -263,6 +263,16 @@ static __always_inline int arch_atomic_fetch_xor(int i, atomic_t *v)
 }
 #define arch_atomic_fetch_xor arch_atomic_fetch_xor
 
+static __always_inline int arch_atomic_fetch_and_or(int i, int j, atomic_t *v)
+{
+	int val = arch_atomic_read(v);
+
+	do { } while (!arch_atomic_try_cmpxchg(v, &val, (val & i) | j));
+
+	return val;
+}
+#define arch_atomic_fetch_and_or arch_atomic_fetch_and_or
+
 #ifdef CONFIG_X86_32
 # include <asm/atomic64_32.h>
 #else
diff --git a/arch/xtensa/include/asm/atomic.h b/arch/xtensa/include/asm/atomic.h
index 4361fe4247e3..6b043cf74df2 100644
--- a/arch/xtensa/include/asm/atomic.h
+++ b/arch/xtensa/include/asm/atomic.h
@@ -177,6 +177,28 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t * v)		\
 	return result;							\
 }
 
+#define ATOMIC_FETCH_OP2(op1, op2)					\
+static inline int arch_atomic_fetch_##op1##_##op2(int i, int j, atomic_t * v)\
+{									\
+	unsigned long tmp;						\
+	int result;							\
+									\
+	__asm__ __volatile__(						\
+			"1:     l32i    %[tmp], %[mem]\n"		\
+			"       wsr     %[tmp], scompare1\n"		\
+			"       " #op1 " %[result], %[tmp], %[i]\n"	\
+			"       " #op2 " %[result], %[result], %[j]\n"	\
+			"       s32c1i  %[result], %[mem]\n"		\
+			"       bne     %[result], %[tmp], 1b\n"	\
+			: [result] "=&a" (result), [tmp] "=&a" (tmp),	\
+			  [mem] "+m" (*v)				\
+			: [i] "a" (i), [j] "a" (j)			\
+			: "memory"					\
+			);						\
+									\
+	return result;							\
+}
+
 #else /* XCHAL_HAVE_S32C1I */
 
 #define ATOMIC_OP(op)							\
@@ -238,6 +260,28 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t * v)		\
 	return vval;							\
 }
 
+#define ATOMIC_FETCH_OP2(op1, op2)					\
+static inline int arch_atomic_fetch_##op1##_##op2(int i, int j, atomic_t * v)\
+{									\
+	unsigned int tmp, vval;						\
+									\
+	__asm__ __volatile__(						\
+			"       rsil    a15,"__stringify(TOPLEVEL)"\n"	\
+			"       l32i    %[result], %[mem]\n"		\
+			"       " #op1 " %[tmp], %[result], %[i]\n"	\
+			"       " #op2 " %[tmp], %[tmp], %[j]\n"	\
+			"       s32i    %[tmp], %[mem]\n"		\
+			"       wsr     a15, ps\n"			\
+			"       rsync\n"				\
+			: [result] "=&a" (vval), [tmp] "=&a" (tmp),	\
+			  [mem] "+m" (*v)				\
+			: [i] "a" (i), [j] "a" (j)			\
+			: "a15", "memory"				\
+			);						\
+									\
+	return vval;							\
+}
+
 #endif /* XCHAL_HAVE_S32C1I */
 
 #define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op) ATOMIC_OP_RETURN(op)
@@ -252,6 +296,11 @@ ATOMIC_OPS(and)
 ATOMIC_OPS(or)
 ATOMIC_OPS(xor)
 
+#undef ATOMIC_OPS
+#define ATOMIC_OPS(op1, op2) ATOMIC_FETCH_OP2(op1, op2)
+
+ATOMIC_OPS(and, or)
+
 #undef ATOMIC_OPS
 #undef ATOMIC_FETCH_OP
 #undef ATOMIC_OP_RETURN
diff --git a/include/asm-generic/atomic-instrumented.h b/include/asm-generic/atomic-instrumented.h
index bc45af52c93b..231a8386ac80 100644
--- a/include/asm-generic/atomic-instrumented.h
+++ b/include/asm-generic/atomic-instrumented.h
@@ -441,6 +441,34 @@ atomic_fetch_xor_relaxed(int i, atomic_t *v)
 	return arch_atomic_fetch_xor_relaxed(i, v);
 }
 
+static __always_inline int
+atomic_fetch_and_or(int i, int j, atomic_t *v)
+{
+	instrument_atomic_read_write(v, sizeof(*v));
+	return arch_atomic_fetch_and_or(i, j, v);
+}
+
+static __always_inline int
+atomic_fetch_and_or_acquire(int i, int j, atomic_t *v)
+{
+	instrument_atomic_read_write(v, sizeof(*v));
+	return arch_atomic_fetch_and_or_acquire(i, j, v);
+}
+
+static __always_inline int
+atomic_fetch_and_or_release(int i, int j, atomic_t *v)
+{
+	instrument_atomic_read_write(v, sizeof(*v));
+	return arch_atomic_fetch_and_or_release(i, j, v);
+}
+
+static __always_inline int
+atomic_fetch_and_or_relaxed(int i, int j, atomic_t *v)
+{
+	instrument_atomic_read_write(v, sizeof(*v));
+	return arch_atomic_fetch_and_or_relaxed(i, j, v);
+}
+
 static __always_inline int
 atomic_xchg(atomic_t *v, int i)
 {
diff --git a/include/asm-generic/atomic.h b/include/asm-generic/atomic.h
index 04b8be9f1a77..474e8cd8e58d 100644
--- a/include/asm-generic/atomic.h
+++ b/include/asm-generic/atomic.h
@@ -50,6 +50,18 @@ static inline int generic_atomic_fetch_##op(int i, atomic_t *v)		\
 	return c;							\
 }
 
+#define ATOMIC_FETCH_OP2(op, c_op1, c_op2)				\
+static inline int generic_atomic_fetch_##op(int i, int j, atomic_t *v)	\
+{									\
+	int c, old;							\
+									\
+	c = v->counter;							\
+	while ((old = arch_cmpxchg(&v->counter, c, (c c_op1 i) c_op2 j)) != c)	\
+		c = old;						\
+									\
+	return c;							\
+}
+
 #else
 
 #include <linux/irqflags.h>
@@ -91,6 +103,20 @@ static inline int generic_atomic_fetch_##op(int i, atomic_t *v)		\
 	return ret;							\
 }
 
+#define ATOMIC_FETCH_OP2(op, c_op1, c_op2)				\
+static inline int generic_atomic_fetch_##op(int i, int j, atomic_t *v)	\
+{									\
+	unsigned long flags;						\
+	int ret;							\
+									\
+	raw_local_irq_save(flags);					\
+	ret = v->counter;						\
+	v->counter = (v->counter c_op1 i) c_op2 j;			\
+	raw_local_irq_restore(flags);					\
+									\
+	return ret;							\
+}
+
 #endif /* CONFIG_SMP */
 
 ATOMIC_OP_RETURN(add, +)
@@ -101,6 +127,7 @@ ATOMIC_FETCH_OP(sub, -)
 ATOMIC_FETCH_OP(and, &)
 ATOMIC_FETCH_OP(or, |)
 ATOMIC_FETCH_OP(xor, ^)
+ATOMIC_FETCH_OP2(and_or, &, |)
 
 ATOMIC_OP(add, +)
 ATOMIC_OP(sub, -)
@@ -109,6 +136,7 @@ ATOMIC_OP(or, |)
 ATOMIC_OP(xor, ^)
 
 #undef ATOMIC_FETCH_OP
+#undef ATOMIC_FETCH_OP2
 #undef ATOMIC_OP_RETURN
 #undef ATOMIC_OP
 
@@ -120,6 +148,7 @@ ATOMIC_OP(xor, ^)
 #define arch_atomic_fetch_and			generic_atomic_fetch_and
 #define arch_atomic_fetch_or			generic_atomic_fetch_or
 #define arch_atomic_fetch_xor			generic_atomic_fetch_xor
+#define arch_atomic_fetch_and_or		generic_atomic_fetch_and_or
 
 #define arch_atomic_add				generic_atomic_add
 #define arch_atomic_sub				generic_atomic_sub
diff --git a/include/linux/atomic-arch-fallback.h b/include/linux/atomic-arch-fallback.h
index a3dba31df01e..92043a8d5b79 100644
--- a/include/linux/atomic-arch-fallback.h
+++ b/include/linux/atomic-arch-fallback.h
@@ -891,6 +891,48 @@ arch_atomic_fetch_xor(int i, atomic_t *v)
 
 #endif /* arch_atomic_fetch_xor_relaxed */
 
+#ifndef arch_atomic_fetch_and_or_relaxed
+#define arch_atomic_fetch_and_or_acquire arch_atomic_fetch_and_or
+#define arch_atomic_fetch_and_or_release arch_atomic_fetch_and_or
+#define arch_atomic_fetch_and_or_relaxed arch_atomic_fetch_and_or
+#else /* arch_atomic_fetch_and_or_relaxed */
+
+#ifndef arch_atomic_fetch_and_or_acquire
+static __always_inline int
+arch_atomic_fetch_and_or_acquire(int i, int j, atomic_t *v)
+{
+	int ret = arch_atomic_fetch_and_or_relaxed(i, j, v);
+	__atomic_acquire_fence();
+	return ret;
+}
+#define arch_atomic_fetch_and_or_acquire arch_atomic_fetch_and_or_acquire
+#endif
+
+#ifndef arch_atomic_fetch_and_or_release
+static __always_inline int
+arch_atomic_fetch_and_or_release(int i, int j, atomic_t *v)
+{
+	__atomic_release_fence();
+	return arch_atomic_fetch_and_or_relaxed(i, j, v);
+}
+#define arch_atomic_fetch_and_or_release arch_atomic_fetch_and_or_release
+#endif
+
+#ifndef arch_atomic_fetch_and_or
+static __always_inline int
+arch_atomic_fetch_and_or(int i, int j, atomic_t *v)
+{
+	int ret;
+	__atomic_pre_full_fence();
+	ret = arch_atomic_fetch_and_or_relaxed(i, j, v);
+	__atomic_post_full_fence();
+	return ret;
+}
+#define arch_atomic_fetch_and_or arch_atomic_fetch_and_or
+#endif
+
+#endif /* arch_atomic_fetch_and_or_relaxed */
+
 #ifndef arch_atomic_xchg_relaxed
 #define arch_atomic_xchg_acquire arch_atomic_xchg
 #define arch_atomic_xchg_release arch_atomic_xchg
-- 
2.32.0

Re: [RFC PATCH v1 1/5] locking/atomic: Implement atomic_fetch_and_or

[Boqun Feng]

Hi,

Thanks for the patchset. Seems that your git send-email command doesn't
add the "In-Reply-to" tag for patch #2 to #5, so they are not threaded
to patch #1. Not a big deal, but archives or email clients use that
information to organize emails. You may want to check the command. Also,
note that you can always use "--dry-run" option to preview the headers
of your patchset ("--dry-run" won't do the actual send).

On Wed, Jul 28, 2021 at 07:48:22PM +0800, Rui Wang wrote:
> From: wangrui <wangrui@loongson.cn>
> 
> This patch introduce a new atomic primitive 'and_or', It may be have three
> types of implemeations:
> 
>  * The generic implementation is based on arch_cmpxchg.
>  * The hardware supports atomic 'and_or' of single instruction.
>  * The hardware supports LL/SC style atomic operations:
> 
>    1:  ll  v1, mem
>        and t1, v1, arg1
>        or  t1, t1, arg2
>        sc  t1, mem
>        beq t1, 0, 1b
> 
> Now that all the architectures have implemented it.
> 
> Signed-by-off: Rui Wang <wangrui@loongson.cn>
> Signed-by-off: hev <r@hev.cc>

First, this should be "Signed-off-by" ;-) Second, is the second
"Signed-off-by" a mistake?

I will look into this for a review, in the meanwhile, but please add
some tests in lib/atomic64_test.c, not only it will do the test at
runtime, also it will generate asm code which helps people to review.

Regards,
Boqun

> ---
>  arch/alpha/include/asm/atomic.h           | 27 ++++++++++++
>  arch/arc/include/asm/atomic.h             | 52 +++++++++++++++++++++++
>  arch/arm/include/asm/atomic.h             | 44 +++++++++++++++++++
>  arch/arm64/include/asm/atomic.h           | 16 +++++++
>  arch/arm64/include/asm/atomic_ll_sc.h     | 33 ++++++++++++++
>  arch/hexagon/include/asm/atomic.h         | 24 +++++++++++
>  arch/ia64/include/asm/atomic.h            | 18 ++++++++
>  arch/m68k/include/asm/atomic.h            | 36 ++++++++++++++++
>  arch/mips/include/asm/atomic.h            | 41 ++++++++++++++++++
>  arch/openrisc/include/asm/atomic.h        | 22 ++++++++++
>  arch/parisc/include/asm/atomic.h          | 20 +++++++++
>  arch/powerpc/include/asm/atomic.h         | 26 ++++++++++++
>  arch/riscv/include/asm/atomic.h           | 25 +++++++++++
>  arch/s390/include/asm/atomic.h            |  2 +
>  arch/s390/include/asm/atomic_ops.h        | 25 +++++++++++
>  arch/sparc/include/asm/atomic_32.h        |  2 +
>  arch/sparc/lib/atomic32.c                 | 17 ++++++++
>  arch/x86/include/asm/atomic.h             | 10 +++++
>  arch/xtensa/include/asm/atomic.h          | 49 +++++++++++++++++++++
>  include/asm-generic/atomic-instrumented.h | 28 ++++++++++++
>  include/asm-generic/atomic.h              | 29 +++++++++++++
>  include/linux/atomic-arch-fallback.h      | 42 ++++++++++++++++++
>  22 files changed, 588 insertions(+)
> 
> diff --git a/arch/alpha/include/asm/atomic.h b/arch/alpha/include/asm/atomic.h
> index f2861a43a61e..deb05ac292b8 100644
> --- a/arch/alpha/include/asm/atomic.h
> +++ b/arch/alpha/include/asm/atomic.h
> @@ -91,6 +91,25 @@ static inline int arch_atomic_fetch_##op##_relaxed(int i, atomic_t *v)	\
>  	return result;							\
>  }
>  
> +#define ATOMIC_FETCH_OP2(op, asm_op1, asm_op2)					\
> +static inline int arch_atomic_fetch_##op##_relaxed(int i, int j, atomic_t *v)	\
> +{									\
> +	long temp, result;						\
> +	__asm__ __volatile__(						\
> +	"1:	ldl_l %2,%1\n"						\
> +	"	" #asm_op1 " %2,%3,%0\n"				\
> +	"	" #asm_op2 " %0,%4,%0\n"				\
> +	"	stl_c %0,%1\n"						\
> +	"	beq %0,2f\n"						\
> +	".subsection 2\n"						\
> +	"2:	br 1b\n"						\
> +	".previous"							\
> +	:"=&r" (temp), "=m" (v->counter), "=&r" (result)		\
> +	:"Ir" (i), "Ir" (j), "m" (v->counter) : "memory");		\
> +	smp_mb();							\
> +	return result;							\
> +}
> +
>  #define ATOMIC64_OP(op, asm_op)						\
>  static __inline__ void arch_atomic64_##op(s64 i, atomic64_t * v)	\
>  {									\
> @@ -182,10 +201,17 @@ ATOMIC_OPS(andnot, bic)
>  ATOMIC_OPS(or, bis)
>  ATOMIC_OPS(xor, xor)
>  
> +#undef ATOMIC_OPS
> +#define ATOMIC_OPS(op, asm_op1, asm_op2)			\
> +	ATOMIC_FETCH_OP2(op, asm_op1, asm_op2)			\
> +
> +ATOMIC_OPS(and_or, and, bis)
> +
>  #define arch_atomic_fetch_and_relaxed		arch_atomic_fetch_and_relaxed
>  #define arch_atomic_fetch_andnot_relaxed	arch_atomic_fetch_andnot_relaxed
>  #define arch_atomic_fetch_or_relaxed		arch_atomic_fetch_or_relaxed
>  #define arch_atomic_fetch_xor_relaxed		arch_atomic_fetch_xor_relaxed
> +#define arch_atomic_fetch_and_or_relaxed	arch_atomic_fetch_and_or_relaxed
>  
>  #define arch_atomic64_fetch_and_relaxed		arch_atomic64_fetch_and_relaxed
>  #define arch_atomic64_fetch_andnot_relaxed	arch_atomic64_fetch_andnot_relaxed
> @@ -197,6 +223,7 @@ ATOMIC_OPS(xor, xor)
>  #undef ATOMIC64_OP_RETURN
>  #undef ATOMIC64_OP
>  #undef ATOMIC_FETCH_OP
> +#undef ATOMIC_FETCH_OP2
>  #undef ATOMIC_OP_RETURN
>  #undef ATOMIC_OP
>  
> diff --git a/arch/arc/include/asm/atomic.h b/arch/arc/include/asm/atomic.h
> index 7a36d79b5b2f..1aa9e0f396d7 100644
> --- a/arch/arc/include/asm/atomic.h
> +++ b/arch/arc/include/asm/atomic.h
> @@ -89,6 +89,35 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t *v)		\
>  	return orig;							\
>  }
>  
> +#define ATOMIC_FETCH_OP2(op, c_op1, c_op2, asm_op1, asm_op2)		\
> +static inline int arch_atomic_fetch_##op(int i, int j, atomic_t *v)	\
> +{									\
> +	unsigned int val, orig;						\
> +									\
> +	/*								\
> +	 * Explicit full memory barrier needed before/after as		\
> +	 * LLOCK/SCOND themselves don't provide any such semantics	\
> +	 */								\
> +	smp_mb();							\
> +									\
> +	__asm__ __volatile__(						\
> +	"1:	llock   %[orig], [%[ctr]]		\n"		\
> +	"	" #asm_op1 " %[val], %[orig], %[i]	\n"		\
> +	"	" #asm_op2 " %[val], %[val], %[j]	\n"		\
> +	"	scond   %[val], [%[ctr]]		\n"		\
> +	"	bnz     1b				\n"		\
> +	: [val]	"=&r"	(val),						\
> +	  [orig] "=&r" (orig)						\
> +	: [ctr]	"r"	(&v->counter),					\
> +	  [i]	"ir"	(i),						\
> +	  [j]	"ir"	(j),						\
> +	: "cc");							\
> +									\
> +	smp_mb();							\
> +									\
> +	return orig;							\
> +}
> +
>  #else	/* !CONFIG_ARC_HAS_LLSC */
>  
>  #ifndef CONFIG_SMP
> @@ -170,6 +199,23 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t *v)		\
>  	return orig;							\
>  }
>  
> +#define ATOMIC_FETCH_OP2(op, c_op1, c_op2, asm_op1, asm_op2)		\
> +static inline int arch_atomic_fetch_##op(int i, int j, atomic_t *v)	\
> +{									\
> +	unsigned long flags;						\
> +	unsigned long orig;						\
> +									\
> +	/*								\
> +	 * spin lock/unlock provides the needed smp_mb() before/after	\
> +	 */								\
> +	atomic_ops_lock(flags);						\
> +	orig = v->counter;						\
> +	v->counter = (orig c_op1 i) c_op2 j;				\
> +	atomic_ops_unlock(flags);					\
> +									\
> +	return orig;							\
> +}
> +
>  #endif /* !CONFIG_ARC_HAS_LLSC */
>  
>  #define ATOMIC_OPS(op, c_op, asm_op)					\
> @@ -190,6 +236,12 @@ ATOMIC_OPS(andnot, &= ~, bic)
>  ATOMIC_OPS(or, |=, or)
>  ATOMIC_OPS(xor, ^=, xor)
>  
> +#undef ATOMIC_OPS
> +#define ATOMIC_OPS(op, c_op1, c_op2, asm_op1, asm_op2)			\
> +	ATOMIC_FETCH_OP2(op, c_op1, c_op2, asm_op1, asm_op2)
> +
> +ATOMIC_OPS(and_or, &, |, and, or)
> +
>  #define arch_atomic_andnot		arch_atomic_andnot
>  #define arch_atomic_fetch_andnot	arch_atomic_fetch_andnot
>  
> diff --git a/arch/arm/include/asm/atomic.h b/arch/arm/include/asm/atomic.h
> index db8512d9a918..faddbc183ced 100644
> --- a/arch/arm/include/asm/atomic.h
> +++ b/arch/arm/include/asm/atomic.h
> @@ -93,6 +93,28 @@ static inline int arch_atomic_fetch_##op##_relaxed(int i, atomic_t *v)	\
>  	return result;							\
>  }
>  
> +#define ATOMIC_FETCH_OP2(op, c_op1, c_op2, asm_op1, asm_op2)			\
> +static inline int arch_atomic_fetch_##op##_relaxed(int i, int j, atomic_t *v)	\
> +{									\
> +	unsigned long tmp;						\
> +	int result, val;						\
> +									\
> +	prefetchw(&v->counter);						\
> +									\
> +	__asm__ __volatile__("@ atomic_fetch_" #op "\n"			\
> +"1:	ldrex	%0, [%4]\n"						\
> +"	" #asm_op1 "	%1, %0, %5\n"					\
> +"	" #asm_op2 "	%1, %1, %6\n"					\
> +"	strex	%2, %1, [%4]\n"						\
> +"	teq	%2, #0\n"						\
> +"	bne	1b"							\
> +	: "=&r" (result), "=&r" (val), "=&r" (tmp), "+Qo" (v->counter)	\
> +	: "r" (&v->counter), "Ir" (i), "Ir" (j)				\
> +	: "cc");							\
> +									\
> +	return result;							\
> +}
> +
>  #define arch_atomic_add_return_relaxed		arch_atomic_add_return_relaxed
>  #define arch_atomic_sub_return_relaxed		arch_atomic_sub_return_relaxed
>  #define arch_atomic_fetch_add_relaxed		arch_atomic_fetch_add_relaxed
> @@ -102,6 +124,7 @@ static inline int arch_atomic_fetch_##op##_relaxed(int i, atomic_t *v)	\
>  #define arch_atomic_fetch_andnot_relaxed	arch_atomic_fetch_andnot_relaxed
>  #define arch_atomic_fetch_or_relaxed		arch_atomic_fetch_or_relaxed
>  #define arch_atomic_fetch_xor_relaxed		arch_atomic_fetch_xor_relaxed
> +#define arch_atomic_fetch_and_or_relaxed	arch_atomic_fetch_and_or_relaxed
>  
>  static inline int arch_atomic_cmpxchg_relaxed(atomic_t *ptr, int old, int new)
>  {
> @@ -197,6 +220,20 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t *v)		\
>  	return val;							\
>  }
>  
> +#define ATOMIC_FETCH_OP2(op, c_op1, c_op2, asm_op1, asm_op2)		\
> +static inline int arch_atomic_fetch_##op(int i, int j, atomic_t *v)	\
> +{									\
> +	unsigned long flags;						\
> +	int val;							\
> +									\
> +	raw_local_irq_save(flags);					\
> +	val = v->counter;						\
> +	v->counter = (val c_op1 i) c_op2 j;				\
> +	raw_local_irq_restore(flags);					\
> +									\
> +	return val;							\
> +}
> +
>  static inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new)
>  {
>  	int ret;
> @@ -235,8 +272,15 @@ ATOMIC_OPS(andnot, &= ~, bic)
>  ATOMIC_OPS(or,  |=, orr)
>  ATOMIC_OPS(xor, ^=, eor)
>  
> +#undef ATOMIC_OPS
> +#define ATOMIC_OPS(op, c_op1, c_op2, asm_op1, asm_op2)		\
> +	ATOMIC_FETCH_OP2(op, c_op1, c_op2, asm_op1, asm_op2)
> +
> +ATOMIC_OPS(and_or, &, |, and, orr)
> +
>  #undef ATOMIC_OPS
>  #undef ATOMIC_FETCH_OP
> +#undef ATOMIC_FETCH_OP2
>  #undef ATOMIC_OP_RETURN
>  #undef ATOMIC_OP
>  
> diff --git a/arch/arm64/include/asm/atomic.h b/arch/arm64/include/asm/atomic.h
> index c9979273d389..3f1cdd3e2ef9 100644
> --- a/arch/arm64/include/asm/atomic.h
> +++ b/arch/arm64/include/asm/atomic.h
> @@ -43,6 +43,12 @@ static __always_inline int arch_##op##name(int i, atomic_t *v)		\
>  	ATOMIC_FETCH_OP(_release, op)					\
>  	ATOMIC_FETCH_OP(        , op)
>  
> +#define ATOMIC_FETCH_OP2(name, op)					\
> +static __always_inline int arch_##op##name(int i, int j, atomic_t *v)	\
> +{									\
> +	return __ll_sc_##op##name(i, j, v);				\
> +}
> +
>  ATOMIC_FETCH_OPS(atomic_fetch_andnot)
>  ATOMIC_FETCH_OPS(atomic_fetch_or)
>  ATOMIC_FETCH_OPS(atomic_fetch_xor)
> @@ -52,7 +58,17 @@ ATOMIC_FETCH_OPS(atomic_fetch_sub)
>  ATOMIC_FETCH_OPS(atomic_add_return)
>  ATOMIC_FETCH_OPS(atomic_sub_return)
>  
> +#undef ATOMIC_FETCH_OPS
> +#define ATOMIC_FETCH_OPS(op)						\
> +	ATOMIC_FETCH_OP2(_relaxed, op)					\
> +	ATOMIC_FETCH_OP2(_acquire, op)					\
> +	ATOMIC_FETCH_OP2(_release, op)					\
> +	ATOMIC_FETCH_OP2(        , op)
> +
> +ATOMIC_FETCH_OPS(atomic_fetch_and_or)
> +
>  #undef ATOMIC_FETCH_OP
> +#undef ATOMIC_FETCH_OP2
>  #undef ATOMIC_FETCH_OPS
>  
>  #define ATOMIC64_OP(op)							\
> diff --git a/arch/arm64/include/asm/atomic_ll_sc.h b/arch/arm64/include/asm/atomic_ll_sc.h
> index 13869b76b58c..90289c536ed6 100644
> --- a/arch/arm64/include/asm/atomic_ll_sc.h
> +++ b/arch/arm64/include/asm/atomic_ll_sc.h
> @@ -97,6 +97,29 @@ __ll_sc_atomic_fetch_##op##name(int i, atomic_t *v)			\
>  	return result;							\
>  }
>  
> +#define ATOMIC_FETCH_OP2(name, mb, acq, rel, cl, op, asm_op1, asm_op2, cstr) \
> +static inline int							\
> +__ll_sc_atomic_fetch_##op##name(int i, int j, atomic_t *v)		\
> +{									\
> +	unsigned long tmp;						\
> +	int val, result;						\
> +									\
> +	asm volatile("// atomic_fetch_" #op #name "\n"			\
> +	__LL_SC_FALLBACK(						\
> +"	prfm	pstl1strm, %3\n"					\
> +"1:	ld" #acq "xr	%w0, %3\n"					\
> +"	" #asm_op1 "	%w1, %w0, %w4\n"				\
> +"	" #asm_op2 "	%w1, %w1, %w5\n"				\
> +"	st" #rel "xr	%w2, %w1, %3\n"					\
> +"	cbnz	%w2, 1b\n"						\
> +"	" #mb )								\
> +	: "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter)	\
> +	: __stringify(cstr) "r" (i), __stringify(cstr) "r" (j)		\
> +	: cl);								\
> +									\
> +	return result;							\
> +}
> +
>  #define ATOMIC_OPS(...)							\
>  	ATOMIC_OP(__VA_ARGS__)						\
>  	ATOMIC_OP_RETURN(        , dmb ish,  , l, "memory", __VA_ARGS__)\
> @@ -129,8 +152,18 @@ ATOMIC_OPS(xor, eor, K)
>   */
>  ATOMIC_OPS(andnot, bic, )
>  
> +#undef ATOMIC_OPS
> +#define ATOMIC_OPS(...)							\
> +	ATOMIC_FETCH_OP2 (        , dmb ish,  , l, "memory", __VA_ARGS__)\
> +	ATOMIC_FETCH_OP2 (_relaxed,        ,  ,  ,         , __VA_ARGS__)\
> +	ATOMIC_FETCH_OP2 (_acquire,        , a,  , "memory", __VA_ARGS__)\
> +	ATOMIC_FETCH_OP2 (_release,        ,  , l, "memory", __VA_ARGS__)
> +
> +ATOMIC_OPS(and_or, and, orr, K)
> +
>  #undef ATOMIC_OPS
>  #undef ATOMIC_FETCH_OP
> +#undef ATOMIC_FETCH_OP2
>  #undef ATOMIC_OP_RETURN
>  #undef ATOMIC_OP
>  
> diff --git a/arch/hexagon/include/asm/atomic.h b/arch/hexagon/include/asm/atomic.h
> index 6e94f8d04146..d944e210085a 100644
> --- a/arch/hexagon/include/asm/atomic.h
> +++ b/arch/hexagon/include/asm/atomic.h
> @@ -130,6 +130,24 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t *v)		\
>  	return output;							\
>  }
>  
> +#define ATOMIC_FETCH_OP2(op1, op2)					\
> +static inline int arch_atomic_fetch_##op1##_##op2(int i, int j, atomic_t *v)\
> +{									\
> +	int output, val;						\
> +									\
> +	__asm__ __volatile__ (						\
> +		"1:	%0 = memw_locked(%2);\n"			\
> +		"	%1 = "#op1 "(%0,%3);\n"				\
> +		"	%1 = "#op2 "(%1,%4);\n"				\
> +		"	memw_locked(%2,P3)=%1;\n"			\
> +		"	if (!P3) jump 1b;\n"				\
> +		: "=&r" (output), "=&r" (val)				\
> +		: "r" (&v->counter), "r" (i), "r" (j)			\
> +		: "memory", "p3"					\
> +	);								\
> +	return output;							\
> +}
> +
>  #define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op) ATOMIC_FETCH_OP(op)
>  
>  ATOMIC_OPS(add)
> @@ -142,8 +160,14 @@ ATOMIC_OPS(and)
>  ATOMIC_OPS(or)
>  ATOMIC_OPS(xor)
>  
> +#undef ATOMIC_OPS
> +#define ATOMIC_OPS(op1, op2) ATOMIC_FETCH_OP2(op1, op2)
> +
> +ATOMIC_OPS(and, or)
> +
>  #undef ATOMIC_OPS
>  #undef ATOMIC_FETCH_OP
> +#undef ATOMIC_FETCH_OP2
>  #undef ATOMIC_OP_RETURN
>  #undef ATOMIC_OP
>  
> diff --git a/arch/ia64/include/asm/atomic.h b/arch/ia64/include/asm/atomic.h
> index 266c429b9137..6190108dcd53 100644
> --- a/arch/ia64/include/asm/atomic.h
> +++ b/arch/ia64/include/asm/atomic.h
> @@ -57,6 +57,21 @@ ia64_atomic_fetch_##op (int i, atomic_t *v)				\
>  	return old;							\
>  }
>  
> +#define ATOMIC_FETCH_OP2(op, c_op1, c_op2)				\
> +static __inline__ int							\
> +ia64_atomic_fetch_##op (int i, int j, atomic_t *v)			\
> +{									\
> +	__s32 old, new;							\
> +	CMPXCHG_BUGCHECK_DECL						\
> +									\
> +	do {								\
> +		CMPXCHG_BUGCHECK(v);					\
> +		old = arch_atomic_read(v);				\
> +		new = (old c_op1 i) c_op2 j;				\
> +	} while (ia64_cmpxchg(acq, v, old, new, sizeof(atomic_t)) != old); \
> +	return old;							\
> +}
> +
>  #define ATOMIC_OPS(op, c_op)						\
>  	ATOMIC_OP(op, c_op)						\
>  	ATOMIC_FETCH_OP(op, c_op)
> @@ -109,6 +124,7 @@ ATOMIC_OPS(sub, -)
>  ATOMIC_FETCH_OP(and, &)
>  ATOMIC_FETCH_OP(or, |)
>  ATOMIC_FETCH_OP(xor, ^)
> +ATOMIC_FETCH_OP2(and_or, &, |)
>  
>  #define arch_atomic_and(i,v)	(void)ia64_atomic_fetch_and(i,v)
>  #define arch_atomic_or(i,v)	(void)ia64_atomic_fetch_or(i,v)
> @@ -117,9 +133,11 @@ ATOMIC_FETCH_OP(xor, ^)
>  #define arch_atomic_fetch_and(i,v)	ia64_atomic_fetch_and(i,v)
>  #define arch_atomic_fetch_or(i,v)	ia64_atomic_fetch_or(i,v)
>  #define arch_atomic_fetch_xor(i,v)	ia64_atomic_fetch_xor(i,v)
> +#define arch_atomic_fetch_and_or(i,j,v)	ia64_atomic_fetch_and_or(i,j,v)
>  
>  #undef ATOMIC_OPS
>  #undef ATOMIC_FETCH_OP
> +#undef ATOMIC_FETCH_OP2
>  #undef ATOMIC_OP
>  
>  #define ATOMIC64_OP(op, c_op)						\
> diff --git a/arch/m68k/include/asm/atomic.h b/arch/m68k/include/asm/atomic.h
> index 8637bf8a2f65..480ecb6534a3 100644
> --- a/arch/m68k/include/asm/atomic.h
> +++ b/arch/m68k/include/asm/atomic.h
> @@ -67,6 +67,22 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t *v)		\
>  	return tmp;							\
>  }
>  
> +#define ATOMIC_FETCH_OP2(op, c_op1, c_op2, asm_op1, asm_op2)		\
> +static inline int arch_atomic_fetch_##op(int i, int j, atomic_t *v)	\
> +{									\
> +	int t, tmp;							\
> +									\
> +	__asm__ __volatile__(						\
> +			"1:	movel %2,%1\n"				\
> +			"	" #asm_op1 "l %3,%1\n"			\
> +			"	" #asm_op2 "l %4,%1\n"			\
> +			"	casl %2,%1,%0\n"			\
> +			"	jne 1b"					\
> +			: "+m" (*v), "=&d" (t), "=&d" (tmp)		\
> +			: "g" (i), "g" (j), "2" (arch_atomic_read(v)));	\
> +	return tmp;							\
> +}
> +
>  #else
>  
>  #define ATOMIC_OP_RETURN(op, c_op, asm_op)				\
> @@ -96,6 +112,20 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t * v)		\
>  	return t;							\
>  }
>  
> +#define ATOMIC_FETCH_OP2(op, c_op1, c_op2, asm_op1, asm_op2)		\
> +static inline int arch_atomic_fetch_##op(int i, int j, atomic_t * v)	\
> +{									\
> +	unsigned long flags;						\
> +	int t;								\
> +									\
> +	local_irq_save(flags);						\
> +	t = v->counter;							\
> +	v->counter = (t c_op1 i) c_op2 j;				\
> +	local_irq_restore(flags);					\
> +									\
> +	return t;							\
> +}
> +
>  #endif /* CONFIG_RMW_INSNS */
>  
>  #define ATOMIC_OPS(op, c_op, asm_op)					\
> @@ -115,6 +145,12 @@ ATOMIC_OPS(and, &=, and)
>  ATOMIC_OPS(or, |=, or)
>  ATOMIC_OPS(xor, ^=, eor)
>  
> +#undef ATOMIC_OPS
> +#define ATOMIC_OPS(op, c_op1, c_op2, asm_op1, asm_op2)			\
> +	ATOMIC_FETCH_OP2(op, c_op1, c_op2, asm_op1, asm_op2)
> +
> +ATOMIC_OPS(and_or, &, |, and, or)
> +
>  #undef ATOMIC_OPS
>  #undef ATOMIC_FETCH_OP
>  #undef ATOMIC_OP_RETURN
> diff --git a/arch/mips/include/asm/atomic.h b/arch/mips/include/asm/atomic.h
> index 95e1f7f3597f..84319b1ab9b6 100644
> --- a/arch/mips/include/asm/atomic.h
> +++ b/arch/mips/include/asm/atomic.h
> @@ -147,6 +147,39 @@ arch_##pfx##_fetch_##op##_relaxed(type i, pfx##_t * v)			\
>  	return result;							\
>  }
>  
> +#define ATOMIC_FETCH_OP2(pfx, op, type, c_op1, c_op2, asm_op1, asm_op2, ll, sc)\
> +static __inline__ type							\
> +arch_##pfx##_fetch_##op##_relaxed(type i, type j, pfx##_t * v)		\
> +{									\
> +	int temp, result;						\
> +									\
> +	if (!kernel_uses_llsc) {					\
> +		unsigned long flags;					\
> +									\
> +		raw_local_irq_save(flags);				\
> +		result = v->counter;					\
> +		v->counter = (result c_op1 i) c_op2 j;			\
> +		raw_local_irq_restore(flags);				\
> +		return result;						\
> +	}								\
> +									\
> +	__asm__ __volatile__(						\
> +	"	.set	push					\n"	\
> +	"	.set	" MIPS_ISA_LEVEL "			\n"	\
> +	"	" __SYNC(full, loongson3_war) "			\n"	\
> +	"1:	" #ll "	%0, %2		# " #pfx "_fetch_" #op "\n"	\
> +	"	" #asm_op1 " %1, %0, %3				\n"	\
> +	"	" #asm_op2 " %1, %1, %4				\n"	\
> +	"	" #sc "	%1, %2					\n"	\
> +	"\t" __SC_BEQZ "%1, 1b					\n"	\
> +	"	.set	pop					\n"	\
> +	: "=&r" (result), "=&r" (temp),					\
> +	  "+" GCC_OFF_SMALL_ASM() (v->counter)				\
> +	: "Ir" (i), "Ir" (j) : __LLSC_CLOBBER);				\
> +									\
> +	return result;							\
> +}
> +
>  #undef ATOMIC_OPS
>  #define ATOMIC_OPS(pfx, op, type, c_op, asm_op, ll, sc)			\
>  	ATOMIC_OP(pfx, op, type, c_op, asm_op, ll, sc)			\
> @@ -179,9 +212,16 @@ ATOMIC_OPS(atomic, and, int, &=, and, ll, sc)
>  ATOMIC_OPS(atomic, or, int, |=, or, ll, sc)
>  ATOMIC_OPS(atomic, xor, int, ^=, xor, ll, sc)
>  
> +#undef ATOMIC_OPS
> +#define ATOMIC_OPS(pfx, op, type, c_op1, c_op2, asm_op1, asm_op2, ll, sc)	\
> +	ATOMIC_FETCH_OP2(pfx, op, type, c_op1, c_op2, asm_op1, asm_op2, ll, sc)
> +
> +ATOMIC_OPS(atomic, and_or, int, &, |, and, or, ll, sc)
> +
>  #define arch_atomic_fetch_and_relaxed	arch_atomic_fetch_and_relaxed
>  #define arch_atomic_fetch_or_relaxed	arch_atomic_fetch_or_relaxed
>  #define arch_atomic_fetch_xor_relaxed	arch_atomic_fetch_xor_relaxed
> +#define arch_atomic_fetch_and_or_relaxed	arch_atomic_fetch_and_or_relaxed
>  
>  #ifdef CONFIG_64BIT
>  ATOMIC_OPS(atomic64, and, s64, &=, and, lld, scd)
> @@ -194,6 +234,7 @@ ATOMIC_OPS(atomic64, xor, s64, ^=, xor, lld, scd)
>  
>  #undef ATOMIC_OPS
>  #undef ATOMIC_FETCH_OP
> +#undef ATOMIC_FETCH_OP2
>  #undef ATOMIC_OP_RETURN
>  #undef ATOMIC_OP
>  
> diff --git a/arch/openrisc/include/asm/atomic.h b/arch/openrisc/include/asm/atomic.h
> index 326167e4783a..04598ef16977 100644
> --- a/arch/openrisc/include/asm/atomic.h
> +++ b/arch/openrisc/include/asm/atomic.h
> @@ -66,6 +66,25 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t *v)		\
>  	return old;							\
>  }
>  
> +#define ATOMIC_FETCH_OP2(op1, op2)					\
> +static inline int arch_atomic_fetch_##op1##_##op2(int i, int j, atomic_t *v)	\
> +{									\
> +	int tmp, old;							\
> +									\
> +	__asm__ __volatile__(						\
> +		"1:	l.lwa	%0,0(%2)	\n"			\
> +		"	l." #op1 " %1,%0,%3	\n"			\
> +		"	l." #op2 " %1,%1,%4	\n"			\
> +		"	l.swa	0(%2),%1	\n"			\
> +		"	l.bnf	1b		\n"			\
> +		"	 l.nop			\n"			\
> +		: "=&r"(old), "=&r"(tmp)				\
> +		: "r"(&v->counter), "r"(i), "r"(j)			\
> +		: "cc", "memory");					\
> +									\
> +	return old;							\
> +}
> +
>  ATOMIC_OP_RETURN(add)
>  ATOMIC_OP_RETURN(sub)
>  
> @@ -74,6 +93,7 @@ ATOMIC_FETCH_OP(sub)
>  ATOMIC_FETCH_OP(and)
>  ATOMIC_FETCH_OP(or)
>  ATOMIC_FETCH_OP(xor)
> +ATOMIC_FETCH_OP2(and, or)
>  
>  ATOMIC_OP(add)
>  ATOMIC_OP(sub)
> @@ -82,6 +102,7 @@ ATOMIC_OP(or)
>  ATOMIC_OP(xor)
>  
>  #undef ATOMIC_FETCH_OP
> +#undef ATOMIC_FETCH_OP2
>  #undef ATOMIC_OP_RETURN
>  #undef ATOMIC_OP
>  
> @@ -92,6 +113,7 @@ ATOMIC_OP(xor)
>  #define arch_atomic_fetch_and	arch_atomic_fetch_and
>  #define arch_atomic_fetch_or	arch_atomic_fetch_or
>  #define arch_atomic_fetch_xor	arch_atomic_fetch_xor
> +#define arch_atomic_fetch_and_or	arch_atomic_fetch_and_or
>  #define arch_atomic_add		arch_atomic_add
>  #define arch_atomic_sub		arch_atomic_sub
>  #define arch_atomic_and		arch_atomic_and
> diff --git a/arch/parisc/include/asm/atomic.h b/arch/parisc/include/asm/atomic.h
> index dd5a299ada69..59b9685ed2b1 100644
> --- a/arch/parisc/include/asm/atomic.h
> +++ b/arch/parisc/include/asm/atomic.h
> @@ -114,6 +114,20 @@ static __inline__ int arch_atomic_fetch_##op(int i, atomic_t *v)	\
>  	return ret;							\
>  }
>  
> +#define ATOMIC_FETCH_OP2(op, c_op1, c_op2)				\
> +static __inline__ int arch_atomic_fetch_##op(int i, int j, atomic_t *v)\
> +{									\
> +	unsigned long flags;						\
> +	int ret;							\
> +									\
> +	_atomic_spin_lock_irqsave(v, flags);				\
> +	ret = v->counter;						\
> +	v->counter = (ret c_op1 i) c_op2 j;				\
> +	_atomic_spin_unlock_irqrestore(v, flags);			\
> +									\
> +	return ret;							\
> +}
> +
>  #define ATOMIC_OPS(op, c_op)						\
>  	ATOMIC_OP(op, c_op)						\
>  	ATOMIC_OP_RETURN(op, c_op)					\
> @@ -131,6 +145,12 @@ ATOMIC_OPS(and, &=)
>  ATOMIC_OPS(or, |=)
>  ATOMIC_OPS(xor, ^=)
>  
> +#undef ATOMIC_OPS
> +#define ATOMIC_OPS(op, c_op1, c_op2)					\
> +	ATOMIC_FETCH_OP2(op, c_op1, c_op2)
> +
> +ATOMIC_OPS(and_or, &, |)
> +
>  #undef ATOMIC_OPS
>  #undef ATOMIC_FETCH_OP
>  #undef ATOMIC_OP_RETURN
> diff --git a/arch/powerpc/include/asm/atomic.h b/arch/powerpc/include/asm/atomic.h
> index a1732a79e92a..c2e966ab4b81 100644
> --- a/arch/powerpc/include/asm/atomic.h
> +++ b/arch/powerpc/include/asm/atomic.h
> @@ -86,6 +86,24 @@ static inline int arch_atomic_fetch_##op##_relaxed(int a, atomic_t *v)	\
>  	return res;							\
>  }
>  
> +#define ATOMIC_FETCH_OP2_RELAXED(op, asm_op1, asm_op2)			\
> +static inline int arch_atomic_fetch_##op##_relaxed(int a, int b, atomic_t *v)\
> +{									\
> +	int res, t;							\
> +									\
> +	__asm__ __volatile__(						\
> +"1:	lwarx	%0,0,%5		# atomic_fetch_" #op "_relaxed\n"	\
> +	#asm_op1 " %1,%3,%0\n"						\
> +	#asm_op2 " %1,%4,%1\n"						\
> +"	stwcx.	%1,0,%5\n"						\
> +"	bne-	1b\n"							\
> +	: "=&r" (res), "=&r" (t), "+m" (v->counter)			\
> +	: "r" (a), "r" (b), "r" (&v->counter)				\
> +	: "cc");							\
> +									\
> +	return res;							\
> +}
> +
>  #define ATOMIC_OPS(op, asm_op)						\
>  	ATOMIC_OP(op, asm_op)						\
>  	ATOMIC_OP_RETURN_RELAXED(op, asm_op)				\
> @@ -109,12 +127,20 @@ ATOMIC_OPS(and, and)
>  ATOMIC_OPS(or, or)
>  ATOMIC_OPS(xor, xor)
>  
> +#undef ATOMIC_OPS
> +#define ATOMIC_OPS(op, asm_op1, asm_op2)			\
> +	ATOMIC_FETCH_OP2_RELAXED(op, asm_op1, asm_op2)
> +
> +ATOMIC_OPS(and_or, and, or)
> +
>  #define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed
>  #define arch_atomic_fetch_or_relaxed  arch_atomic_fetch_or_relaxed
>  #define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed
> +#define arch_atomic_fetch_and_or_relaxed arch_atomic_fetch_and_or_relaxed
>  
>  #undef ATOMIC_OPS
>  #undef ATOMIC_FETCH_OP_RELAXED
> +#undef ATOMIC_FETCH_OP2_RELAXED
>  #undef ATOMIC_OP_RETURN_RELAXED
>  #undef ATOMIC_OP
>  
> diff --git a/arch/riscv/include/asm/atomic.h b/arch/riscv/include/asm/atomic.h
> index ac9bdf4fc404..572ca0ae2e76 100644
> --- a/arch/riscv/include/asm/atomic.h
> +++ b/arch/riscv/include/asm/atomic.h
> @@ -110,6 +110,24 @@ c_type arch_atomic##prefix##_fetch_##op(c_type i, atomic##prefix##_t *v)	\
>  	return ret;							\
>  }
>  
> +#define ATOMIC_FETCH_OP2(op, asm_op1, asm_op2, asm_type, c_type, prefix)	\
> +static __always_inline								\
> +c_type arch_atomic##prefix##_fetch_##op##_relaxed(c_type i,	c_type j,	\
> +					     atomic##prefix##_t *v)	\
> +{									\
> +	register c_type ret, tmp;					\
> +	__asm__ __volatile__ (						\
> +		"0:	lr." #asm_type " %0,  %2\n"			\
> +		"	" #asm_op1 "%1, %0, %3\n"			\
> +		"	" #asm_op2 "%1, %1, %4\n"			\
> +		"	sc." #asm_type " %1, %1, %2\n"			\
> +		"	bnez     %1, 0b\n"				\
> +		: "=r" (ret), "=&r" (tmp), "+A" (v->counter)		\
> +		: "r" (i), "r" (j)					\
> +		: "memory");						\
> +	return ret;							\
> +}
> +
>  #define ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_type, c_type, prefix)	\
>  static __always_inline							\
>  c_type arch_atomic##prefix##_##op##_return_relaxed(c_type i,		\
> @@ -175,9 +193,15 @@ ATOMIC_OPS(and, and, i)
>  ATOMIC_OPS( or,  or, i)
>  ATOMIC_OPS(xor, xor, i)
>  
> +#define ATOMIC_OPS(op, asm_op1, asm_op2, I)			\
> +        ATOMIC_FETCH_OP2(op, asm_op1, asm_op2, I, w, int,)
> +
> +ATOMIC_OPS(and_or, and, or, w)
> +
>  #define arch_atomic_fetch_and_relaxed	arch_atomic_fetch_and_relaxed
>  #define arch_atomic_fetch_or_relaxed	arch_atomic_fetch_or_relaxed
>  #define arch_atomic_fetch_xor_relaxed	arch_atomic_fetch_xor_relaxed
> +#define arch_atomic_fetch_and_or_relaxed	arch_atomic_fetch_and_or_relaxed
>  #define arch_atomic_fetch_and		arch_atomic_fetch_and
>  #define arch_atomic_fetch_or		arch_atomic_fetch_or
>  #define arch_atomic_fetch_xor		arch_atomic_fetch_xor
> @@ -194,6 +218,7 @@ ATOMIC_OPS(xor, xor, i)
>  #undef ATOMIC_OPS
>  
>  #undef ATOMIC_FETCH_OP
> +#undef ATOMIC_FETCH_OP2
>  #undef ATOMIC_OP_RETURN
>  
>  /* This is required to provide a full barrier on success. */
> diff --git a/arch/s390/include/asm/atomic.h b/arch/s390/include/asm/atomic.h
> index 7138d189cc42..abebd658c1fa 100644
> --- a/arch/s390/include/asm/atomic.h
> +++ b/arch/s390/include/asm/atomic.h
> @@ -62,6 +62,7 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t *v)		\
>  ATOMIC_OPS(and)
>  ATOMIC_OPS(or)
>  ATOMIC_OPS(xor)
> +ATOMIC_OPS(and_or)
>  
>  #undef ATOMIC_OPS
>  
> @@ -71,6 +72,7 @@ ATOMIC_OPS(xor)
>  #define arch_atomic_fetch_and		arch_atomic_fetch_and
>  #define arch_atomic_fetch_or		arch_atomic_fetch_or
>  #define arch_atomic_fetch_xor		arch_atomic_fetch_xor
> +#define arch_atomic_fetch_and_or	arch_atomic_fetch_and_or
>  
>  #define arch_atomic_xchg(v, new)	(arch_xchg(&((v)->counter), new))
>  
> diff --git a/arch/s390/include/asm/atomic_ops.h b/arch/s390/include/asm/atomic_ops.h
> index 50510e08b893..d396f2e2eb9a 100644
> --- a/arch/s390/include/asm/atomic_ops.h
> +++ b/arch/s390/include/asm/atomic_ops.h
> @@ -154,6 +154,31 @@ __ATOMIC64_OPS(__atomic64_xor, "xgr")
>  
>  #endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */
>  
> +#define __ATOMIC_OP2(op_name, op1, op2)				\
> +static inline int op_name(int i, int j, int *ptr)			\
> +{									\
> +	int old, new;							\
> +									\
> +	asm volatile(							\
> +		"0:	lr	%[new],%[old]\n"			\
> +		op1 "	%[new],%[i]\n"					\
> +		op2 "	%[new],%[j]\n"					\
> +		"	cs	%[old],%[new],%[ptr]\n"			\
> +		"	jl	0b"					\
> +		: [old] "=d" (old), [new] "=&d" (new), [ptr] "+Q" (*ptr)\
> +		: [i] "d" (i), [j] "d" (j), "0" (*ptr) : "cc", "memory");\
> +	return old;							\
> +}
> +
> +#define __ATOMIC_OPS(op_name, op1_string, op2_string)			\
> +	__ATOMIC_OP2(op_name, op1_string, op2_string)			\
> +	__ATOMIC_OP2(op_name##_barrier, op1_string, op2_string)
> +
> +__ATOMIC_OPS(__atomic_and_or, "ngr", "ogr")
> +
> +#undef __ATOMIC_OPS
> +#undef __ATOMIC_OP2
> +
>  static inline int __atomic_cmpxchg(int *ptr, int old, int new)
>  {
>  	asm volatile(
> diff --git a/arch/sparc/include/asm/atomic_32.h b/arch/sparc/include/asm/atomic_32.h
> index d775daa83d12..d062b20eb64c 100644
> --- a/arch/sparc/include/asm/atomic_32.h
> +++ b/arch/sparc/include/asm/atomic_32.h
> @@ -23,6 +23,7 @@ int arch_atomic_fetch_add(int, atomic_t *);
>  int arch_atomic_fetch_and(int, atomic_t *);
>  int arch_atomic_fetch_or(int, atomic_t *);
>  int arch_atomic_fetch_xor(int, atomic_t *);
> +int arch_atomic_fetch_and_or(int, int, atomic_t *);
>  int arch_atomic_cmpxchg(atomic_t *, int, int);
>  int arch_atomic_xchg(atomic_t *, int);
>  int arch_atomic_fetch_add_unless(atomic_t *, int, int);
> @@ -40,6 +41,7 @@ void arch_atomic_set(atomic_t *, int);
>  #define arch_atomic_and(i, v)	((void)arch_atomic_fetch_and((i), (v)))
>  #define arch_atomic_or(i, v)	((void)arch_atomic_fetch_or((i), (v)))
>  #define arch_atomic_xor(i, v)	((void)arch_atomic_fetch_xor((i), (v)))
> +#define arch_atomic_and_or(i, j, v)	((void)arch_atomic_fetch_and_or((i), (j), (v)))
>  
>  #define arch_atomic_sub_return(i, v)	(arch_atomic_add_return(-(int)(i), (v)))
>  #define arch_atomic_fetch_sub(i, v)	(arch_atomic_fetch_add (-(int)(i), (v)))
> diff --git a/arch/sparc/lib/atomic32.c b/arch/sparc/lib/atomic32.c
> index 8b81d0f00c97..aefb6d91985e 100644
> --- a/arch/sparc/lib/atomic32.c
> +++ b/arch/sparc/lib/atomic32.c
> @@ -43,6 +43,21 @@ int arch_atomic_fetch_##op(int i, atomic_t *v)				\
>  }									\
>  EXPORT_SYMBOL(arch_atomic_fetch_##op);
>  
> +#define ATOMIC_FETCH_OP2(op, c_op1, c_op2)				\
> +int arch_atomic_fetch_##op(int i, int j, atomic_t *v)			\
> +{									\
> +	int ret;							\
> +	unsigned long flags;						\
> +	spin_lock_irqsave(ATOMIC_HASH(v), flags);			\
> +									\
> +	ret = v->counter;						\
> +	v->counter = (ret c_op1 i) c_op2 j;				\
> +									\
> +	spin_unlock_irqrestore(ATOMIC_HASH(v), flags);			\
> +	return ret;							\
> +}									\
> +EXPORT_SYMBOL(arch_atomic_fetch_##op);
> +
>  #define ATOMIC_OP_RETURN(op, c_op)					\
>  int arch_atomic_##op##_return(int i, atomic_t *v)			\
>  {									\
> @@ -63,8 +78,10 @@ ATOMIC_FETCH_OP(add, +=)
>  ATOMIC_FETCH_OP(and, &=)
>  ATOMIC_FETCH_OP(or, |=)
>  ATOMIC_FETCH_OP(xor, ^=)
> +ATOMIC_FETCH_OP2(and_or, &, |)
>  
>  #undef ATOMIC_FETCH_OP
> +#undef ATOMIC_FETCH_OP2
>  #undef ATOMIC_OP_RETURN
>  
>  int arch_atomic_xchg(atomic_t *v, int new)
> diff --git a/arch/x86/include/asm/atomic.h b/arch/x86/include/asm/atomic.h
> index 5e754e895767..145dce45d02a 100644
> --- a/arch/x86/include/asm/atomic.h
> +++ b/arch/x86/include/asm/atomic.h
> @@ -263,6 +263,16 @@ static __always_inline int arch_atomic_fetch_xor(int i, atomic_t *v)
>  }
>  #define arch_atomic_fetch_xor arch_atomic_fetch_xor
>  
> +static __always_inline int arch_atomic_fetch_and_or(int i, int j, atomic_t *v)
> +{
> +	int val = arch_atomic_read(v);
> +
> +	do { } while (!arch_atomic_try_cmpxchg(v, &val, (val & i) | j));
> +
> +	return val;
> +}
> +#define arch_atomic_fetch_and_or arch_atomic_fetch_and_or
> +
>  #ifdef CONFIG_X86_32
>  # include <asm/atomic64_32.h>
>  #else
> diff --git a/arch/xtensa/include/asm/atomic.h b/arch/xtensa/include/asm/atomic.h
> index 4361fe4247e3..6b043cf74df2 100644
> --- a/arch/xtensa/include/asm/atomic.h
> +++ b/arch/xtensa/include/asm/atomic.h
> @@ -177,6 +177,28 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t * v)		\
>  	return result;							\
>  }
>  
> +#define ATOMIC_FETCH_OP2(op1, op2)					\
> +static inline int arch_atomic_fetch_##op1##_##op2(int i, int j, atomic_t * v)\
> +{									\
> +	unsigned long tmp;						\
> +	int result;							\
> +									\
> +	__asm__ __volatile__(						\
> +			"1:     l32i    %[tmp], %[mem]\n"		\
> +			"       wsr     %[tmp], scompare1\n"		\
> +			"       " #op1 " %[result], %[tmp], %[i]\n"	\
> +			"       " #op2 " %[result], %[result], %[j]\n"	\
> +			"       s32c1i  %[result], %[mem]\n"		\
> +			"       bne     %[result], %[tmp], 1b\n"	\
> +			: [result] "=&a" (result), [tmp] "=&a" (tmp),	\
> +			  [mem] "+m" (*v)				\
> +			: [i] "a" (i), [j] "a" (j)			\
> +			: "memory"					\
> +			);						\
> +									\
> +	return result;							\
> +}
> +
>  #else /* XCHAL_HAVE_S32C1I */
>  
>  #define ATOMIC_OP(op)							\
> @@ -238,6 +260,28 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t * v)		\
>  	return vval;							\
>  }
>  
> +#define ATOMIC_FETCH_OP2(op1, op2)					\
> +static inline int arch_atomic_fetch_##op1##_##op2(int i, int j, atomic_t * v)\
> +{									\
> +	unsigned int tmp, vval;						\
> +									\
> +	__asm__ __volatile__(						\
> +			"       rsil    a15,"__stringify(TOPLEVEL)"\n"	\
> +			"       l32i    %[result], %[mem]\n"		\
> +			"       " #op1 " %[tmp], %[result], %[i]\n"	\
> +			"       " #op2 " %[tmp], %[tmp], %[j]\n"	\
> +			"       s32i    %[tmp], %[mem]\n"		\
> +			"       wsr     a15, ps\n"			\
> +			"       rsync\n"				\
> +			: [result] "=&a" (vval), [tmp] "=&a" (tmp),	\
> +			  [mem] "+m" (*v)				\
> +			: [i] "a" (i), [j] "a" (j)			\
> +			: "a15", "memory"				\
> +			);						\
> +									\
> +	return vval;							\
> +}
> +
>  #endif /* XCHAL_HAVE_S32C1I */
>  
>  #define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op) ATOMIC_OP_RETURN(op)
> @@ -252,6 +296,11 @@ ATOMIC_OPS(and)
>  ATOMIC_OPS(or)
>  ATOMIC_OPS(xor)
>  
> +#undef ATOMIC_OPS
> +#define ATOMIC_OPS(op1, op2) ATOMIC_FETCH_OP2(op1, op2)
> +
> +ATOMIC_OPS(and, or)
> +
>  #undef ATOMIC_OPS
>  #undef ATOMIC_FETCH_OP
>  #undef ATOMIC_OP_RETURN
> diff --git a/include/asm-generic/atomic-instrumented.h b/include/asm-generic/atomic-instrumented.h
> index bc45af52c93b..231a8386ac80 100644
> --- a/include/asm-generic/atomic-instrumented.h
> +++ b/include/asm-generic/atomic-instrumented.h
> @@ -441,6 +441,34 @@ atomic_fetch_xor_relaxed(int i, atomic_t *v)
>  	return arch_atomic_fetch_xor_relaxed(i, v);
>  }
>  
> +static __always_inline int
> +atomic_fetch_and_or(int i, int j, atomic_t *v)
> +{
> +	instrument_atomic_read_write(v, sizeof(*v));
> +	return arch_atomic_fetch_and_or(i, j, v);
> +}
> +
> +static __always_inline int
> +atomic_fetch_and_or_acquire(int i, int j, atomic_t *v)
> +{
> +	instrument_atomic_read_write(v, sizeof(*v));
> +	return arch_atomic_fetch_and_or_acquire(i, j, v);
> +}
> +
> +static __always_inline int
> +atomic_fetch_and_or_release(int i, int j, atomic_t *v)
> +{
> +	instrument_atomic_read_write(v, sizeof(*v));
> +	return arch_atomic_fetch_and_or_release(i, j, v);
> +}
> +
> +static __always_inline int
> +atomic_fetch_and_or_relaxed(int i, int j, atomic_t *v)
> +{
> +	instrument_atomic_read_write(v, sizeof(*v));
> +	return arch_atomic_fetch_and_or_relaxed(i, j, v);
> +}
> +
>  static __always_inline int
>  atomic_xchg(atomic_t *v, int i)
>  {
> diff --git a/include/asm-generic/atomic.h b/include/asm-generic/atomic.h
> index 04b8be9f1a77..474e8cd8e58d 100644
> --- a/include/asm-generic/atomic.h
> +++ b/include/asm-generic/atomic.h
> @@ -50,6 +50,18 @@ static inline int generic_atomic_fetch_##op(int i, atomic_t *v)		\
>  	return c;							\
>  }
>  
> +#define ATOMIC_FETCH_OP2(op, c_op1, c_op2)				\
> +static inline int generic_atomic_fetch_##op(int i, int j, atomic_t *v)	\
> +{									\
> +	int c, old;							\
> +									\
> +	c = v->counter;							\
> +	while ((old = arch_cmpxchg(&v->counter, c, (c c_op1 i) c_op2 j)) != c)	\
> +		c = old;						\
> +									\
> +	return c;							\
> +}
> +
>  #else
>  
>  #include <linux/irqflags.h>
> @@ -91,6 +103,20 @@ static inline int generic_atomic_fetch_##op(int i, atomic_t *v)		\
>  	return ret;							\
>  }
>  
> +#define ATOMIC_FETCH_OP2(op, c_op1, c_op2)				\
> +static inline int generic_atomic_fetch_##op(int i, int j, atomic_t *v)	\
> +{									\
> +	unsigned long flags;						\
> +	int ret;							\
> +									\
> +	raw_local_irq_save(flags);					\
> +	ret = v->counter;						\
> +	v->counter = (v->counter c_op1 i) c_op2 j;			\
> +	raw_local_irq_restore(flags);					\
> +									\
> +	return ret;							\
> +}
> +
>  #endif /* CONFIG_SMP */
>  
>  ATOMIC_OP_RETURN(add, +)
> @@ -101,6 +127,7 @@ ATOMIC_FETCH_OP(sub, -)
>  ATOMIC_FETCH_OP(and, &)
>  ATOMIC_FETCH_OP(or, |)
>  ATOMIC_FETCH_OP(xor, ^)
> +ATOMIC_FETCH_OP2(and_or, &, |)
>  
>  ATOMIC_OP(add, +)
>  ATOMIC_OP(sub, -)
> @@ -109,6 +136,7 @@ ATOMIC_OP(or, |)
>  ATOMIC_OP(xor, ^)
>  
>  #undef ATOMIC_FETCH_OP
> +#undef ATOMIC_FETCH_OP2
>  #undef ATOMIC_OP_RETURN
>  #undef ATOMIC_OP
>  
> @@ -120,6 +148,7 @@ ATOMIC_OP(xor, ^)
>  #define arch_atomic_fetch_and			generic_atomic_fetch_and
>  #define arch_atomic_fetch_or			generic_atomic_fetch_or
>  #define arch_atomic_fetch_xor			generic_atomic_fetch_xor
> +#define arch_atomic_fetch_and_or		generic_atomic_fetch_and_or
>  
>  #define arch_atomic_add				generic_atomic_add
>  #define arch_atomic_sub				generic_atomic_sub
> diff --git a/include/linux/atomic-arch-fallback.h b/include/linux/atomic-arch-fallback.h
> index a3dba31df01e..92043a8d5b79 100644
> --- a/include/linux/atomic-arch-fallback.h
> +++ b/include/linux/atomic-arch-fallback.h
> @@ -891,6 +891,48 @@ arch_atomic_fetch_xor(int i, atomic_t *v)
>  
>  #endif /* arch_atomic_fetch_xor_relaxed */
>  
> +#ifndef arch_atomic_fetch_and_or_relaxed
> +#define arch_atomic_fetch_and_or_acquire arch_atomic_fetch_and_or
> +#define arch_atomic_fetch_and_or_release arch_atomic_fetch_and_or
> +#define arch_atomic_fetch_and_or_relaxed arch_atomic_fetch_and_or
> +#else /* arch_atomic_fetch_and_or_relaxed */
> +
> +#ifndef arch_atomic_fetch_and_or_acquire
> +static __always_inline int
> +arch_atomic_fetch_and_or_acquire(int i, int j, atomic_t *v)
> +{
> +	int ret = arch_atomic_fetch_and_or_relaxed(i, j, v);
> +	__atomic_acquire_fence();
> +	return ret;
> +}
> +#define arch_atomic_fetch_and_or_acquire arch_atomic_fetch_and_or_acquire
> +#endif
> +
> +#ifndef arch_atomic_fetch_and_or_release
> +static __always_inline int
> +arch_atomic_fetch_and_or_release(int i, int j, atomic_t *v)
> +{
> +	__atomic_release_fence();
> +	return arch_atomic_fetch_and_or_relaxed(i, j, v);
> +}
> +#define arch_atomic_fetch_and_or_release arch_atomic_fetch_and_or_release
> +#endif
> +
> +#ifndef arch_atomic_fetch_and_or
> +static __always_inline int
> +arch_atomic_fetch_and_or(int i, int j, atomic_t *v)
> +{
> +	int ret;
> +	__atomic_pre_full_fence();
> +	ret = arch_atomic_fetch_and_or_relaxed(i, j, v);
> +	__atomic_post_full_fence();
> +	return ret;
> +}
> +#define arch_atomic_fetch_and_or arch_atomic_fetch_and_or
> +#endif
> +
> +#endif /* arch_atomic_fetch_and_or_relaxed */
> +
>  #ifndef arch_atomic_xchg_relaxed
>  #define arch_atomic_xchg_acquire arch_atomic_xchg
>  #define arch_atomic_xchg_release arch_atomic_xchg
> -- 
> 2.32.0
> 

Re: [RFC PATCH v1 1/5] locking/atomic: Implement atomic_fetch_and_or

[Hev]

Hi, Boqun,

On Wed, Jul 28, 2021 at 8:15 PM Boqun Feng <boqun.feng@gmail.com> wrote:
>
> Hi,
>
> Thanks for the patchset. Seems that your git send-email command doesn't
> add the "In-Reply-to" tag for patch #2 to #5, so they are not threaded
> to patch #1. Not a big deal, but archives or email clients use that
> information to organize emails. You may want to check the command. Also,
> note that you can always use "--dry-run" option to preview the headers
> of your patchset ("--dry-run" won't do the actual send).
Thanks for your advice.

>
> On Wed, Jul 28, 2021 at 07:48:22PM +0800, Rui Wang wrote:
> > From: wangrui <wangrui@loongson.cn>
> >
> > This patch introduce a new atomic primitive 'and_or', It may be have three
> > types of implemeations:
> >
> >  * The generic implementation is based on arch_cmpxchg.
> >  * The hardware supports atomic 'and_or' of single instruction.
> >  * The hardware supports LL/SC style atomic operations:
> >
> >    1:  ll  v1, mem
> >        and t1, v1, arg1
> >        or  t1, t1, arg2
> >        sc  t1, mem
> >        beq t1, 0, 1b
> >
> > Now that all the architectures have implemented it.
> >
> > Signed-by-off: Rui Wang <wangrui@loongson.cn>
> > Signed-by-off: hev <r@hev.cc>
>
> First, this should be "Signed-off-by" ;-) Second, is the second
> "Signed-off-by" a mistake?
Beginner's luck :-)

>
> I will look into this for a review, in the meanwhile, but please add
> some tests in lib/atomic64_test.c, not only it will do the test at
> runtime, also it will generate asm code which helps people to review.
>
> Regards,
> Boqun
>

Regards,
Rui

Re: [RFC PATCH v1 1/5] locking/atomic: Implement atomic_fetch_and_or

[Peter Zijlstra]

On Wed, Jul 28, 2021 at 07:48:22PM +0800, Rui Wang wrote:
> From: wangrui <wangrui@loongson.cn>
> 
> This patch introduce a new atomic primitive 'and_or', It may be have three
> types of implemeations:
> 
>  * The generic implementation is based on arch_cmpxchg.
>  * The hardware supports atomic 'and_or' of single instruction.
>  * The hardware supports LL/SC style atomic operations:
> 
>    1:  ll  v1, mem
>        and t1, v1, arg1
>        or  t1, t1, arg2
>        sc  t1, mem
>        beq t1, 0, 1b
> 
> Now that all the architectures have implemented it.
> 
> Signed-by-off: Rui Wang <wangrui@loongson.cn>
> Signed-by-off: hev <r@hev.cc>
> ---

>  include/asm-generic/atomic-instrumented.h | 28 ++++++++++++
>  include/asm-generic/atomic.h              | 29 +++++++++++++
>  include/linux/atomic-arch-fallback.h      | 42 ++++++++++++++++++
>  22 files changed, 588 insertions(+)

> diff --git a/include/asm-generic/atomic-instrumented.h b/include/asm-generic/atomic-instrumented.h
> index bc45af52c93b..231a8386ac80 100644
> --- a/include/asm-generic/atomic-instrumented.h
> +++ b/include/asm-generic/atomic-instrumented.h
> @@ -441,6 +441,34 @@ atomic_fetch_xor_relaxed(int i, atomic_t *v)
>  	return arch_atomic_fetch_xor_relaxed(i, v);
>  }
>  
> +static __always_inline int
> +atomic_fetch_and_or(int i, int j, atomic_t *v)
> +{
> +	instrument_atomic_read_write(v, sizeof(*v));
> +	return arch_atomic_fetch_and_or(i, j, v);
> +}
> +
> +static __always_inline int
> +atomic_fetch_and_or_acquire(int i, int j, atomic_t *v)
> +{
> +	instrument_atomic_read_write(v, sizeof(*v));
> +	return arch_atomic_fetch_and_or_acquire(i, j, v);
> +}
> +
> +static __always_inline int
> +atomic_fetch_and_or_release(int i, int j, atomic_t *v)
> +{
> +	instrument_atomic_read_write(v, sizeof(*v));
> +	return arch_atomic_fetch_and_or_release(i, j, v);
> +}
> +
> +static __always_inline int
> +atomic_fetch_and_or_relaxed(int i, int j, atomic_t *v)
> +{
> +	instrument_atomic_read_write(v, sizeof(*v));
> +	return arch_atomic_fetch_and_or_relaxed(i, j, v);
> +}
> +
>  static __always_inline int
>  atomic_xchg(atomic_t *v, int i)
>  {
> diff --git a/include/asm-generic/atomic.h b/include/asm-generic/atomic.h
> index 04b8be9f1a77..474e8cd8e58d 100644
> --- a/include/asm-generic/atomic.h
> +++ b/include/asm-generic/atomic.h
> @@ -50,6 +50,18 @@ static inline int generic_atomic_fetch_##op(int i, atomic_t *v)		\
>  	return c;							\
>  }
>  
> +#define ATOMIC_FETCH_OP2(op, c_op1, c_op2)				\
> +static inline int generic_atomic_fetch_##op(int i, int j, atomic_t *v)	\
> +{									\
> +	int c, old;							\
> +									\
> +	c = v->counter;							\
> +	while ((old = arch_cmpxchg(&v->counter, c, (c c_op1 i) c_op2 j)) != c)	\
> +		c = old;						\
> +									\
> +	return c;							\
> +}
> +
>  #else
>  
>  #include <linux/irqflags.h>
> @@ -91,6 +103,20 @@ static inline int generic_atomic_fetch_##op(int i, atomic_t *v)		\
>  	return ret;							\
>  }
>  
> +#define ATOMIC_FETCH_OP2(op, c_op1, c_op2)				\
> +static inline int generic_atomic_fetch_##op(int i, int j, atomic_t *v)	\
> +{									\
> +	unsigned long flags;						\
> +	int ret;							\
> +									\
> +	raw_local_irq_save(flags);					\
> +	ret = v->counter;						\
> +	v->counter = (v->counter c_op1 i) c_op2 j;			\
> +	raw_local_irq_restore(flags);					\
> +									\
> +	return ret;							\
> +}
> +
>  #endif /* CONFIG_SMP */
>  
>  ATOMIC_OP_RETURN(add, +)
> @@ -101,6 +127,7 @@ ATOMIC_FETCH_OP(sub, -)
>  ATOMIC_FETCH_OP(and, &)
>  ATOMIC_FETCH_OP(or, |)
>  ATOMIC_FETCH_OP(xor, ^)
> +ATOMIC_FETCH_OP2(and_or, &, |)
>  
>  ATOMIC_OP(add, +)
>  ATOMIC_OP(sub, -)
> @@ -109,6 +136,7 @@ ATOMIC_OP(or, |)
>  ATOMIC_OP(xor, ^)
>  
>  #undef ATOMIC_FETCH_OP
> +#undef ATOMIC_FETCH_OP2
>  #undef ATOMIC_OP_RETURN
>  #undef ATOMIC_OP
>  
> @@ -120,6 +148,7 @@ ATOMIC_OP(xor, ^)
>  #define arch_atomic_fetch_and			generic_atomic_fetch_and
>  #define arch_atomic_fetch_or			generic_atomic_fetch_or
>  #define arch_atomic_fetch_xor			generic_atomic_fetch_xor
> +#define arch_atomic_fetch_and_or		generic_atomic_fetch_and_or
>  
>  #define arch_atomic_add				generic_atomic_add
>  #define arch_atomic_sub				generic_atomic_sub
> diff --git a/include/linux/atomic-arch-fallback.h b/include/linux/atomic-arch-fallback.h
> index a3dba31df01e..92043a8d5b79 100644
> --- a/include/linux/atomic-arch-fallback.h
> +++ b/include/linux/atomic-arch-fallback.h
> @@ -891,6 +891,48 @@ arch_atomic_fetch_xor(int i, atomic_t *v)
>  
>  #endif /* arch_atomic_fetch_xor_relaxed */
>  
> +#ifndef arch_atomic_fetch_and_or_relaxed
> +#define arch_atomic_fetch_and_or_acquire arch_atomic_fetch_and_or
> +#define arch_atomic_fetch_and_or_release arch_atomic_fetch_and_or
> +#define arch_atomic_fetch_and_or_relaxed arch_atomic_fetch_and_or
> +#else /* arch_atomic_fetch_and_or_relaxed */
> +
> +#ifndef arch_atomic_fetch_and_or_acquire
> +static __always_inline int
> +arch_atomic_fetch_and_or_acquire(int i, int j, atomic_t *v)
> +{
> +	int ret = arch_atomic_fetch_and_or_relaxed(i, j, v);
> +	__atomic_acquire_fence();
> +	return ret;
> +}
> +#define arch_atomic_fetch_and_or_acquire arch_atomic_fetch_and_or_acquire
> +#endif
> +
> +#ifndef arch_atomic_fetch_and_or_release
> +static __always_inline int
> +arch_atomic_fetch_and_or_release(int i, int j, atomic_t *v)
> +{
> +	__atomic_release_fence();
> +	return arch_atomic_fetch_and_or_relaxed(i, j, v);
> +}
> +#define arch_atomic_fetch_and_or_release arch_atomic_fetch_and_or_release
> +#endif
> +
> +#ifndef arch_atomic_fetch_and_or
> +static __always_inline int
> +arch_atomic_fetch_and_or(int i, int j, atomic_t *v)
> +{
> +	int ret;
> +	__atomic_pre_full_fence();
> +	ret = arch_atomic_fetch_and_or_relaxed(i, j, v);
> +	__atomic_post_full_fence();
> +	return ret;
> +}
> +#define arch_atomic_fetch_and_or arch_atomic_fetch_and_or
> +#endif
> +
> +#endif /* arch_atomic_fetch_and_or_relaxed */
> +
>  #ifndef arch_atomic_xchg_relaxed
>  #define arch_atomic_xchg_acquire arch_atomic_xchg
>  #define arch_atomic_xchg_release arch_atomic_xchg

Urgh.. please start from something like the below and then run:

  scripts/atomic/gen-atomics.sh

The below isn't quite right, because it'll use try_cmpxchg() for
atomic_andnot_or(), which by being a void atomic should be _relaxed. I'm
not entirely sure how to make that happen in a hurry.

---

diff --git a/scripts/atomic/atomics.tbl b/scripts/atomic/atomics.tbl
index fbee2f6190d9..3aaa0caa6b2d 100755
--- a/scripts/atomic/atomics.tbl
+++ b/scripts/atomic/atomics.tbl
@@ -39,3 +39,4 @@ inc_not_zero		b	v
 inc_unless_negative	b	v
 dec_unless_positive	b	v
 dec_if_positive		i	v
+andnot_or		vF	v	i:m	i:o
diff --git a/scripts/atomic/fallbacks/andnot_or b/scripts/atomic/fallbacks/andnot_or
new file mode 100644
index 000000000000..f50e78d6c53a
--- /dev/null
+++ b/scripts/atomic/fallbacks/andnot_or
@@ -0,0 +1,15 @@
+cat <<EOF
+static __always_inline ${ret}
+arch_${atomic}_${pfx}andnot_or${sfx}${order}(${atomic}_t *v, ${int} m, ${int} o)
+{
+	${retstmt}({
+		${int} N, O = atomic_read(v);
+		do {
+			N = O;
+			N &= ~m;
+			N |= o;
+		} while (!arch_${atomic}_try_cmpxchg${order}(v, &O, N));
+		O;
+	});
+}
+EOF
diff --git a/scripts/atomic/fallbacks/fetch_andnot_or b/scripts/atomic/fallbacks/fetch_andnot_or
deleted file mode 100644
index e69de29bb2d1..000000000000

Re: [RFC PATCH v1 1/5] locking/atomic: Implement atomic_fetch_and_or

[Peter Zijlstra]

On Wed, Jul 28, 2021 at 02:58:35PM +0200, Peter Zijlstra wrote:
> Urgh.. please start from something like the below and then run:
> 
>   scripts/atomic/gen-atomics.sh
> 
> The below isn't quite right, because it'll use try_cmpxchg() for
> atomic_andnot_or(), which by being a void atomic should be _relaxed. I'm
> not entirely sure how to make that happen in a hurry.
> 

Of note, the below is on top of tip/locking/core, which includes a bunch
of atomic work from Mark.

> ---
> 
> diff --git a/scripts/atomic/atomics.tbl b/scripts/atomic/atomics.tbl
> index fbee2f6190d9..3aaa0caa6b2d 100755
> --- a/scripts/atomic/atomics.tbl
> +++ b/scripts/atomic/atomics.tbl
> @@ -39,3 +39,4 @@ inc_not_zero		b	v
>  inc_unless_negative	b	v
>  dec_unless_positive	b	v
>  dec_if_positive		i	v
> +andnot_or		vF	v	i:m	i:o
> diff --git a/scripts/atomic/fallbacks/andnot_or b/scripts/atomic/fallbacks/andnot_or
> new file mode 100644
> index 000000000000..f50e78d6c53a
> --- /dev/null
> +++ b/scripts/atomic/fallbacks/andnot_or
> @@ -0,0 +1,15 @@
> +cat <<EOF
> +static __always_inline ${ret}
> +arch_${atomic}_${pfx}andnot_or${sfx}${order}(${atomic}_t *v, ${int} m, ${int} o)
> +{
> +	${retstmt}({
> +		${int} N, O = atomic_read(v);
> +		do {
> +			N = O;
> +			N &= ~m;
> +			N |= o;
> +		} while (!arch_${atomic}_try_cmpxchg${order}(v, &O, N));
> +		O;
> +	});
> +}
> +EOF
> diff --git a/scripts/atomic/fallbacks/fetch_andnot_or b/scripts/atomic/fallbacks/fetch_andnot_or
> deleted file mode 100644
> index e69de29bb2d1..000000000000
> 

Re: [RFC PATCH v1 1/5] locking/atomic: Implement atomic_fetch_and_or

[Peter Zijlstra]

On Wed, Jul 28, 2021 at 02:58:35PM +0200, Peter Zijlstra wrote:
> The below isn't quite right, because it'll use try_cmpxchg() for
> atomic_andnot_or(), which by being a void atomic should be _relaxed. I'm
> not entirely sure how to make that happen in a hurry.
> 
> ---

This seems to do the trick.

diff --git a/scripts/atomic/atomics.tbl b/scripts/atomic/atomics.tbl
index fbee2f6190d9..3aaa0caa6b2d 100755
--- a/scripts/atomic/atomics.tbl
+++ b/scripts/atomic/atomics.tbl
@@ -39,3 +39,4 @@ inc_not_zero		b	v
 inc_unless_negative	b	v
 dec_unless_positive	b	v
 dec_if_positive		i	v
+andnot_or		vF	v	i:m	i:o
diff --git a/scripts/atomic/fallbacks/andnot_or b/scripts/atomic/fallbacks/andnot_or
new file mode 100644
index 000000000000..995d80db8578
--- /dev/null
+++ b/scripts/atomic/fallbacks/andnot_or
@@ -0,0 +1,23 @@
+local try_order=${order}
+
+if meta_has_ret "${meta}"; then
+	:;
+else
+	try_order="_relaxed"
+fi
+
+cat <<EOF
+static __always_inline ${ret}
+arch_${atomic}_${pfx}andnot_or${sfx}${order}(${atomic}_t *v, ${int} m, ${int} o)
+{
+	${retstmt}({
+		${int} N, O = atomic_read(v);
+		do {
+			N = O;
+			N &= ~m;
+			N |= o;
+		} while (!arch_${atomic}_try_cmpxchg${try_order}(v, &O, N));
+		O;
+	});
+}
+EOF
diff --git a/scripts/atomic/fallbacks/fetch_andnot_or b/scripts/atomic/fallbacks/fetch_andnot_or
deleted file mode 100644
index e69de29bb2d1..000000000000

Re: [RFC PATCH v1 1/5] locking/atomic: Implement atomic_fetch_and_or

[Peter Zijlstra]

On Wed, Jul 28, 2021 at 03:16:54PM +0200, Peter Zijlstra wrote:
> On Wed, Jul 28, 2021 at 02:58:35PM +0200, Peter Zijlstra wrote:
> > The below isn't quite right, because it'll use try_cmpxchg() for
> > atomic_andnot_or(), which by being a void atomic should be _relaxed. I'm
> > not entirely sure how to make that happen in a hurry.
> > 
> > ---
> 
> This seems to do the trick.
> 

Mark suggested this, which is probably nicer still.

---
diff --git a/scripts/atomic/atomics.tbl b/scripts/atomic/atomics.tbl
index fbee2f6190d9..3aaa0caa6b2d 100755
--- a/scripts/atomic/atomics.tbl
+++ b/scripts/atomic/atomics.tbl
@@ -39,3 +39,4 @@ inc_not_zero		b	v
 inc_unless_negative	b	v
 dec_unless_positive	b	v
 dec_if_positive		i	v
+andnot_or		vF	v	i:m	i:o
diff --git a/scripts/atomic/fallbacks/andnot_or b/scripts/atomic/fallbacks/andnot_or
new file mode 100644
index 000000000000..0fb3a728c0ff
--- /dev/null
+++ b/scripts/atomic/fallbacks/andnot_or
@@ -0,0 +1,24 @@
+local try_order=${order}
+
+#
+# non-value returning atomics are implicity relaxed
+#
+if [ -z "${retstmt}" ]; then
+	try_order="_relaxed"
+fi
+
+cat <<EOF
+static __always_inline ${ret}
+arch_${atomic}_${pfx}andnot_or${sfx}${order}(${atomic}_t *v, ${int} m, ${int} o)
+{
+	${retstmt}({
+		${int} N, O = atomic_read(v);
+		do {
+			N = O;
+			N &= ~m;
+			N |= o;
+		} while (!arch_${atomic}_try_cmpxchg${try_order}(v, &O, N));
+		O;
+	});
+}
+EOF
diff --git a/scripts/atomic/fallbacks/fetch_andnot_or b/scripts/atomic/fallbacks/fetch_andnot_or
deleted file mode 100644
index e69de29bb2d1..000000000000

Re: [RFC PATCH v1 1/5] locking/atomic: Implement atomic_fetch_and_or

[hev]

Hi, Peter,

On Wed, Jul 28, 2021 at 9:21 PM Peter Zijlstra <peterz@infradead.org> wrote:
>
> On Wed, Jul 28, 2021 at 03:16:54PM +0200, Peter Zijlstra wrote:
> > On Wed, Jul 28, 2021 at 02:58:35PM +0200, Peter Zijlstra wrote:
> > > The below isn't quite right, because it'll use try_cmpxchg() for
> > > atomic_andnot_or(), which by being a void atomic should be _relaxed. I'm
> > > not entirely sure how to make that happen in a hurry.
> > >
> > > ---
> >
> > This seems to do the trick.
> >
>
> Mark suggested this, which is probably nicer still.
Wow, Amazing! so the architecture dependent can be implemented one by one.

Regards
Rui

>
> ---
> diff --git a/scripts/atomic/atomics.tbl b/scripts/atomic/atomics.tbl
> index fbee2f6190d9..3aaa0caa6b2d 100755
> --- a/scripts/atomic/atomics.tbl
> +++ b/scripts/atomic/atomics.tbl
> @@ -39,3 +39,4 @@ inc_not_zero          b       v
>  inc_unless_negative    b       v
>  dec_unless_positive    b       v
>  dec_if_positive                i       v
> +andnot_or              vF      v       i:m     i:o
> diff --git a/scripts/atomic/fallbacks/andnot_or b/scripts/atomic/fallbacks/andnot_or
> new file mode 100644
> index 000000000000..0fb3a728c0ff
> --- /dev/null
> +++ b/scripts/atomic/fallbacks/andnot_or
> @@ -0,0 +1,24 @@
> +local try_order=${order}
> +
> +#
> +# non-value returning atomics are implicity relaxed
> +#
> +if [ -z "${retstmt}" ]; then
> +       try_order="_relaxed"
> +fi
> +
> +cat <<EOF
> +static __always_inline ${ret}
> +arch_${atomic}_${pfx}andnot_or${sfx}${order}(${atomic}_t *v, ${int} m, ${int} o)
> +{
> +       ${retstmt}({
> +               ${int} N, O = atomic_read(v);
> +               do {
> +                       N = O;
> +                       N &= ~m;
> +                       N |= o;
> +               } while (!arch_${atomic}_try_cmpxchg${try_order}(v, &O, N));
> +               O;
> +       });
> +}
> +EOF
> diff --git a/scripts/atomic/fallbacks/fetch_andnot_or b/scripts/atomic/fallbacks/fetch_andnot_or
> deleted file mode 100644
> index e69de29bb2d1..000000000000

Re: [RFC PATCH v1 1/5] locking/atomic: Implement atomic_fetch_and_or

[Peter Zijlstra]

On Thu, Jul 29, 2021 at 09:58:02AM +0800, hev wrote:
> Hi, Peter,
> 
> On Wed, Jul 28, 2021 at 9:21 PM Peter Zijlstra <peterz@infradead.org> wrote:
> >
> > On Wed, Jul 28, 2021 at 03:16:54PM +0200, Peter Zijlstra wrote:
> > > On Wed, Jul 28, 2021 at 02:58:35PM +0200, Peter Zijlstra wrote:
> > > > The below isn't quite right, because it'll use try_cmpxchg() for
> > > > atomic_andnot_or(), which by being a void atomic should be _relaxed. I'm
> > > > not entirely sure how to make that happen in a hurry.
> > > >
> > > > ---
> > >
> > > This seems to do the trick.
> > >
> >
> > Mark suggested this, which is probably nicer still.
> Wow, Amazing! so the architecture dependent can be implemented one by one.

Nah, this is just the fallback, you still need individual arch code to
optimize this and get proper LL/SC primitives.

Re: [RFC PATCH v1 1/5] locking/atomic: Implement atomic_fetch_and_or

[hev]

Hi, Peter,

On Thu, Jul 29, 2021 at 4:24 PM Peter Zijlstra <peterz@infradead.org> wrote:
>
> On Thu, Jul 29, 2021 at 09:58:02AM +0800, hev wrote:
> > Hi, Peter,
> >
> > On Wed, Jul 28, 2021 at 9:21 PM Peter Zijlstra <peterz@infradead.org> wrote:
> > >
> > > On Wed, Jul 28, 2021 at 03:16:54PM +0200, Peter Zijlstra wrote:
> > > > On Wed, Jul 28, 2021 at 02:58:35PM +0200, Peter Zijlstra wrote:
> > > > > The below isn't quite right, because it'll use try_cmpxchg() for
> > > > > atomic_andnot_or(), which by being a void atomic should be _relaxed. I'm
> > > > > not entirely sure how to make that happen in a hurry.
> > > > >
> > > > > ---
> > > >
> > > > This seems to do the trick.
> > > >
> > >
> > > Mark suggested this, which is probably nicer still.
> > Wow, Amazing! so the architecture dependent can be implemented one by one.
>
> Nah, this is just the fallback, you still need individual arch code to
> optimize this and get proper LL/SC primitives.

Okay, already started. If the atomic andnot_or seems to be ok, I will
send the rest of the patches.

Regards
Rui

Re: [RFC PATCH v1 1/5] locking/atomic: Implement atomic_fetch_and_or

[Will Deacon]

On Wed, Jul 28, 2021 at 07:48:22PM +0800, Rui Wang wrote:
> From: wangrui <wangrui@loongson.cn>
> 
> This patch introduce a new atomic primitive 'and_or', It may be have three
> types of implemeations:
> 
>  * The generic implementation is based on arch_cmpxchg.
>  * The hardware supports atomic 'and_or' of single instruction.

Do any architectures actually support this instruction?

On arm64, we can clear arbitrary bits and we can set arbitrary bits, but we
can't combine the two in a fashion which provides atomicity and
forward-progress guarantees.

Please can you explain how this new primitive will be used, in case there's
an alternative way of doing it which maps better to what CPUs can actually
do?

Cheers,

Will

Re: [RFC PATCH v1 1/5] locking/atomic: Implement atomic_fetch_and_or

[hev]

Hi, Will,

On Thu, Jul 29, 2021 at 5:39 PM Will Deacon <will@kernel.org> wrote:
>
> On Wed, Jul 28, 2021 at 07:48:22PM +0800, Rui Wang wrote:
> > From: wangrui <wangrui@loongson.cn>
> >
> > This patch introduce a new atomic primitive 'and_or', It may be have three
> > types of implemeations:
> >
> >  * The generic implementation is based on arch_cmpxchg.
> >  * The hardware supports atomic 'and_or' of single instruction.
>
> Do any architectures actually support this instruction?
No, I'm not sure now.

>
> On arm64, we can clear arbitrary bits and we can set arbitrary bits, but we
> can't combine the two in a fashion which provides atomicity and
> forward-progress guarantees.
>
> Please can you explain how this new primitive will be used, in case there's
> an alternative way of doing it which maps better to what CPUs can actually
> do?
I think we can easily exchange arbitrary bits of a machine word with atomic
andnot_or/and_or. Otherwise, we can only use xchg8/16 to do it. It depends on
hardware support, and the key point is that the bits to be exchanged
must be in the
same sub-word. qspinlock adjusted memory layout for this reason, and waste some
bits(_Q_PENDING_BITS == 8).

In the case of qspinlock xchg_tail, I think there is no change in the
assembly code
after switching to atomic andnot_or, for the architecture that
supports CAS instructions.
But for LL/SC style architectures, We can implement xchg for sub-word
better with new
primitive and clear[1]. And in fact, it reduces the number of retries
when the two memory
load values are not equal.

If the hardware supports this atomic semantics, we will get better
performance and flexibility.
I think the hardware is easy to support.

[1] https://github.com/heiher/linux/commit/f77e1c6e4e579543177010bef2b394479c50b6cf

Regards
Rui

>
> Cheers,
>
> Will

Re: [RFC PATCH v1 1/5] locking/atomic: Implement atomic_fetch_and_or

[Will Deacon]

On Thu, Jul 29, 2021 at 06:18:28PM +0800, hev wrote:
> On Thu, Jul 29, 2021 at 5:39 PM Will Deacon <will@kernel.org> wrote:
> >
> > On Wed, Jul 28, 2021 at 07:48:22PM +0800, Rui Wang wrote:
> > > From: wangrui <wangrui@loongson.cn>
> > >
> > > This patch introduce a new atomic primitive 'and_or', It may be have three
> > > types of implemeations:
> > >
> > >  * The generic implementation is based on arch_cmpxchg.
> > >  * The hardware supports atomic 'and_or' of single instruction.
> >
> > Do any architectures actually support this instruction?
> No, I'm not sure now.
> 
> >
> > On arm64, we can clear arbitrary bits and we can set arbitrary bits, but we
> > can't combine the two in a fashion which provides atomicity and
> > forward-progress guarantees.
> >
> > Please can you explain how this new primitive will be used, in case there's
> > an alternative way of doing it which maps better to what CPUs can actually
> > do?
> I think we can easily exchange arbitrary bits of a machine word with atomic
> andnot_or/and_or. Otherwise, we can only use xchg8/16 to do it. It depends on
> hardware support, and the key point is that the bits to be exchanged
> must be in the
> same sub-word. qspinlock adjusted memory layout for this reason, and waste some
> bits(_Q_PENDING_BITS == 8).

No, it's not about wasting bits -- short xchg() is exactly what you want to
do here, it's just that when you get more than 13 bits of CPU number (which
is, err, unusual) then we need more space in the lockword to track the tail,
and so the other fields end up sharing bytes.

> In the case of qspinlock xchg_tail, I think there is no change in the
> assembly code
> after switching to atomic andnot_or, for the architecture that
> supports CAS instructions.
> But for LL/SC style architectures, We can implement xchg for sub-word
> better with new
> primitive and clear[1]. And in fact, it reduces the number of retries
> when the two memory
> load values are not equal.

The only system using LL/SC with this many CPUs is probably Power, and their
atomics are dirt slow anyway.

> If the hardware supports this atomic semantics, we will get better
> performance and flexibility.
> I think the hardware is easy to support.

The issue I have is exposing these new functions as first-class members of
the atomics API. On architectures with AMO instructions, falling back to
cmpxchg() will have a radically different performance profile when compared
to many of the other atomics operations and so I don't think we should add
them without very good justification.

At the very least, we could update the atomics documentation to call out
unconditional functions which are likely to loop around cmpxchg()
internally. We already have things like atomic_add_unless() and
atomic_dec_if_positive() but their conditional nature makes it much less
surprising than something like atomic_and_or() imo.

Will

Re: [RFC PATCH v1 1/5] locking/atomic: Implement atomic_fetch_and_or

[Waiman Long]

On 7/29/21 6:18 AM, hev wrote:
> Hi, Will,
>
> On Thu, Jul 29, 2021 at 5:39 PM Will Deacon <will@kernel.org> wrote:
>> On Wed, Jul 28, 2021 at 07:48:22PM +0800, Rui Wang wrote:
>>> From: wangrui <wangrui@loongson.cn>
>>>
>>> This patch introduce a new atomic primitive 'and_or', It may be have three
>>> types of implemeations:
>>>
>>>   * The generic implementation is based on arch_cmpxchg.
>>>   * The hardware supports atomic 'and_or' of single instruction.
>> Do any architectures actually support this instruction?
> No, I'm not sure now.
>
>> On arm64, we can clear arbitrary bits and we can set arbitrary bits, but we
>> can't combine the two in a fashion which provides atomicity and
>> forward-progress guarantees.
>>
>> Please can you explain how this new primitive will be used, in case there's
>> an alternative way of doing it which maps better to what CPUs can actually
>> do?
> I think we can easily exchange arbitrary bits of a machine word with atomic
> andnot_or/and_or. Otherwise, we can only use xchg8/16 to do it. It depends on
> hardware support, and the key point is that the bits to be exchanged
> must be in the
> same sub-word. qspinlock adjusted memory layout for this reason, and waste some
> bits(_Q_PENDING_BITS == 8).

It is not actually a waste of bits. With _Q_PENDING_BITS==8, more 
optimized code can be used for pending bit processing. It is only in the 
rare case that NR_CPUS >= 16k - 1 that we have to fall back to 
_Q_PENDING_BITS==1. In fact, that should be the only condition that will 
make _Q_PENDING_BITS=1.

Cheers,
Longman

Re: [RFC PATCH v1 1/5] locking/atomic: Implement atomic_fetch_and_or

[hev]

Hi,

On Sat, Jul 31, 2021 at 2:40 AM Waiman Long <llong@redhat.com> wrote:
>
> On 7/29/21 6:18 AM, hev wrote:
> > Hi, Will,
> >
> > On Thu, Jul 29, 2021 at 5:39 PM Will Deacon <will@kernel.org> wrote:
> >> On Wed, Jul 28, 2021 at 07:48:22PM +0800, Rui Wang wrote:
> >>> From: wangrui <wangrui@loongson.cn>
> >>>
> >>> This patch introduce a new atomic primitive 'and_or', It may be have three
> >>> types of implemeations:
> >>>
> >>>   * The generic implementation is based on arch_cmpxchg.
> >>>   * The hardware supports atomic 'and_or' of single instruction.
> >> Do any architectures actually support this instruction?
> > No, I'm not sure now.
> >
> >> On arm64, we can clear arbitrary bits and we can set arbitrary bits, but we
> >> can't combine the two in a fashion which provides atomicity and
> >> forward-progress guarantees.
> >>
> >> Please can you explain how this new primitive will be used, in case there's
> >> an alternative way of doing it which maps better to what CPUs can actually
> >> do?
> > I think we can easily exchange arbitrary bits of a machine word with atomic
> > andnot_or/and_or. Otherwise, we can only use xchg8/16 to do it. It depends on
> > hardware support, and the key point is that the bits to be exchanged
> > must be in the
> > same sub-word. qspinlock adjusted memory layout for this reason, and waste some
> > bits(_Q_PENDING_BITS == 8).
>
> It is not actually a waste of bits. With _Q_PENDING_BITS==8, more
> optimized code can be used for pending bit processing. It is only in the
> rare case that NR_CPUS >= 16k - 1 that we have to fall back to
> _Q_PENDING_BITS==1. In fact, that should be the only condition that will
> make _Q_PENDING_BITS=1.

Yes, you are right. The memory layout is adjusted so that
locked/pending and tail do ont share bits, so normal store
instructions can be used to clear_pending and
clear_pending_set_locked. It's faster than atomic.

Regards,
Rui

Re: [RFC PATCH v1 1/5] locking/atomic: Implement atomic_fetch_and_or

[Huacai Chen]

Hi, all,

On Sat, Jul 31, 2021 at 2:40 AM Waiman Long <llong@redhat.com> wrote:
>
> On 7/29/21 6:18 AM, hev wrote:
> > Hi, Will,
> >
> > On Thu, Jul 29, 2021 at 5:39 PM Will Deacon <will@kernel.org> wrote:
> >> On Wed, Jul 28, 2021 at 07:48:22PM +0800, Rui Wang wrote:
> >>> From: wangrui <wangrui@loongson.cn>
> >>>
> >>> This patch introduce a new atomic primitive 'and_or', It may be have three
> >>> types of implemeations:
> >>>
> >>>   * The generic implementation is based on arch_cmpxchg.
> >>>   * The hardware supports atomic 'and_or' of single instruction.
> >> Do any architectures actually support this instruction?
> > No, I'm not sure now.
> >
> >> On arm64, we can clear arbitrary bits and we can set arbitrary bits, but we
> >> can't combine the two in a fashion which provides atomicity and
> >> forward-progress guarantees.
> >>
> >> Please can you explain how this new primitive will be used, in case there's
> >> an alternative way of doing it which maps better to what CPUs can actually
> >> do?
> > I think we can easily exchange arbitrary bits of a machine word with atomic
> > andnot_or/and_or. Otherwise, we can only use xchg8/16 to do it. It depends on
> > hardware support, and the key point is that the bits to be exchanged
> > must be in the
> > same sub-word. qspinlock adjusted memory layout for this reason, and waste some
> > bits(_Q_PENDING_BITS == 8).
>
> It is not actually a waste of bits. With _Q_PENDING_BITS==8, more
> optimized code can be used for pending bit processing. It is only in the
> rare case that NR_CPUS >= 16k - 1 that we have to fall back to
> _Q_PENDING_BITS==1. In fact, that should be the only condition that will
> make _Q_PENDING_BITS=1.
Our original goal is to let LoongArch (and CSKY, RISC-V, etc) can use
qspinlock, but these archs lack sub-word xchg/cmpxchg. Arnd suggests
we not use qspinlock, but LoongArch has large SMP (and NUMA) so we
need it. Peter suggests we implement atomic_fetch_and_or, but it seems
not agreed by everyone. So, I think we can only fix the
badly-implemented xchg_small() for MIPS and LoongArch.

Huacai
>
> Cheers,
> Longman
>

Re: [RFC PATCH v1 1/5] locking/atomic: Implement atomic_fetch_and_or

[Geert Uytterhoeven]

Hi Rui,

On Wed, Jul 28, 2021 at 1:50 PM Rui Wang <wangrui@loongson.cn> wrote:
> From: wangrui <wangrui@loongson.cn>
>
> This patch introduce a new atomic primitive 'and_or', It may be have three
> types of implemeations:
>
>  * The generic implementation is based on arch_cmpxchg.
>  * The hardware supports atomic 'and_or' of single instruction.
>  * The hardware supports LL/SC style atomic operations:
>
>    1:  ll  v1, mem
>        and t1, v1, arg1
>        or  t1, t1, arg2
>        sc  t1, mem
>        beq t1, 0, 1b
>
> Now that all the architectures have implemented it.
>
> Signed-by-off: Rui Wang <wangrui@loongson.cn>
> Signed-by-off: hev <r@hev.cc>

> --- a/arch/m68k/include/asm/atomic.h
> +++ b/arch/m68k/include/asm/atomic.h
> @@ -67,6 +67,22 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t *v)         \
>         return tmp;                                                     \
>  }
>
> +#define ATOMIC_FETCH_OP2(op, c_op1, c_op2, asm_op1, asm_op2)           \
> +static inline int arch_atomic_fetch_##op(int i, int j, atomic_t *v)    \
> +{                                                                      \
> +       int t, tmp;                                                     \
> +                                                                       \
> +       __asm__ __volatile__(                                           \
> +                       "1:     movel %2,%1\n"                          \
> +                       "       " #asm_op1 "l %3,%1\n"                  \
> +                       "       " #asm_op2 "l %4,%1\n"                  \
> +                       "       casl %2,%1,%0\n"                        \
> +                       "       jne 1b"                                 \
> +                       : "+m" (*v), "=&d" (t), "=&d" (tmp)             \
> +                       : "g" (i), "g" (j), "2" (arch_atomic_read(v))); \

"di" (i), "di" (j)

cfr. "[PATCH v2] m68k: Fix asm register constraints for atomic ops"
https://lore.kernel.org/linux-m68k/20210809112903.3898660-1-geert@linux-m68k.org/

> +       return tmp;                                                     \
> +}
> +
>  #else
>
>  #define ATOMIC_OP_RETURN(op, c_op, asm_op)                             \

Gr{oetje,eeting}s,

                        Geert

-- 
Geert Uytterhoeven -- There's lots of Linux beyond ia32 -- geert@linux-m68k.org

In personal conversations with technical people, I call myself a hacker. But
when I'm talking to journalists I just say "programmer" or something like that.
                                -- Linus Torvalds