[kvm-unit-tests PATCH v2 0/8] Some cleanup patches

[kvm-unit-tests PATCH v2 0/8] Some cleanup patches

[Marc Hartmayer]

This patch series should be applied on top of Janosch's patch series
([kvm-unit-tests PATCH v2 1/7] s390x: Cleanup flat.lds).

Changelog:
v1->v2:
 + worked in comments from Claudio and Janosch
 + added r-b's from Janosch
 + added a new patch: `s390x/Makefile: add an extra `%.aux.o` target`

Marc Hartmayer (8):
  .gitignore: ignore `s390x/comm.key` file
  s390x/Makefile: simplify `%.hdr` target rules
  s390x/Makefile: fix `*.gbin` target dependencies
  s390x/Makefile: refactor CPPFLAGS
  s390x: use C pre-processor for linker script generation
  s390x: define a macro for the stack frame size
  lib/linux/const.h: test for `__ASSEMBLER__` as well
  s390x/Makefile: add an extra `%.aux.o` target

 .gitignore                                  |  2 ++
 lib/linux/const.h                           |  2 +-
 lib/s390x/asm-offsets.c                     |  1 +
 s390x/Makefile                              | 33 +++++++++++----------
 s390x/cstart64.S                            |  2 +-
 s390x/{flat.lds => flat.lds.S}              |  4 ++-
 s390x/gs.c                                  |  5 ++--
 s390x/macros.S                              |  4 +--
 s390x/snippets/asm/{flat.lds => flat.lds.S} |  0
 s390x/snippets/c/{flat.lds => flat.lds.S}   |  6 ++--
 10 files changed, 35 insertions(+), 24 deletions(-)
 rename s390x/{flat.lds => flat.lds.S} (93%)
 rename s390x/snippets/asm/{flat.lds => flat.lds.S} (100%)
 rename s390x/snippets/c/{flat.lds => flat.lds.S} (88%)

-- 
2.34.1

[kvm-unit-tests PATCH v2 1/8] .gitignore: ignore `s390x/comm.key` file

[Marc Hartmayer]

Ignore the Secure Execution Customer Communication Key file.

Signed-off-by: Marc Hartmayer <mhartmay@linux.ibm.com>
Reviewed-by: Janosch Frank <frankja@linux.ibm.com>
---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 33529b65a7e3..601822d67325 100644
--- a/.gitignore
+++ b/.gitignore
@@ -28,5 +28,6 @@ cscope.*
 /s390x/*.bin
 /s390x/snippets/*/*.gbin
 /efi-tests/*
+/s390x/comm.key
 /s390x/snippets/*/*.hdr
 /s390x/snippets/*/*.*obj
-- 
2.34.1

[kvm-unit-tests PATCH v2 2/8] s390x/Makefile: simplify `%.hdr` target rules

[Marc Hartmayer]

Merge the two Makefile target rules `$(SNIPPET_DIR)/asm/%.hdr` and
`$(SNIPPET_DIR)/c/%.hdr` into one target rule.

Signed-off-by: Marc Hartmayer <mhartmay@linux.ibm.com>
Reviewed-by: Janosch Frank <frankja@linux.ibm.com>
---
 s390x/Makefile | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/s390x/Makefile b/s390x/Makefile
index 97a616111680..660ff06f1e7c 100644
--- a/s390x/Makefile
+++ b/s390x/Makefile
@@ -145,10 +145,7 @@ $(SNIPPET_DIR)/c/%.gbin: $(SNIPPET_DIR)/c/%.o $(snippet_lib) $(FLATLIBS)
 	$(OBJCOPY) -O binary -j ".rodata" -j ".lowcore" -j ".text" -j ".data" -j ".bss" --set-section-flags .bss=alloc,load,contents $@ $@
 	truncate -s '%4096' $@
 
-$(SNIPPET_DIR)/asm/%.hdr: $(SNIPPET_DIR)/asm/%.gbin $(HOST_KEY_DOCUMENT)
-	$(GEN_SE_HEADER) -k $(HOST_KEY_DOCUMENT) -c $<,0x0,0x00000000000000420000000000000000 --psw-addr 0x4000 -o $@
-
-$(SNIPPET_DIR)/c/%.hdr: $(SNIPPET_DIR)/c/%.gbin $(HOST_KEY_DOCUMENT)
+%.hdr: %.gbin $(HOST_KEY_DOCUMENT)
 	$(GEN_SE_HEADER) -k $(HOST_KEY_DOCUMENT) -c $<,0x0,0x00000000000000420000000000000000 --psw-addr 0x4000 -o $@
 
 .SECONDARY:
-- 
2.34.1

[kvm-unit-tests PATCH v2 3/8] s390x/Makefile: fix `*.gbin` target dependencies

[Marc Hartmayer]

If the linker scripts change, then the .gbin binaries must be rebuilt.
While at it, replace `$(SRCDIR)/s390x/snippets` with `$(SNIPPET_DIR)`
for these Makefile rules.

Signed-off-by: Marc Hartmayer <mhartmay@linux.ibm.com>
---
 s390x/Makefile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/s390x/Makefile b/s390x/Makefile
index 660ff06f1e7c..71e6563bbb61 100644
--- a/s390x/Makefile
+++ b/s390x/Makefile
@@ -135,13 +135,13 @@ $(SNIPPET_DIR)/asm/%.o: $(SNIPPET_DIR)/asm/%.S $(asm-offsets)
 $(SNIPPET_DIR)/c/%.o: $(SNIPPET_DIR)/c/%.c $(asm-offsets)
 	$(CC) $(CFLAGS) -c -nostdlib -o $@ $<
 
-$(SNIPPET_DIR)/asm/%.gbin: $(SNIPPET_DIR)/asm/%.o
-	$(CC) $(LDFLAGS) -o $@ -T $(SRCDIR)/s390x/snippets/asm/flat.lds $<
+$(SNIPPET_DIR)/asm/%.gbin: $(SNIPPET_DIR)/asm/%.o $(SNIPPET_DIR)/asm/flat.lds
+	$(CC) $(LDFLAGS) -o $@ -T $(SNIPPET_DIR)/asm/flat.lds $<
 	$(OBJCOPY) -O binary -j ".rodata" -j ".lowcore" -j ".text" -j ".data" -j ".bss" --set-section-flags .bss=alloc,load,contents $@ $@
 	truncate -s '%4096' $@
 
-$(SNIPPET_DIR)/c/%.gbin: $(SNIPPET_DIR)/c/%.o $(snippet_lib) $(FLATLIBS)
-	$(CC) $(LDFLAGS) -o $@ -T $(SRCDIR)/s390x/snippets/c/flat.lds $< $(snippet_lib) $(FLATLIBS)
+$(SNIPPET_DIR)/c/%.gbin: $(SNIPPET_DIR)/c/%.o $(snippet_lib) $(FLATLIBS) $(SNIPPET_DIR)/c/flat.lds
+	$(CC) $(LDFLAGS) -o $@ -T $(SNIPPET_DIR)/c/flat.lds $< $(snippet_lib) $(FLATLIBS)
 	$(OBJCOPY) -O binary -j ".rodata" -j ".lowcore" -j ".text" -j ".data" -j ".bss" --set-section-flags .bss=alloc,load,contents $@ $@
 	truncate -s '%4096' $@
 
-- 
2.34.1

[kvm-unit-tests PATCH v2 4/8] s390x/Makefile: refactor CPPFLAGS

[Marc Hartmayer]

This change makes it easier to reuse them. While at it, remove `lib`
include path since it seems to be unused.

Signed-off-by: Marc Hartmayer <mhartmay@linux.ibm.com>
---
 s390x/Makefile | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/s390x/Makefile b/s390x/Makefile
index 71e6563bbb61..8719f0c837cf 100644
--- a/s390x/Makefile
+++ b/s390x/Makefile
@@ -63,9 +63,12 @@ test_cases: $(tests)
 test_cases_binary: $(tests_binary)
 test_cases_pv: $(tests_pv_binary)
 
+INCLUDE_PATHS = $(SRCDIR)/lib $(SRCDIR)/lib/s390x
+CPPFLAGS = $(addprefix -I,$(INCLUDE_PATHS))
+
 CFLAGS += -std=gnu99
 CFLAGS += -ffreestanding
-CFLAGS += -I $(SRCDIR)/lib -I $(SRCDIR)/lib/s390x -I lib
+CFLAGS += $(CPPFLAGS)
 CFLAGS += -O2
 CFLAGS += -march=zEC12
 CFLAGS += -mbackchain
-- 
2.34.1

[kvm-unit-tests PATCH v2 5/8] s390x: use C pre-processor for linker script generation

[Marc Hartmayer]

Use the C pre-processor for the linker script generation. For example,
this enables us the use of constants in the "linker scripts" `*.lds.S`.

Signed-off-by: Marc Hartmayer <mhartmay@linux.ibm.com>
---
 .gitignore                                  | 1 +
 s390x/Makefile                              | 6 ++++--
 s390x/{flat.lds => flat.lds.S}              | 0
 s390x/snippets/asm/{flat.lds => flat.lds.S} | 0
 s390x/snippets/c/{flat.lds => flat.lds.S}   | 0
 5 files changed, 5 insertions(+), 2 deletions(-)
 rename s390x/{flat.lds => flat.lds.S} (100%)
 rename s390x/snippets/asm/{flat.lds => flat.lds.S} (100%)
 rename s390x/snippets/c/{flat.lds => flat.lds.S} (100%)

diff --git a/.gitignore b/.gitignore
index 601822d67325..29f352c5ceb6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -31,3 +31,4 @@ cscope.*
 /s390x/comm.key
 /s390x/snippets/*/*.hdr
 /s390x/snippets/*/*.*obj
+/s390x/**/*.lds
diff --git a/s390x/Makefile b/s390x/Makefile
index 8719f0c837cf..44ccca8102d6 100644
--- a/s390x/Makefile
+++ b/s390x/Makefile
@@ -76,7 +76,7 @@ CFLAGS += -fno-delete-null-pointer-checks
 LDFLAGS += -nostdlib -Wl,--build-id=none
 
 # We want to keep intermediate files
-.PRECIOUS: %.o
+.PRECIOUS: %.o %.lds
 
 asm-offsets = lib/$(ARCH)/asm-offsets.h
 include $(SRCDIR)/scripts/asm-offsets.mak
@@ -159,6 +159,8 @@ $(SNIPPET_DIR)/c/%.gbin: $(SNIPPET_DIR)/c/%.o $(snippet_lib) $(FLATLIBS) $(SNIPP
 %.hdr.obj: %.hdr
 	$(OBJCOPY) -I binary -O elf64-s390 -B "s390:64-bit" $< $@
 
+%.lds: %.lds.S
+	$(CPP) $(autodepend-flags) $(CPPFLAGS) -P -C -o $@ $<
 
 .SECONDEXPANSION:
 %.elf: $(FLATLIBS) $(asmlib) $(SRCDIR)/s390x/flat.lds $$(snippets-obj) $$(snippet-hdr-obj) %.o
@@ -211,7 +213,7 @@ $(snippet_asmlib): $$(patsubst %.o,%.S,$$@) $(asm-offsets)
 
 
 arch_clean: asm_offsets_clean
-	$(RM) $(TEST_DIR)/*.{o,elf,bin} $(SNIPPET_DIR)/*/*.{o,elf,*bin,*obj,hdr} $(SNIPPET_DIR)/asm/.*.d $(TEST_DIR)/.*.d lib/s390x/.*.d $(comm-key)
+	$(RM) $(TEST_DIR)/*.{o,elf,bin,lds} $(SNIPPET_DIR)/*/*.{o,elf,*bin,*obj,hdr,lds} $(SNIPPET_DIR)/asm/.*.d $(TEST_DIR)/.*.d lib/s390x/.*.d $(comm-key)
 
 generated-files = $(asm-offsets)
 $(tests:.elf=.o) $(asmlib) $(cflatobjs): $(generated-files)
diff --git a/s390x/flat.lds b/s390x/flat.lds.S
similarity index 100%
rename from s390x/flat.lds
rename to s390x/flat.lds.S
diff --git a/s390x/snippets/asm/flat.lds b/s390x/snippets/asm/flat.lds.S
similarity index 100%
rename from s390x/snippets/asm/flat.lds
rename to s390x/snippets/asm/flat.lds.S
diff --git a/s390x/snippets/c/flat.lds b/s390x/snippets/c/flat.lds.S
similarity index 100%
rename from s390x/snippets/c/flat.lds
rename to s390x/snippets/c/flat.lds.S
-- 
2.34.1

[kvm-unit-tests PATCH v2 6/8] s390x: define a macro for the stack frame size

[Marc Hartmayer]

Define and use a macro for the stack frame size.

Signed-off-by: Marc Hartmayer <mhartmay@linux.ibm.com>
---
 lib/s390x/asm-offsets.c     | 1 +
 s390x/Makefile              | 2 +-
 s390x/cstart64.S            | 2 +-
 s390x/flat.lds.S            | 4 +++-
 s390x/gs.c                  | 5 +++--
 s390x/macros.S              | 4 ++--
 s390x/snippets/c/flat.lds.S | 6 ++++--
 7 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/lib/s390x/asm-offsets.c b/lib/s390x/asm-offsets.c
index f612f3277a95..188dd2e51181 100644
--- a/lib/s390x/asm-offsets.c
+++ b/lib/s390x/asm-offsets.c
@@ -87,6 +87,7 @@ int main(void)
 	OFFSET(STACK_FRAME_INT_GRS0, stack_frame_int, grs0);
 	OFFSET(STACK_FRAME_INT_GRS1, stack_frame_int, grs1);
 	DEFINE(STACK_FRAME_INT_SIZE, sizeof(struct stack_frame_int));
+	DEFINE(STACK_FRAME_SIZE, sizeof(struct stack_frame));
 
 	return 0;
 }
diff --git a/s390x/Makefile b/s390x/Makefile
index 44ccca8102d6..9a8e2af1b2be 100644
--- a/s390x/Makefile
+++ b/s390x/Makefile
@@ -159,7 +159,7 @@ $(SNIPPET_DIR)/c/%.gbin: $(SNIPPET_DIR)/c/%.o $(snippet_lib) $(FLATLIBS) $(SNIPP
 %.hdr.obj: %.hdr
 	$(OBJCOPY) -I binary -O elf64-s390 -B "s390:64-bit" $< $@
 
-%.lds: %.lds.S
+%.lds: %.lds.S $(asm-offsets)
 	$(CPP) $(autodepend-flags) $(CPPFLAGS) -P -C -o $@ $<
 
 .SECONDEXPANSION:
diff --git a/s390x/cstart64.S b/s390x/cstart64.S
index 6f83da2a6c0a..468ace3ea4df 100644
--- a/s390x/cstart64.S
+++ b/s390x/cstart64.S
@@ -38,7 +38,7 @@ start:
 	/* setup stack */
 	larl	%r15, stackptr
 	/* Clear first stack frame */
-	xc      0(160,%r15), 0(%r15)
+	xc      0(STACK_FRAME_SIZE,%r15), 0(%r15)
 	/* setup initial PSW mask + control registers*/
 	larl	%r1, initial_psw
 	lpswe	0(%r1)
diff --git a/s390x/flat.lds.S b/s390x/flat.lds.S
index 952f6cd457ed..0cb7e383cc76 100644
--- a/s390x/flat.lds.S
+++ b/s390x/flat.lds.S
@@ -1,3 +1,5 @@
+#include <asm/asm-offsets.h>
+
 SECTIONS
 {
 	.lowcore : {
@@ -44,6 +46,6 @@ SECTIONS
 	/*
 	 * stackptr set with initial stack frame preallocated
 	 */
-	stackptr = . - 160;
+	stackptr = . - STACK_FRAME_SIZE;
 	stacktop = .;
 }
diff --git a/s390x/gs.c b/s390x/gs.c
index 4993eb8f43a9..2c2b972d7e65 100644
--- a/s390x/gs.c
+++ b/s390x/gs.c
@@ -9,6 +9,7 @@
  *    Janosch Frank <frankja@linux.ibm.com>
  */
 #include <libcflat.h>
+#include <asm/asm-offsets.h>
 #include <asm/page.h>
 #include <asm/facility.h>
 #include <asm/interrupt.h>
@@ -41,8 +42,8 @@ extern void gs_handler_asm(void);
 	    "	    aghi    %r15,-320\n" 		/* Allocate stack frame */
 	    "	    stmg    %r0,%r13,192(%r15)\n" 	/* Store regs to save area */
 	    "	    stg	    %r14,312(%r15)\n"
-	    "	    la	    %r2,160(%r15)\n" 		/* Store gscb address in this_cb */
-	    "	    .insn   rxy,0xe30000000049,0,160(%r15)\n" /* stgsc */
+	    "	    la	    %r2," xstr(STACK_FRAME_SIZE) "(%r15) \n" 		/* Store gscb address in this_cb */
+	    "	    .insn   rxy,0xe30000000049,0," xstr(STACK_FRAME_SIZE) "(%r15)\n" /* stgsc */
 	    "	    lg	    %r14,24(%r2)\n" 		/* Get GSEPLA from GSCB*/
 	    "	    lg	    %r14,40(%r14)\n" 		/* Get GSERA from GSEPL*/
 	    "	    stg	    %r14,304(%r15)\n" 		/* Store GSERA in r14 of reg save area */
diff --git a/s390x/macros.S b/s390x/macros.S
index 13cff299488f..e2a56a366c70 100644
--- a/s390x/macros.S
+++ b/s390x/macros.S
@@ -21,14 +21,14 @@
 	/* Save the stack address in GR2 which is the first function argument */
 	lgr     %r2, %r15
 	/* Allocate stack space for called C function, as specified in s390 ELF ABI */
-	slgfi   %r15, 160
+	slgfi   %r15, STACK_FRAME_SIZE
 	/*
 	 * Save the address of the interrupt stack into the back chain
 	 * of the called function.
 	 */
 	stg     %r2, STACK_FRAME_INT_BACKCHAIN(%r15)
 	brasl	%r14, \c_func
-	algfi   %r15, 160
+	algfi   %r15, STACK_FRAME_SIZE
 	RESTORE_REGS_STACK
 	lpswe	\old_psw
 	.endm
diff --git a/s390x/snippets/c/flat.lds.S b/s390x/snippets/c/flat.lds.S
index 9e5eb66bec23..468b5f1eebe8 100644
--- a/s390x/snippets/c/flat.lds.S
+++ b/s390x/snippets/c/flat.lds.S
@@ -1,3 +1,5 @@
+#include <asm/asm-offsets.h>
+
 SECTIONS
 {
 	.lowcore : {
@@ -18,9 +20,9 @@ SECTIONS
 	. = 0x4000;
 	/*
 	 * The stack grows down from 0x4000 to 0x2000, we pre-allocoate
-	 * a frame via the -160.
+	 * a frame via the -STACK_FRAME_SIZE.
 	 */
-	stackptr = . - 160;
+	stackptr = . - STACK_FRAME_SIZE;
 	stacktop = .;
 	/* Start text 0x4000 */
 	.text : {
-- 
2.34.1

[kvm-unit-tests PATCH v2 7/8] lib/linux/const.h: test for `__ASSEMBLER__` as well

[Marc Hartmayer]

On s390x we're using the preprocessor for generating our linker scripts
out of assembly files. The macro `__ASSEMBLER__` is defined with value 1
when preprocessing assembly language using gcc. [1] Therefore, let's
check for the macro `__ASSEMBLER__` in `lib/linux/const.h` as well. Thus
we can use macros that makes use of the `_AC` or `_AT` macro in the
"linker scripts".

[1] https://gcc.gnu.org/onlinedocs/cpp/Standard-Predefined-Macros.html

Signed-off-by: Marc Hartmayer <mhartmay@linux.ibm.com>
---
 lib/linux/const.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/linux/const.h b/lib/linux/const.h
index c872bfd25e13..be114dc4a553 100644
--- a/lib/linux/const.h
+++ b/lib/linux/const.h
@@ -12,7 +12,7 @@
  * leave it unchanged in asm.
  */
 
-#ifdef __ASSEMBLY__
+#if defined(__ASSEMBLY__) || defined(__ASSEMBLER__)
 #define _AC(X,Y)	X
 #define _AT(T,X)	X
 #else
-- 
2.34.1

[kvm-unit-tests PATCH v2 8/8] s390x/Makefile: add an extra `%.aux.o` target

[Marc Hartmayer]

It's unusual to create multiple files in one target rule, therefore
let's create an extra target for `%.aux.o`. As a side effect, this
change fixes the dependency tracking of the prerequisites of `.aux.o`
(lib/auxinfo.c wasn't listed before).

Signed-off-by: Marc Hartmayer <mhartmay@linux.ibm.com>
---
 s390x/Makefile | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/s390x/Makefile b/s390x/Makefile
index 9a8e2af1b2be..6fa62416c0e9 100644
--- a/s390x/Makefile
+++ b/s390x/Makefile
@@ -162,13 +162,14 @@ $(SNIPPET_DIR)/c/%.gbin: $(SNIPPET_DIR)/c/%.o $(snippet_lib) $(FLATLIBS) $(SNIPP
 %.lds: %.lds.S $(asm-offsets)
 	$(CPP) $(autodepend-flags) $(CPPFLAGS) -P -C -o $@ $<
 
+%.aux.o: $(SRCDIR)/lib/auxinfo.c
+	$(CC) $(CFLAGS) -c -o $@ $^ -DPROGNAME=\"$(@:.aux.o=.elf)\"
+
 .SECONDEXPANSION:
-%.elf: $(FLATLIBS) $(asmlib) $(SRCDIR)/s390x/flat.lds $$(snippets-obj) $$(snippet-hdr-obj) %.o
-	$(CC) $(CFLAGS) -c -o $(@:.elf=.aux.o) $(SRCDIR)/lib/auxinfo.c -DPROGNAME=\"$@\"
+%.elf: $(FLATLIBS) $(asmlib) $(SRCDIR)/s390x/flat.lds $$(snippets-obj) $$(snippet-hdr-obj) %.o %.aux.o
 	@$(CC) $(LDFLAGS) -o $@ -T $(SRCDIR)/s390x/flat.lds \
-		$(filter %.o, $^) $(FLATLIBS) $(snippets-obj) $(snippet-hdr-obj) $(@:.elf=.aux.o) || \
+		$(filter %.o, $^) $(FLATLIBS) $(snippets-obj) $(snippet-hdr-obj) || \
 		{ echo "Failure probably caused by missing definition of gen-se-header executable"; exit 1; }
-	$(RM) $(@:.elf=.aux.o)
 	@chmod a-x $@
 
 # Secure Execution Customer Communication Key file
-- 
2.34.1

Re: [kvm-unit-tests PATCH v2 5/8] s390x: use C pre-processor for linker script generation

[Marc Hartmayer]

Marc Hartmayer <mhartmay@linux.ibm.com> writes:

> Use the C pre-processor for the linker script generation. For example,
> this enables us the use of constants in the "linker scripts" `*.lds.S`.
>

Sorry, I forgot to change the commit message (Claudio’s comment):

“s390x: use preprocessor for linker script generation

The old `.lds` linker scripts are renamed to `.lds.S` and the actual
.lds` scripts are generated by the assembler preprocessor. This change
allows us to use constants defined by macros in the `.lds.S` files.”

-- 
Kind regards / Beste Grüße
   Marc Hartmayer

IBM Deutschland Research & Development GmbH
Vorsitzender des Aufsichtsrats: Gregor Pillen 
Geschäftsführung: David Faller
Sitz der Gesellschaft: Böblingen
Registergericht: Amtsgericht Stuttgart, HRB 243294

Re: [kvm-unit-tests PATCH v2 8/8] s390x/Makefile: add an extra `%.aux.o` target

[Marc Hartmayer]

Marc Hartmayer <mhartmay@linux.ibm.com> writes:

> It's unusual to create multiple files in one target rule, therefore
> let's create an extra target for `%.aux.o`. As a side effect, this
> change fixes the dependency tracking of the prerequisites of `.aux.o`
> (lib/auxinfo.c wasn't listed before).
>
> Signed-off-by: Marc Hartmayer <mhartmay@linux.ibm.com>
> ---
>  s390x/Makefile | 9 +++++----
>  1 file changed, 5 insertions(+), 4 deletions(-)
>
> diff --git a/s390x/Makefile b/s390x/Makefile
> index 9a8e2af1b2be..6fa62416c0e9 100644
> --- a/s390x/Makefile
> +++ b/s390x/Makefile
> @@ -162,13 +162,14 @@ $(SNIPPET_DIR)/c/%.gbin: $(SNIPPET_DIR)/c/%.o $(snippet_lib) $(FLATLIBS) $(SNIPP
>  %.lds: %.lds.S $(asm-offsets)
>  	$(CPP) $(autodepend-flags) $(CPPFLAGS) -P -C -o $@ $<
>  
> +%.aux.o: $(SRCDIR)/lib/auxinfo.c
> +	$(CC) $(CFLAGS) -c -o $@ $^ -DPROGNAME=\"$(@:.aux.o=.elf)\"
> +
>  .SECONDEXPANSION:
> -%.elf: $(FLATLIBS) $(asmlib) $(SRCDIR)/s390x/flat.lds $$(snippets-obj) $$(snippet-hdr-obj) %.o
> -	$(CC) $(CFLAGS) -c -o $(@:.elf=.aux.o) $(SRCDIR)/lib/auxinfo.c -DPROGNAME=\"$@\"
> +%.elf: $(FLATLIBS) $(asmlib) $(SRCDIR)/s390x/flat.lds $$(snippets-obj) $$(snippet-hdr-obj) %.o %.aux.o
>  	@$(CC) $(LDFLAGS) -o $@ -T $(SRCDIR)/s390x/flat.lds \
> -		$(filter %.o, $^) $(FLATLIBS) $(snippets-obj) $(snippet-hdr-obj) $(@:.elf=.aux.o) || \
> +		$(filter %.o, $^) $(FLATLIBS) $(snippets-obj) $(snippet-hdr-obj) || \
>  		{ echo "Failure probably caused by missing definition of gen-se-header executable"; exit 1; }
> -	$(RM) $(@:.elf=.aux.o)
>  	@chmod a-x $@
>  
>  # Secure Execution Customer Communication Key file
> -- 
> 2.34.1
>

I don’t have a strong opinion about this patch, but we could either
change the ARM & Power Makefiles as well or just drop this patch.

-- 
Kind regards / Beste Grüße
   Marc Hartmayer

IBM Deutschland Research & Development GmbH
Vorsitzender des Aufsichtsrats: Gregor Pillen 
Geschäftsführung: David Faller
Sitz der Gesellschaft: Böblingen
Registergericht: Amtsgericht Stuttgart, HRB 243294

Re: [kvm-unit-tests PATCH v2 1/8] .gitignore: ignore `s390x/comm.key` file

[Nico Boehr]

Quoting Marc Hartmayer (2023-01-19 12:40:38)
> Ignore the Secure Execution Customer Communication Key file.
> 
> Signed-off-by: Marc Hartmayer <mhartmay@linux.ibm.com>
> Reviewed-by: Janosch Frank <frankja@linux.ibm.com>

good find, thanks.

Reviewed-by: Nico Boehr <nrb@linux.ibm.com>

Re: [kvm-unit-tests PATCH v2 3/8] s390x/Makefile: fix `*.gbin` target dependencies

[Janosch Frank]

On 1/19/23 12:40, Marc Hartmayer wrote:
> If the linker scripts change, then the .gbin binaries must be rebuilt.
> While at it, replace `$(SRCDIR)/s390x/snippets` with `$(SNIPPET_DIR)`
> for these Makefile rules.
> 
> Signed-off-by: Marc Hartmayer <mhartmay@linux.ibm.com>

Reviewed-by: Janosch Frank <frankja@linux.ibm.com>

> ---
>   s390x/Makefile | 8 ++++----
>   1 file changed, 4 insertions(+), 4 deletions(-)
> 
> diff --git a/s390x/Makefile b/s390x/Makefile
> index 660ff06f1e7c..71e6563bbb61 100644
> --- a/s390x/Makefile
> +++ b/s390x/Makefile
> @@ -135,13 +135,13 @@ $(SNIPPET_DIR)/asm/%.o: $(SNIPPET_DIR)/asm/%.S $(asm-offsets)
>   $(SNIPPET_DIR)/c/%.o: $(SNIPPET_DIR)/c/%.c $(asm-offsets)
>   	$(CC) $(CFLAGS) -c -nostdlib -o $@ $<
>   
> -$(SNIPPET_DIR)/asm/%.gbin: $(SNIPPET_DIR)/asm/%.o
> -	$(CC) $(LDFLAGS) -o $@ -T $(SRCDIR)/s390x/snippets/asm/flat.lds $<
> +$(SNIPPET_DIR)/asm/%.gbin: $(SNIPPET_DIR)/asm/%.o $(SNIPPET_DIR)/asm/flat.lds
> +	$(CC) $(LDFLAGS) -o $@ -T $(SNIPPET_DIR)/asm/flat.lds $<
>   	$(OBJCOPY) -O binary -j ".rodata" -j ".lowcore" -j ".text" -j ".data" -j ".bss" --set-section-flags .bss=alloc,load,contents $@ $@
>   	truncate -s '%4096' $@
>   
> -$(SNIPPET_DIR)/c/%.gbin: $(SNIPPET_DIR)/c/%.o $(snippet_lib) $(FLATLIBS)
> -	$(CC) $(LDFLAGS) -o $@ -T $(SRCDIR)/s390x/snippets/c/flat.lds $< $(snippet_lib) $(FLATLIBS)
> +$(SNIPPET_DIR)/c/%.gbin: $(SNIPPET_DIR)/c/%.o $(snippet_lib) $(FLATLIBS) $(SNIPPET_DIR)/c/flat.lds
> +	$(CC) $(LDFLAGS) -o $@ -T $(SNIPPET_DIR)/c/flat.lds $< $(snippet_lib) $(FLATLIBS)
>   	$(OBJCOPY) -O binary -j ".rodata" -j ".lowcore" -j ".text" -j ".data" -j ".bss" --set-section-flags .bss=alloc,load,contents $@ $@
>   	truncate -s '%4096' $@
>   

Re: [kvm-unit-tests PATCH v2 4/8] s390x/Makefile: refactor CPPFLAGS

[Janosch Frank]

On 1/19/23 12:40, Marc Hartmayer wrote:
> This change makes it easier to reuse them. While at it, remove `lib`
> include path since it seems to be unused.
> 
> Signed-off-by: Marc Hartmayer <mhartmay@linux.ibm.com>

Reviewed-by: Janosch Frank <frankja@linux.ibm.com>

> ---
>   s390x/Makefile | 5 ++++-
>   1 file changed, 4 insertions(+), 1 deletion(-)
> 
> diff --git a/s390x/Makefile b/s390x/Makefile
> index 71e6563bbb61..8719f0c837cf 100644
> --- a/s390x/Makefile
> +++ b/s390x/Makefile
> @@ -63,9 +63,12 @@ test_cases: $(tests)
>   test_cases_binary: $(tests_binary)
>   test_cases_pv: $(tests_pv_binary)
>   
> +INCLUDE_PATHS = $(SRCDIR)/lib $(SRCDIR)/lib/s390x
> +CPPFLAGS = $(addprefix -I,$(INCLUDE_PATHS))
> +
>   CFLAGS += -std=gnu99
>   CFLAGS += -ffreestanding
> -CFLAGS += -I $(SRCDIR)/lib -I $(SRCDIR)/lib/s390x -I lib
> +CFLAGS += $(CPPFLAGS)
>   CFLAGS += -O2
>   CFLAGS += -march=zEC12
>   CFLAGS += -mbackchain

Re: [kvm-unit-tests PATCH v2 7/8] lib/linux/const.h: test for `__ASSEMBLER__` as well

[Janosch Frank]

On 1/19/23 12:40, Marc Hartmayer wrote:
> On s390x we're using the preprocessor for generating our linker scripts
> out of assembly files. The macro `__ASSEMBLER__` is defined with value 1
> when preprocessing assembly language using gcc. [1] Therefore, let's
> check for the macro `__ASSEMBLER__` in `lib/linux/const.h` as well. Thus
> we can use macros that makes use of the `_AC` or `_AT` macro in the
> "linker scripts".
> 
> [1] https://gcc.gnu.org/onlinedocs/cpp/Standard-Predefined-Macros.html

TIL: the whole ASSEMBLY / ASSEMBLER problem is a rabbit hole in itself.

Reviewed-by: Janosch Frank <frankja@linux.ibm.com>

> 
> Signed-off-by: Marc Hartmayer <mhartmay@linux.ibm.com>
> ---
>   lib/linux/const.h | 2 +-
>   1 file changed, 1 insertion(+), 1 deletion(-)
> 
> diff --git a/lib/linux/const.h b/lib/linux/const.h
> index c872bfd25e13..be114dc4a553 100644
> --- a/lib/linux/const.h
> +++ b/lib/linux/const.h
> @@ -12,7 +12,7 @@
>    * leave it unchanged in asm.
>    */
>   
> -#ifdef __ASSEMBLY__
> +#if defined(__ASSEMBLY__) || defined(__ASSEMBLER__)
>   #define _AC(X,Y)	X
>   #define _AT(T,X)	X
>   #else

Re: [kvm-unit-tests PATCH v2 5/8] s390x: use C pre-processor for linker script generation

[Nina Schoetterl-Glausch]

On Thu, 2023-01-19 at 12:40 +0100, Marc Hartmayer wrote:
> Use the C pre-processor for the linker script generation. For example,
> this enables us the use of constants in the "linker scripts" `*.lds.S`.
> 
> Signed-off-by: Marc Hartmayer <mhartmay@linux.ibm.com>
> ---
>  .gitignore                                  | 1 +
>  s390x/Makefile                              | 6 ++++--
>  s390x/{flat.lds => flat.lds.S}              | 0
>  s390x/snippets/asm/{flat.lds => flat.lds.S} | 0
>  s390x/snippets/c/{flat.lds => flat.lds.S}   | 0
>  5 files changed, 5 insertions(+), 2 deletions(-)
>  rename s390x/{flat.lds => flat.lds.S} (100%)
>  rename s390x/snippets/asm/{flat.lds => flat.lds.S} (100%)
>  rename s390x/snippets/c/{flat.lds => flat.lds.S} (100%)
> 
> diff --git a/.gitignore b/.gitignore
> index 601822d67325..29f352c5ceb6 100644
> --- a/.gitignore
> +++ b/.gitignore
> @@ -31,3 +31,4 @@ cscope.*
>  /s390x/comm.key
>  /s390x/snippets/*/*.hdr
>  /s390x/snippets/*/*.*obj
> +/s390x/**/*.lds
> diff --git a/s390x/Makefile b/s390x/Makefile
> index 8719f0c837cf..44ccca8102d6 100644
> --- a/s390x/Makefile
> +++ b/s390x/Makefile
> @@ -76,7 +76,7 @@ CFLAGS += -fno-delete-null-pointer-checks
>  LDFLAGS += -nostdlib -Wl,--build-id=none
>  
>  # We want to keep intermediate files
> -.PRECIOUS: %.o
> +.PRECIOUS: %.o %.lds
>  
>  asm-offsets = lib/$(ARCH)/asm-offsets.h
>  include $(SRCDIR)/scripts/asm-offsets.mak
> @@ -159,6 +159,8 @@ $(SNIPPET_DIR)/c/%.gbin: $(SNIPPET_DIR)/c/%.o $(snippet_lib) $(FLATLIBS) $(SNIPP
>  %.hdr.obj: %.hdr
>  	$(OBJCOPY) -I binary -O elf64-s390 -B "s390:64-bit" $< $@
>  
> +%.lds: %.lds.S
> +	$(CPP) $(autodepend-flags) $(CPPFLAGS) -P -C -o $@ $<

Where is CPP defined?
Do you need the $(autodepend-flags)? It generates a rule with target flat.lds.o.
I don't think that would be used anywhere.
In the next patch you add $(asm-offsets) as a prerequisite, if the generated rule would
be effective, you wouldn't need that, would you?
>  
>  .SECONDEXPANSION:
>  %.elf: $(FLATLIBS) $(asmlib) $(SRCDIR)/s390x/flat.lds $$(snippets-obj) $$(snippet-hdr-obj) %.o
> @@ -211,7 +213,7 @@ $(snippet_asmlib): $$(patsubst %.o,%.S,$$@) $(asm-offsets)
>  
>  
>  arch_clean: asm_offsets_clean
> -	$(RM) $(TEST_DIR)/*.{o,elf,bin} $(SNIPPET_DIR)/*/*.{o,elf,*bin,*obj,hdr} $(SNIPPET_DIR)/asm/.*.d $(TEST_DIR)/.*.d lib/s390x/.*.d $(comm-key)
> +	$(RM) $(TEST_DIR)/*.{o,elf,bin,lds} $(SNIPPET_DIR)/*/*.{o,elf,*bin,*obj,hdr,lds} $(SNIPPET_DIR)/asm/.*.d $(TEST_DIR)/.*.d lib/s390x/.*.d $(comm-key)
>  
>  generated-files = $(asm-offsets)
>  $(tests:.elf=.o) $(asmlib) $(cflatobjs): $(generated-files)
> diff --git a/s390x/flat.lds b/s390x/flat.lds.S
> similarity index 100%
> rename from s390x/flat.lds
> rename to s390x/flat.lds.S
> diff --git a/s390x/snippets/asm/flat.lds b/s390x/snippets/asm/flat.lds.S
> similarity index 100%
> rename from s390x/snippets/asm/flat.lds
> rename to s390x/snippets/asm/flat.lds.S
> diff --git a/s390x/snippets/c/flat.lds b/s390x/snippets/c/flat.lds.S
> similarity index 100%
> rename from s390x/snippets/c/flat.lds
> rename to s390x/snippets/c/flat.lds.S

Re: [kvm-unit-tests PATCH v2 5/8] s390x: use C pre-processor for linker script generation

[Marc Hartmayer]

Nina Schoetterl-Glausch <nsg@linux.ibm.com> writes:

> On Thu, 2023-01-19 at 12:40 +0100, Marc Hartmayer wrote:
>> Use the C pre-processor for the linker script generation. For example,
>> this enables us the use of constants in the "linker scripts" `*.lds.S`.
>> 
>> Signed-off-by: Marc Hartmayer <mhartmay@linux.ibm.com>
>> ---
>>  .gitignore                                  | 1 +
>>  s390x/Makefile                              | 6 ++++--
>>  s390x/{flat.lds => flat.lds.S}              | 0
>>  s390x/snippets/asm/{flat.lds => flat.lds.S} | 0
>>  s390x/snippets/c/{flat.lds => flat.lds.S}   | 0
>>  5 files changed, 5 insertions(+), 2 deletions(-)
>>  rename s390x/{flat.lds => flat.lds.S} (100%)
>>  rename s390x/snippets/asm/{flat.lds => flat.lds.S} (100%)
>>  rename s390x/snippets/c/{flat.lds => flat.lds.S} (100%)
>> 
>> diff --git a/.gitignore b/.gitignore
>> index 601822d67325..29f352c5ceb6 100644
>> --- a/.gitignore
>> +++ b/.gitignore
>> @@ -31,3 +31,4 @@ cscope.*
>>  /s390x/comm.key
>>  /s390x/snippets/*/*.hdr
>>  /s390x/snippets/*/*.*obj
>> +/s390x/**/*.lds
>> diff --git a/s390x/Makefile b/s390x/Makefile
>> index 8719f0c837cf..44ccca8102d6 100644
>> --- a/s390x/Makefile
>> +++ b/s390x/Makefile
>> @@ -76,7 +76,7 @@ CFLAGS += -fno-delete-null-pointer-checks
>>  LDFLAGS += -nostdlib -Wl,--build-id=none
>>  
>>  # We want to keep intermediate files
>> -.PRECIOUS: %.o
>> +.PRECIOUS: %.o %.lds
>>  
>>  asm-offsets = lib/$(ARCH)/asm-offsets.h
>>  include $(SRCDIR)/scripts/asm-offsets.mak
>> @@ -159,6 +159,8 @@ $(SNIPPET_DIR)/c/%.gbin: $(SNIPPET_DIR)/c/%.o $(snippet_lib) $(FLATLIBS) $(SNIPP
>>  %.hdr.obj: %.hdr
>>  	$(OBJCOPY) -I binary -O elf64-s390 -B "s390:64-bit" $< $@
>>  
>> +%.lds: %.lds.S
>> +	$(CPP) $(autodepend-flags) $(CPPFLAGS) -P -C -o $@ $<
>
> Where is CPP defined?
> Do you need the $(autodepend-flags)? It generates a rule with target flat.lds.o.
                                                                       ^^^^^^^^^^

Where does it generate a new Makefile rule? It generates the dependency
file used for the dependency tracking with the name: .flat.lds.S.d (or
similar)

> I don't think that would be used anywhere.

It’s used for the dependency tracking. See line

$KUT/Makefile:122

-include */.*.d */*/.*.d

> In the next patch you add $(asm-offsets) as a prerequisite, if the generated rule would
> be effective, you wouldn't need that, would you?

I thought, $(asm-offsets) is used to make sure that the file
<asm/asm-offsets.h> is generated.

[…snip]

-- 
Kind regards / Beste Grüße
   Marc Hartmayer

IBM Deutschland Research & Development GmbH
Vorsitzender des Aufsichtsrats: Gregor Pillen 
Geschäftsführung: David Faller
Sitz der Gesellschaft: Böblingen
Registergericht: Amtsgericht Stuttgart, HRB 243294

Re: [kvm-unit-tests PATCH v2 5/8] s390x: use C pre-processor for linker script generation

[Nina Schoetterl-Glausch]

On Wed, 2023-02-01 at 11:00 +0100, Marc Hartmayer wrote:
> Nina Schoetterl-Glausch <nsg@linux.ibm.com> writes:
> 
> > On Thu, 2023-01-19 at 12:40 +0100, Marc Hartmayer wrote:
> > > Use the C pre-processor for the linker script generation. For example,
> > > this enables us the use of constants in the "linker scripts" `*.lds.S`.
> > > 
> > > Signed-off-by: Marc Hartmayer <mhartmay@linux.ibm.com>
> > > ---
> > >  .gitignore                                  | 1 +
> > >  s390x/Makefile                              | 6 ++++--
> > >  s390x/{flat.lds => flat.lds.S}              | 0
> > >  s390x/snippets/asm/{flat.lds => flat.lds.S} | 0
> > >  s390x/snippets/c/{flat.lds => flat.lds.S}   | 0
> > >  5 files changed, 5 insertions(+), 2 deletions(-)
> > >  rename s390x/{flat.lds => flat.lds.S} (100%)
> > >  rename s390x/snippets/asm/{flat.lds => flat.lds.S} (100%)
> > >  rename s390x/snippets/c/{flat.lds => flat.lds.S} (100%)
> > > 
> > > diff --git a/.gitignore b/.gitignore
> > > index 601822d67325..29f352c5ceb6 100644
> > > --- a/.gitignore
> > > +++ b/.gitignore
> > > @@ -31,3 +31,4 @@ cscope.*
> > >  /s390x/comm.key
> > >  /s390x/snippets/*/*.hdr
> > >  /s390x/snippets/*/*.*obj
> > > +/s390x/**/*.lds
> > > diff --git a/s390x/Makefile b/s390x/Makefile
> > > index 8719f0c837cf..44ccca8102d6 100644
> > > --- a/s390x/Makefile
> > > +++ b/s390x/Makefile
> > > @@ -76,7 +76,7 @@ CFLAGS += -fno-delete-null-pointer-checks
> > >  LDFLAGS += -nostdlib -Wl,--build-id=none
> > >  
> > >  # We want to keep intermediate files
> > > -.PRECIOUS: %.o
> > > +.PRECIOUS: %.o %.lds
> > >  
> > >  asm-offsets = lib/$(ARCH)/asm-offsets.h
> > >  include $(SRCDIR)/scripts/asm-offsets.mak
> > > @@ -159,6 +159,8 @@ $(SNIPPET_DIR)/c/%.gbin: $(SNIPPET_DIR)/c/%.o $(snippet_lib) $(FLATLIBS) $(SNIPP
> > >  %.hdr.obj: %.hdr
> > >  	$(OBJCOPY) -I binary -O elf64-s390 -B "s390:64-bit" $< $@
> > >  
> > > +%.lds: %.lds.S
> > > +	$(CPP) $(autodepend-flags) $(CPPFLAGS) -P -C -o $@ $<
> > 
> > Where is CPP defined?
> > Do you need the $(autodepend-flags)? It generates a rule with target flat.lds.o.
>                                                                        ^^^^^^^^^^
> 
> Where does it generate a new Makefile rule? It generates the dependency
> file used for the dependency tracking with the name: .flat.lds.S.d (or
> similar)

Yes and that file contains the rule.
cat s390x/.flat.d
flat.lds.o: /foo/bar/s390x/flat.lds.S \
 /foo/bar/lib/asm/asm-offsets.h \
 /foo/bar/lib/generated/asm-offsets.h

> 
> > I don't think that would be used anywhere.
> 
> It’s used for the dependency tracking. See line
> 
> $KUT/Makefile:122
> 
> -include */.*.d */*/.*.d

Indeed, but the target flat.lds.o isn't used anywhere, is it?
So if flat.lds.S included some other header and that changed,
flat.lds wouldn't be rebuild.
So you would either need to generate a rule with flat.lds as target
or make it depend on flat.lds.o somehow.

> 
> > In the next patch you add $(asm-offsets) as a prerequisite, if the generated rule would
> > be effective, you wouldn't need that, would you?
> 
> I thought, $(asm-offsets) is used to make sure that the file
> <asm/asm-offsets.h> is generated.

I was asking if this is necessary, since s390x/.flat.d contains the asm-offset headers
as prerequisites. So if the dependency file is included, the headers will be build,
but I guess that this is circular, in order to generate the dependencies,
the asm-offset headers must already have been built.
> 
> […snip]
> 

Re: [kvm-unit-tests PATCH v2 5/8] s390x: use C pre-processor for linker script generation

[Marc Hartmayer]

Nina Schoetterl-Glausch <nsg@linux.ibm.com> writes:

> On Wed, 2023-02-01 at 11:00 +0100, Marc Hartmayer wrote:
>> Nina Schoetterl-Glausch <nsg@linux.ibm.com> writes:
>> 
>> > On Thu, 2023-01-19 at 12:40 +0100, Marc Hartmayer wrote:
>> > > Use the C pre-processor for the linker script generation. For example,
>> > > this enables us the use of constants in the "linker scripts" `*.lds.S`.
>> > > 
>> > > Signed-off-by: Marc Hartmayer <mhartmay@linux.ibm.com>
>> > > ---
>> > >  .gitignore                                  | 1 +
>> > >  s390x/Makefile                              | 6 ++++--
>> > >  s390x/{flat.lds => flat.lds.S}              | 0
>> > >  s390x/snippets/asm/{flat.lds => flat.lds.S} | 0
>> > >  s390x/snippets/c/{flat.lds => flat.lds.S}   | 0
>> > >  5 files changed, 5 insertions(+), 2 deletions(-)
>> > >  rename s390x/{flat.lds => flat.lds.S} (100%)
>> > >  rename s390x/snippets/asm/{flat.lds => flat.lds.S} (100%)
>> > >  rename s390x/snippets/c/{flat.lds => flat.lds.S} (100%)
>> > > 
>> > > diff --git a/.gitignore b/.gitignore
>> > > index 601822d67325..29f352c5ceb6 100644
>> > > --- a/.gitignore
>> > > +++ b/.gitignore
>> > > @@ -31,3 +31,4 @@ cscope.*
>> > >  /s390x/comm.key
>> > >  /s390x/snippets/*/*.hdr
>> > >  /s390x/snippets/*/*.*obj
>> > > +/s390x/**/*.lds
>> > > diff --git a/s390x/Makefile b/s390x/Makefile
>> > > index 8719f0c837cf..44ccca8102d6 100644
>> > > --- a/s390x/Makefile
>> > > +++ b/s390x/Makefile
>> > > @@ -76,7 +76,7 @@ CFLAGS += -fno-delete-null-pointer-checks
>> > >  LDFLAGS += -nostdlib -Wl,--build-id=none
>> > >  
>> > >  # We want to keep intermediate files
>> > > -.PRECIOUS: %.o
>> > > +.PRECIOUS: %.o %.lds
>> > >  
>> > >  asm-offsets = lib/$(ARCH)/asm-offsets.h
>> > >  include $(SRCDIR)/scripts/asm-offsets.mak
>> > > @@ -159,6 +159,8 @@ $(SNIPPET_DIR)/c/%.gbin: $(SNIPPET_DIR)/c/%.o $(snippet_lib) $(FLATLIBS) $(SNIPP
>> > >  %.hdr.obj: %.hdr
>> > >  	$(OBJCOPY) -I binary -O elf64-s390 -B "s390:64-bit" $< $@
>> > >  
>> > > +%.lds: %.lds.S
>> > > +	$(CPP) $(autodepend-flags) $(CPPFLAGS) -P -C -o $@ $<
>> > 
>> > Where is CPP defined?
>> > Do you need the $(autodepend-flags)? It generates a rule with target flat.lds.o.
>>                                                                        ^^^^^^^^^^
>> 
>> Where does it generate a new Makefile rule? It generates the dependency
>> file used for the dependency tracking with the name: .flat.lds.S.d (or
>> similar)
>
> Yes and that file contains the rule.
> cat s390x/.flat.d
> flat.lds.o: /foo/bar/s390x/flat.lds.S \
>  /foo/bar/lib/asm/asm-offsets.h \
>  /foo/bar/lib/generated/asm-offsets.h

Yep, right… thanks for pointing it out! So I could either don’t use
$(autodepend-flags) or adapt it:

I could change autodepend-flags to:

--- i/Makefile
+++ w/Makefile
@@ -94,7 +94,7 @@ CFLAGS += $(wmissing_parameter_type)
 CFLAGS += $(wold_style_declaration)
 CFLAGS += -Woverride-init -Wmissing-prototypes -Wstrict-prototypes
 
-autodepend-flags = -MMD -MF $(dir $*).$(notdir $*).d
+autodepend-flags = -MMD -MF $(dir $*).$(notdir $*).d -MT $@

Shouldn’t break anything (we have to double check of course) and it
results in a Makefile rule like this for the linker scripts:

s390x/snippets/c/flat.lds: s390x/snippets/c/flat.lds.S \
 /foo/kvm-unit-tests/lib/asm/asm-offsets.h \
 /foo/kvm-unit-tests/lib/generated/asm-offsets.h


>
>> 
>> > I don't think that would be used anywhere.
>> 
>> It’s used for the dependency tracking. See line
>> 
>> $KUT/Makefile:122
>> 
>> -include */.*.d */*/.*.d
>
> Indeed, but the target flat.lds.o isn't used anywhere, is it?

No, it’s not and it doesn’t make any sense.

> So if flat.lds.S included some other header and that changed,
> flat.lds wouldn't be rebuild.
> So you would either need to generate a rule with flat.lds as target
> or make it depend on flat.lds.o somehow.
>
>> 
>> > In the next patch you add $(asm-offsets) as a prerequisite, if the generated rule would
>> > be effective, you wouldn't need that, would you?
>> 
>> I thought, $(asm-offsets) is used to make sure that the file
>> <asm/asm-offsets.h> is generated.
>
> I was asking if this is necessary, since s390x/.flat.d contains the asm-offset headers
> as prerequisites. So if the dependency file is included, the headers will be build,
> but I guess that this is circular, in order to generate the dependencies,
> the asm-offset headers must already have been built.

Hmm - I’ll check.

Thanks for the feedback!

>> 
>> […snip]
>> 
>
-- 
Kind regards / Beste Grüße
   Marc Hartmayer

IBM Deutschland Research & Development GmbH
Vorsitzender des Aufsichtsrats: Gregor Pillen 
Geschäftsführung: David Faller
Sitz der Gesellschaft: Böblingen
Registergericht: Amtsgericht Stuttgart, HRB 243294

Re: [kvm-unit-tests PATCH v2 5/8] s390x: use C pre-processor for linker script generation

[Marc Hartmayer]

Nina Schoetterl-Glausch <nsg@linux.ibm.com> writes:

> On Wed, 2023-02-01 at 11:00 +0100, Marc Hartmayer wrote:
>> Nina Schoetterl-Glausch <nsg@linux.ibm.com> writes:
>> 
>> > On Thu, 2023-01-19 at 12:40 +0100, Marc Hartmayer wrote:
>> > > Use the C pre-processor for the linker script generation. For example,
>> > > this enables us the use of constants in the "linker scripts" `*.lds.S`.
>> > > 
>> > > Signed-off-by: Marc Hartmayer <mhartmay@linux.ibm.com>
>> > > ---
>> > >  .gitignore                                  | 1 +
>> > >  s390x/Makefile                              | 6 ++++--
>> > >  s390x/{flat.lds => flat.lds.S}              | 0
>> > >  s390x/snippets/asm/{flat.lds => flat.lds.S} | 0
>> > >  s390x/snippets/c/{flat.lds => flat.lds.S}   | 0
>> > >  5 files changed, 5 insertions(+), 2 deletions(-)
>> > >  rename s390x/{flat.lds => flat.lds.S} (100%)
>> > >  rename s390x/snippets/asm/{flat.lds => flat.lds.S} (100%)
>> > >  rename s390x/snippets/c/{flat.lds => flat.lds.S} (100%)
>> > > 
>> > > diff --git a/.gitignore b/.gitignore
>> > > index 601822d67325..29f352c5ceb6 100644
>> > > --- a/.gitignore
>> > > +++ b/.gitignore
>> > > @@ -31,3 +31,4 @@ cscope.*
>> > >  /s390x/comm.key
>> > >  /s390x/snippets/*/*.hdr
>> > >  /s390x/snippets/*/*.*obj
>> > > +/s390x/**/*.lds
>> > > diff --git a/s390x/Makefile b/s390x/Makefile
>> > > index 8719f0c837cf..44ccca8102d6 100644
>> > > --- a/s390x/Makefile
>> > > +++ b/s390x/Makefile
>> > > @@ -76,7 +76,7 @@ CFLAGS += -fno-delete-null-pointer-checks
>> > >  LDFLAGS += -nostdlib -Wl,--build-id=none
>> > >  
>> > >  # We want to keep intermediate files
>> > > -.PRECIOUS: %.o
>> > > +.PRECIOUS: %.o %.lds
>> > >  
>> > >  asm-offsets = lib/$(ARCH)/asm-offsets.h
>> > >  include $(SRCDIR)/scripts/asm-offsets.mak
>> > > @@ -159,6 +159,8 @@ $(SNIPPET_DIR)/c/%.gbin: $(SNIPPET_DIR)/c/%.o $(snippet_lib) $(FLATLIBS) $(SNIPP
>> > >  %.hdr.obj: %.hdr
>> > >  	$(OBJCOPY) -I binary -O elf64-s390 -B "s390:64-bit" $< $@
>> > >  
>> > > +%.lds: %.lds.S
>> > > +	$(CPP) $(autodepend-flags) $(CPPFLAGS) -P -C -o $@ $<
>> > 
>> > Where is CPP defined?
>> > Do you need the $(autodepend-flags)? It generates a rule with target flat.lds.o.
>>                                                                        ^^^^^^^^^^
>> 
>> Where does it generate a new Makefile rule? It generates the dependency
>> file used for the dependency tracking with the name: .flat.lds.S.d (or
>> similar)
>
> Yes and that file contains the rule.
> cat s390x/.flat.d
> flat.lds.o: /foo/bar/s390x/flat.lds.S \
>  /foo/bar/lib/asm/asm-offsets.h \
>  /foo/bar/lib/generated/asm-offsets.h
>
>> 
>> > I don't think that would be used anywhere.
>> 
>> It’s used for the dependency tracking. See line
>> 
>> $KUT/Makefile:122
>> 
>> -include */.*.d */*/.*.d
>
> Indeed, but the target flat.lds.o isn't used anywhere, is it?
> So if flat.lds.S included some other header and that changed,
> flat.lds wouldn't be rebuild.
> So you would either need to generate a rule with flat.lds as target
> or make it depend on flat.lds.o somehow.
>
>> 
>> > In the next patch you add $(asm-offsets) as a prerequisite, if the generated rule would
>> > be effective, you wouldn't need that, would you?
>> 
>> I thought, $(asm-offsets) is used to make sure that the file
>> <asm/asm-offsets.h> is generated.
>
> I was asking if this is necessary, since s390x/.flat.d contains the asm-offset headers
> as prerequisites. So if the dependency file is included, the headers will be build,
> but I guess that this is circular, in order to generate the dependencies,
> the asm-offset headers must already have been built.

Hmm, no, this won’t work unless the dependency file was already created
(chicken-egg problem). That’s probably the reason for this Makefile
rule:

generated-files = $(asm-offsets)
$(tests:.elf=.o) $(asmlib) $(cflatobjs): $(generated-files)

>> 
>> […snip]
>> 
>
-- 
Kind regards / Beste Grüße
   Marc Hartmayer

IBM Deutschland Research & Development GmbH
Vorsitzender des Aufsichtsrats: Gregor Pillen 
Geschäftsführung: David Faller
Sitz der Gesellschaft: Böblingen
Registergericht: Amtsgericht Stuttgart, HRB 243294

Re: [kvm-unit-tests PATCH v2 5/8] s390x: use C pre-processor for linker script generation

[Marc Hartmayer]

Nina Schoetterl-Glausch <nsg@linux.ibm.com> writes:

> On Thu, 2023-01-19 at 12:40 +0100, Marc Hartmayer wrote:
>> Use the C pre-processor for the linker script generation. For example,
>> this enables us the use of constants in the "linker scripts" `*.lds.S`.
>> 
>> Signed-off-by: Marc Hartmayer <mhartmay@linux.ibm.com>
>> ---
>>  .gitignore                                  | 1 +
>>  s390x/Makefile                              | 6 ++++--
>>  s390x/{flat.lds => flat.lds.S}              | 0
>>  s390x/snippets/asm/{flat.lds => flat.lds.S} | 0
>>  s390x/snippets/c/{flat.lds => flat.lds.S}   | 0
>>  5 files changed, 5 insertions(+), 2 deletions(-)
>>  rename s390x/{flat.lds => flat.lds.S} (100%)
>>  rename s390x/snippets/asm/{flat.lds => flat.lds.S} (100%)
>>  rename s390x/snippets/c/{flat.lds => flat.lds.S} (100%)
>> 
>> diff --git a/.gitignore b/.gitignore
>> index 601822d67325..29f352c5ceb6 100644
>> --- a/.gitignore
>> +++ b/.gitignore
>> @@ -31,3 +31,4 @@ cscope.*
>>  /s390x/comm.key
>>  /s390x/snippets/*/*.hdr
>>  /s390x/snippets/*/*.*obj
>> +/s390x/**/*.lds
>> diff --git a/s390x/Makefile b/s390x/Makefile
>> index 8719f0c837cf..44ccca8102d6 100644
>> --- a/s390x/Makefile
>> +++ b/s390x/Makefile
>> @@ -76,7 +76,7 @@ CFLAGS += -fno-delete-null-pointer-checks
>>  LDFLAGS += -nostdlib -Wl,--build-id=none
>>  
>>  # We want to keep intermediate files
>> -.PRECIOUS: %.o
>> +.PRECIOUS: %.o %.lds
>>  
>>  asm-offsets = lib/$(ARCH)/asm-offsets.h
>>  include $(SRCDIR)/scripts/asm-offsets.mak
>> @@ -159,6 +159,8 @@ $(SNIPPET_DIR)/c/%.gbin: $(SNIPPET_DIR)/c/%.o $(snippet_lib) $(FLATLIBS) $(SNIPP
>>  %.hdr.obj: %.hdr
>>  	$(OBJCOPY) -I binary -O elf64-s390 -B "s390:64-bit" $< $@
>>  
>> +%.lds: %.lds.S
>> +	$(CPP) $(autodepend-flags) $(CPPFLAGS) -P -C -o $@ $<
>
> Where is CPP defined?

Sry, I forgot to answer this question. For GNU Make, it’s defined
implicitly:

https://www.gnu.org/software/make/manual/html_node/Implicit-Variables.html

[…snip]

-- 
Kind regards / Beste Grüße
   Marc Hartmayer

IBM Deutschland Research & Development GmbH
Vorsitzender des Aufsichtsrats: Gregor Pillen 
Geschäftsführung: David Faller
Sitz der Gesellschaft: Böblingen
Registergericht: Amtsgericht Stuttgart, HRB 243294

Re: [kvm-unit-tests PATCH v2 6/8] s390x: define a macro for the stack frame size

[Nina Schoetterl-Glausch]

[-- Attachment #1: Type: text/plain, Size: 2080 bytes --]

On Thu, 2023-01-19 at 12:40 +0100, Marc Hartmayer wrote:
> Define and use a macro for the stack frame size.
> 
> Signed-off-by: Marc Hartmayer <mhartmay@linux.ibm.com>

Reviewed-by: Nina Schoetterl-Glausch <nsg@linux.ibm.com>
> ---
>  lib/s390x/asm-offsets.c     | 1 +
>  s390x/Makefile              | 2 +-
>  s390x/cstart64.S            | 2 +-
>  s390x/flat.lds.S            | 4 +++-
>  s390x/gs.c                  | 5 +++--
>  s390x/macros.S              | 4 ++--
>  s390x/snippets/c/flat.lds.S | 6 ++++--
>  7 files changed, 15 insertions(+), 9 deletions(-)
> 
[...]

> diff --git a/s390x/gs.c b/s390x/gs.c
> index 4993eb8f43a9..2c2b972d7e65 100644
> --- a/s390x/gs.c
> +++ b/s390x/gs.c
> @@ -9,6 +9,7 @@
>   *    Janosch Frank <frankja@linux.ibm.com>
>   */
>  #include <libcflat.h>
> +#include <asm/asm-offsets.h>
>  #include <asm/page.h>
>  #include <asm/facility.h>
>  #include <asm/interrupt.h>
> @@ -41,8 +42,8 @@ extern void gs_handler_asm(void);
>  	    "	    aghi    %r15,-320\n" 		/* Allocate stack frame */
One could argue that the macro should be used here also.
320 = stack_size + register_save_area + control_block
>  	    "	    stmg    %r0,%r13,192(%r15)\n" 	/* Store regs to save area */
>  	    "	    stg	    %r14,312(%r15)\n"
> -	    "	    la	    %r2,160(%r15)\n" 		/* Store gscb address in this_cb */
> -	    "	    .insn   rxy,0xe30000000049,0,160(%r15)\n" /* stgsc */
> +	    "	    la	    %r2," xstr(STACK_FRAME_SIZE) "(%r15) \n" 		/* Store gscb address in this_cb */
> +	    "	    .insn   rxy,0xe30000000049,0," xstr(STACK_FRAME_SIZE) "(%r15)\n" /* stgsc */

The comment indentation is a bit messed up now.

>  	    "	    lg	    %r14,24(%r2)\n" 		/* Get GSEPLA from GSCB*/
>  	    "	    lg	    %r14,40(%r14)\n" 		/* Get GSERA from GSEPL*/
>  	    "	    stg	    %r14,304(%r15)\n" 		/* Store GSERA in r14 of reg save area */

Nothing to do with your changes, but the whitespace in this asm block is pretty messy.
Have a look at the attached patch for some suggestion, but feel free to ignore them.

[...]


[-- Attachment #2: gs_handler_asm.patch --]
[-- Type: text/x-patch, Size: 2385 bytes --]

diff --git a/s390x/gs.c b/s390x/gs.c
index 2c2b972d..9ae893ea 100644
--- a/s390x/gs.c
+++ b/s390x/gs.c
@@ -36,22 +36,27 @@ static inline unsigned long load_guarded(unsigned long *p)
 
 /* guarded-storage event handler and finally it calls gs_handler */
 extern void gs_handler_asm(void);
-	asm(".globl gs_handler_asm\n"
-	    "gs_handler_asm:\n"
-	    "	    lgr	    %r14,%r15\n" 		/* Save current stack address in r14 */
-	    "	    aghi    %r15,-320\n" 		/* Allocate stack frame */
-	    "	    stmg    %r0,%r13,192(%r15)\n" 	/* Store regs to save area */
-	    "	    stg	    %r14,312(%r15)\n"
-	    "	    la	    %r2," xstr(STACK_FRAME_SIZE) "(%r15) \n" 		/* Store gscb address in this_cb */
-	    "	    .insn   rxy,0xe30000000049,0," xstr(STACK_FRAME_SIZE) "(%r15)\n" /* stgsc */
-	    "	    lg	    %r14,24(%r2)\n" 		/* Get GSEPLA from GSCB*/
-	    "	    lg	    %r14,40(%r14)\n" 		/* Get GSERA from GSEPL*/
-	    "	    stg	    %r14,304(%r15)\n" 		/* Store GSERA in r14 of reg save area */
-	    "	    brasl   %r14,gs_handler\n" 		/* Jump to gs_handler */
-	    "	    lmg	    %r0,%r15,192(%r15)\n" 	/* Restore regs */
-	    "	    aghi    %r14, 6\n" 			/* Add lgg instr len to GSERA */
-	    "	    br	    %r14\n" 			/* Jump to next instruction after lgg */
-	    "	    .size gs_handler_asm,.-gs_handler_asm\n");
+	asm (          ".macro	STGSC	args:vararg\n"
+		"	.insn	rxy,0xe30000000049,\\args\n"
+		"	.endm\n"
+		"	.globl	gs_handler_asm\n"
+		"gs_handler_asm:\n"
+		"	lgr	%r14,%r15\n"				/* Save current stack address in r14 */
+		".Lgs_handler_frame = 16*8+32+" xstr(STACK_FRAME_SIZE) "\n"
+		"	aghi	%r15,-(.Lgs_handler_frame)\n"		/* Allocate stack frame */
+		"	stmg	%r0,%r13,192(%r15)\n"			/* Store regs to save area */
+		"	stg	%r14,312(%r15)\n"
+		"	la	%r2," xstr(STACK_FRAME_SIZE) "(%r15)\n"	/* Store gscb address in this_cb */
+		"	STGSC	%r0," xstr(STACK_FRAME_SIZE) "(%r15)\n"
+		"	lg	%r14,24(%r2)\n"				/* Get GSEPLA from GSCB*/
+		"	lg	%r14,40(%r14)\n"			/* Get GSERA from GSEPL*/
+		"	stg	%r14,304(%r15)\n"			/* Store GSERA in r14 of reg save area */
+		"	brasl	%r14,gs_handler\n"			/* Jump to gs_handler */
+		"	lmg	%r0,%r15,192(%r15)\n"			/* Restore regs */
+		"	aghi	%r14, 6\n"				/* Add lgg instr len to GSERA */
+		"	br	%r14\n"					/* Jump to next instruction after lgg */
+		".size gs_handler_asm,.-gs_handler_asm\n"
+	);
 
 void gs_handler(struct gs_cb *this_cb)
 {