[PATCH/RFC 0/4] dma ops and virtio

[PATCH/RFC 0/4] dma ops and virtio

[Christian Borntraeger]

This is an RFC to check if I am on the right track.  There
are some attempts to unify the dma ops (Christoph) as well
as some attempts to make virtio use the dma API (Andy).

At todays discussion at the kernel summit, we concluded that
we want to use the same code on all platforms, whereever
possible, so having a dummy dma_op might be the easiest
solution to keep virtio-ccw as similar as possible to
virtio-pci. Andy Lutomirski will rework his patchset to
unconditionally use the dma ops.  We will also need a
compatibility quirk for powerpc to bypass the iommu mappings
on older QEMU version (which translates to all versions as
of today) and per device, e.g.  via device tree.  Ben
Herrenschmidt will look into that.

Here is a very quick (still untested) shot at providing the s390 part:
- patch1: dummy dma ops, inspired by the alpha code
- patch2: replace some of the alpha functions with the dummy ones
- patch3: allow per device dma ops for s390
- patch4: wire up virtio dma ops

TODOs
- test (s390 virtio-ccw with Andis changes, s390 pci) and review
- check i386 nommu dma ops for potential improvements in the noop 
  handlers
- make dma-noop only compile when necessary

Christian Borntraeger (4):
  Provide simple noop dma ops
  alpha: use common noop dma ops
  s390/dma: Allow per device dma ops
  s390/virtio: use noop dma ops

 arch/alpha/kernel/pci-noop.c        | 46 ++--------------------
 arch/s390/include/asm/device.h      |  6 ++-
 arch/s390/include/asm/dma-mapping.h |  2 +-
 arch/s390/pci/pci.c                 |  1 +
 drivers/s390/virtio/kvm_virtio.c    |  2 +
 drivers/s390/virtio/virtio_ccw.c    |  2 +
 include/linux/dma-mapping.h         |  2 +
 lib/Makefile                        |  2 +-
 lib/dma-noop.c                      | 77 +++++++++++++++++++++++++++++++++++++
 9 files changed, 95 insertions(+), 45 deletions(-)
 create mode 100644 lib/dma-noop.c

-- 
2.4.3

[PATCH 1/4] Provide simple noop dma ops

[Christian Borntraeger]

We are going to require dma_ops for several common drivers, even for
systems that do have an identity mapping. Lets provide some minimal
no-op dma_ops that can be used for that purpose.

Signed-off-by: Christian Borntraeger <borntraeger@de.ibm.com>
---
 include/linux/dma-mapping.h |  2 ++
 lib/Makefile                |  2 +-
 lib/dma-noop.c              | 77 +++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 80 insertions(+), 1 deletion(-)
 create mode 100644 lib/dma-noop.c

diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h
index ac07ff0..7912f54 100644
--- a/include/linux/dma-mapping.h
+++ b/include/linux/dma-mapping.h
@@ -66,6 +66,8 @@ struct dma_map_ops {
 	int is_phys;
 };
 
+extern struct dma_map_ops dma_noop_ops;
+
 #define DMA_BIT_MASK(n)	(((n) == 64) ? ~0ULL : ((1ULL<<(n))-1))
 
 #define DMA_MASK_NONE	0x0ULL
diff --git a/lib/Makefile b/lib/Makefile
index 13a7c6a..b04ba71 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -13,7 +13,7 @@ lib-y := ctype.o string.o vsprintf.o cmdline.o \
 	 sha1.o md5.o irq_regs.o argv_split.o \
 	 proportions.o flex_proportions.o ratelimit.o show_mem.o \
 	 is_single_threaded.o plist.o decompress.o kobject_uevent.o \
-	 earlycpio.o seq_buf.o nmi_backtrace.o
+	 earlycpio.o seq_buf.o nmi_backtrace.o dma-noop.o
 
 obj-$(CONFIG_ARCH_HAS_DEBUG_STRICT_USER_COPY_CHECKS) += usercopy.o
 lib-$(CONFIG_MMU) += ioremap.o
diff --git a/lib/dma-noop.c b/lib/dma-noop.c
new file mode 100644
index 0000000..3ce31302
--- /dev/null
+++ b/lib/dma-noop.c
@@ -0,0 +1,77 @@
+/*
+ *	lib/dma-noop.c
+ *
+ * Stub DMA noop-ops
+ */
+#include <linux/export.h>
+#include <linux/mm.h>
+#include <linux/dma-mapping.h>
+#include <linux/scatterlist.h>
+
+static void *dma_noop_alloc(struct device *dev, size_t size,
+			    dma_addr_t *dma_handle, gfp_t gfp,
+			    struct dma_attrs *attrs)
+{
+	void *ret;
+
+	ret = (void *)__get_free_pages(gfp, get_order(size));
+	if (ret) {
+		memset(ret, 0, size);
+		*dma_handle = virt_to_phys(ret);
+	}
+	return ret;
+}
+
+static void dma_noop_free(struct device *dev, size_t size,
+			  void *cpu_addr, dma_addr_t dma_addr,
+			  struct dma_attrs *attrs)
+{
+	free_pages((unsigned long)cpu_addr, get_order(size));
+}
+
+static dma_addr_t dma_noop_map_page(struct device *dev, struct page *page,
+				      unsigned long offset, size_t size,
+				      enum dma_data_direction dir,
+				      struct dma_attrs *attrs)
+{
+	return page_to_phys(page) + offset;
+}
+
+static int dma_noop_map_sg(struct device *dev, struct scatterlist *sgl, int nents,
+			     enum dma_data_direction dir, struct dma_attrs *attrs)
+{
+	int i;
+	struct scatterlist *sg;
+
+	for_each_sg(sgl, sg, nents, i) {
+		void *va;
+
+		BUG_ON(!sg_page(sg));
+		va = sg_virt(sg);
+		sg_dma_address(sg) = (dma_addr_t)virt_to_phys(va);
+		sg_dma_len(sg) = sg->length;
+	}
+
+	return nents;
+}
+
+static int dma_noop_mapping_error(struct device *dev, dma_addr_t dma_addr)
+{
+	return 0;
+}
+
+static int dma_noop_supported(struct device *dev, u64 mask)
+{
+	return 1;
+}
+
+struct dma_map_ops dma_noop_ops = {
+	.alloc			= dma_noop_alloc,
+	.free			= dma_noop_free,
+	.map_page		= dma_noop_map_page,
+	.map_sg			= dma_noop_map_sg,
+	.mapping_error		= dma_noop_mapping_error,
+	.dma_supported		= dma_noop_supported,
+};
+
+EXPORT_SYMBOL(dma_noop_ops);
-- 
2.4.3

[PATCH 2/4] alpha: use common noop dma ops

[Christian Borntraeger]

Some of the alpha pci noop dma ops are identical to the common ones.
Use them.

Signed-off-by: Christian Borntraeger <borntraeger@de.ibm.com>
---
 arch/alpha/kernel/pci-noop.c | 46 ++++----------------------------------------
 1 file changed, 4 insertions(+), 42 deletions(-)

diff --git a/arch/alpha/kernel/pci-noop.c b/arch/alpha/kernel/pci-noop.c
index 2b1f4a1..8e735b5e 100644
--- a/arch/alpha/kernel/pci-noop.c
+++ b/arch/alpha/kernel/pci-noop.c
@@ -123,44 +123,6 @@ static void *alpha_noop_alloc_coherent(struct device *dev, size_t size,
 	return ret;
 }
 
-static void alpha_noop_free_coherent(struct device *dev, size_t size,
-				     void *cpu_addr, dma_addr_t dma_addr,
-				     struct dma_attrs *attrs)
-{
-	free_pages((unsigned long)cpu_addr, get_order(size));
-}
-
-static dma_addr_t alpha_noop_map_page(struct device *dev, struct page *page,
-				      unsigned long offset, size_t size,
-				      enum dma_data_direction dir,
-				      struct dma_attrs *attrs)
-{
-	return page_to_pa(page) + offset;
-}
-
-static int alpha_noop_map_sg(struct device *dev, struct scatterlist *sgl, int nents,
-			     enum dma_data_direction dir, struct dma_attrs *attrs)
-{
-	int i;
-	struct scatterlist *sg;
-
-	for_each_sg(sgl, sg, nents, i) {
-		void *va;
-
-		BUG_ON(!sg_page(sg));
-		va = sg_virt(sg);
-		sg_dma_address(sg) = (dma_addr_t)virt_to_phys(va);
-		sg_dma_len(sg) = sg->length;
-	}
-
-	return nents;
-}
-
-static int alpha_noop_mapping_error(struct device *dev, dma_addr_t dma_addr)
-{
-	return 0;
-}
-
 static int alpha_noop_supported(struct device *dev, u64 mask)
 {
 	return mask < 0x00ffffffUL ? 0 : 1;
@@ -168,10 +130,10 @@ static int alpha_noop_supported(struct device *dev, u64 mask)
 
 struct dma_map_ops alpha_noop_ops = {
 	.alloc			= alpha_noop_alloc_coherent,
-	.free			= alpha_noop_free_coherent,
-	.map_page		= alpha_noop_map_page,
-	.map_sg			= alpha_noop_map_sg,
-	.mapping_error		= alpha_noop_mapping_error,
+	.free			= dma_noop_free_coherent,
+	.map_page		= dma_noop_map_page,
+	.map_sg			= dma_noop_map_sg,
+	.mapping_error		= dma_noop_mapping_error,
 	.dma_supported		= alpha_noop_supported,
 };
 
-- 
2.4.3

[PATCH 3/4] s390/dma: Allow per device dma ops

[Christian Borntraeger]

As virtio-ccw now has dma ops, we can no longer default to the PCI ones.
Make use of dev_archdata to keep the dma_ops per device.

Signed-off-by: Christian Borntraeger <borntraeger@de.ibm.com>
---
 arch/s390/include/asm/device.h      | 6 +++++-
 arch/s390/include/asm/dma-mapping.h | 2 +-
 arch/s390/pci/pci.c                 | 1 +
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/arch/s390/include/asm/device.h b/arch/s390/include/asm/device.h
index d8f9872..4a9f35e 100644
--- a/arch/s390/include/asm/device.h
+++ b/arch/s390/include/asm/device.h
@@ -3,5 +3,9 @@
  *
  * This file is released under the GPLv2
  */
-#include <asm-generic/device.h>
+struct dev_archdata {
+	struct dma_map_ops *dma_ops;
+};
 
+struct pdev_archdata {
+};
diff --git a/arch/s390/include/asm/dma-mapping.h b/arch/s390/include/asm/dma-mapping.h
index b3fd54d..71404f1 100644
--- a/arch/s390/include/asm/dma-mapping.h
+++ b/arch/s390/include/asm/dma-mapping.h
@@ -15,7 +15,7 @@ extern struct dma_map_ops s390_dma_ops;
 
 static inline struct dma_map_ops *get_dma_ops(struct device *dev)
 {
-	return &s390_dma_ops;
+	return dev->archdata.dma_ops;
 }
 
 static inline void dma_cache_sync(struct device *dev, void *vaddr, size_t size,
diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c
index 7ef12a3..9913915 100644
--- a/arch/s390/pci/pci.c
+++ b/arch/s390/pci/pci.c
@@ -649,6 +649,7 @@ int pcibios_add_device(struct pci_dev *pdev)
 
 	zdev->pdev = pdev;
 	pdev->dev.groups = zpci_attr_groups;
+	pdev->dev.archdata.dma_ops = &s390_dma_ops;
 	zpci_map_resources(pdev);
 
 	for (i = 0; i < PCI_BAR_COUNT; i++) {
-- 
2.4.3

[PATCH 4/4] s390/virtio: use noop dma ops

[Christian Borntraeger]

With all infrastructure in place, lets provide dma_ops for virtio
devices on s390.

Signed-off-by: Christian Borntraeger <borntraeger@de.ibm.com>
---
 drivers/s390/virtio/kvm_virtio.c | 2 ++
 drivers/s390/virtio/virtio_ccw.c | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/drivers/s390/virtio/kvm_virtio.c b/drivers/s390/virtio/kvm_virtio.c
index 53fb975..05adaa9 100644
--- a/drivers/s390/virtio/kvm_virtio.c
+++ b/drivers/s390/virtio/kvm_virtio.c
@@ -13,6 +13,7 @@
 #include <linux/kernel_stat.h>
 #include <linux/init.h>
 #include <linux/bootmem.h>
+#include <linux/dma-mapping.h>
 #include <linux/err.h>
 #include <linux/virtio.h>
 #include <linux/virtio_config.h>
@@ -318,6 +319,7 @@ static void add_kvm_device(struct kvm_device_desc *d, unsigned int offset)
 		return;
 	}
 
+	kdev->vdev.dev.archdata.dma_ops = &dma_noop_ops;
 	kdev->vdev.dev.parent = kvm_root;
 	kdev->vdev.id.device = d->type;
 	kdev->vdev.config = &kvm_vq_configspace_ops;
diff --git a/drivers/s390/virtio/virtio_ccw.c b/drivers/s390/virtio/virtio_ccw.c
index 1cda784..8fb7a6b 100644
--- a/drivers/s390/virtio/virtio_ccw.c
+++ b/drivers/s390/virtio/virtio_ccw.c
@@ -13,6 +13,7 @@
 #include <linux/kernel_stat.h>
 #include <linux/init.h>
 #include <linux/bootmem.h>
+#include <linux/dma-mapping.h>
 #include <linux/err.h>
 #include <linux/virtio.h>
 #include <linux/virtio_config.h>
@@ -1093,6 +1094,7 @@ static void virtio_ccw_auto_online(void *data, async_cookie_t cookie)
 	struct ccw_device *cdev = data;
 	int ret;
 
+	cdev->dev.archdata.dma_ops = &dma_noop_ops;
 	ret = ccw_device_set_online(cdev);
 	if (ret)
 		dev_warn(&cdev->dev, "Failed to set online: %d\n", ret);
-- 
2.4.3

Re: [PATCH 1/4] Provide simple noop dma ops

[Joerg Roedel]

Hi Christian,

On Tue, Oct 27, 2015 at 11:48:48PM +0100, Christian Borntraeger wrote:
> +static dma_addr_t dma_noop_map_page(struct device *dev, struct page *page,
> +				      unsigned long offset, size_t size,
> +				      enum dma_data_direction dir,
> +				      struct dma_attrs *attrs)
> +{
> +	return page_to_phys(page) + offset;
> +}

X86 also has its own version of these noop dma_ops, see
arch/x86/kernel/pci-nommu.c. This one also checks the dma_mask and
prints a warning if the physical address doesn't fit into the mask.

I think this would make sense here too, and that we can also make x86
use the same generic noop-dma-ops your are introducing.


	Joerg

Re: [PATCH 4/4] s390/virtio: use noop dma ops

[Joerg Roedel]

On Tue, Oct 27, 2015 at 11:48:51PM +0100, Christian Borntraeger wrote:
> @@ -1093,6 +1094,7 @@ static void virtio_ccw_auto_online(void *data, async_cookie_t cookie)
>  	struct ccw_device *cdev = data;
>  	int ret;
>  
> +	cdev->dev.archdata.dma_ops = &dma_noop_ops;
>  	ret = ccw_device_set_online(cdev);
>  	if (ret)
>  		dev_warn(&cdev->dev, "Failed to set online: %d\n", ret);

Hmm, drivers usually don't deal with setting the dma_ops for their
devices, as they depend on the platform and not so much on the device
itself.

Can you do this special-case handling from device independent platform
code, where you also setup dma_ops for other devices?


	Joerg

Re: [PATCH/RFC 0/4] dma ops and virtio

[Joerg Roedel]

Hi Christian,

On Tue, Oct 27, 2015 at 11:48:47PM +0100, Christian Borntraeger wrote:
> Here is a very quick (still untested) shot at providing the s390 part:
> - patch1: dummy dma ops, inspired by the alpha code
> - patch2: replace some of the alpha functions with the dummy ones
> - patch3: allow per device dma ops for s390
> - patch4: wire up virtio dma ops

Thanks for the patches, I think they are a good start. I sent you
comments for two of the patches, the others look good to me.


	Joerg

Re: [PATCH 4/4] s390/virtio: use noop dma ops

[Cornelia Huck]

On Wed, 28 Oct 2015 09:43:34 +0900
Joerg Roedel <jroedel@suse.de> wrote:

> On Tue, Oct 27, 2015 at 11:48:51PM +0100, Christian Borntraeger wrote:
> > @@ -1093,6 +1094,7 @@ static void virtio_ccw_auto_online(void *data, async_cookie_t cookie)
> >  	struct ccw_device *cdev = data;
> >  	int ret;
> >  
> > +	cdev->dev.archdata.dma_ops = &dma_noop_ops;
> >  	ret = ccw_device_set_online(cdev);
> >  	if (ret)
> >  		dev_warn(&cdev->dev, "Failed to set online: %d\n", ret);
> 
> Hmm, drivers usually don't deal with setting the dma_ops for their
> devices, as they depend on the platform and not so much on the device
> itself.
> 
> Can you do this special-case handling from device independent platform
> code, where you also setup dma_ops for other devices?

Hm, maybe at the bus level?

pci devices get s390_dma_ops, ccw devices get the noop dma ops (just a
bit of dead weight for non-virtio ccw devices, I guess).

The old style s390-virtio devices are the odd ones around, but I'd like
to invest the least time possible there to keep them going.

Re: [PATCH/RFC 0/4] dma ops and virtio

[Andy Lutomirski]

On Tue, Oct 27, 2015 at 3:48 PM, Christian Borntraeger
<borntraeger@de.ibm.com> wrote:
> This is an RFC to check if I am on the right track.  There
> are some attempts to unify the dma ops (Christoph) as well
> as some attempts to make virtio use the dma API (Andy).
>
> At todays discussion at the kernel summit, we concluded that
> we want to use the same code on all platforms, whereever
> possible, so having a dummy dma_op might be the easiest
> solution to keep virtio-ccw as similar as possible to
> virtio-pci. Andy Lutomirski will rework his patchset to
> unconditionally use the dma ops.  We will also need a
> compatibility quirk for powerpc to bypass the iommu mappings
> on older QEMU version (which translates to all versions as
> of today) and per device, e.g.  via device tree.  Ben
> Herrenschmidt will look into that.

The combination of these patches plus my series don't link for me
unless I enable PCI.  Would s390 need to select HAS_DMA from VIRTIO or
something similar?

Also, is it possible to use QEMU to emulate an s390x?  Even just:

qemu-system-s390x -M s390-ccw-virtio

exits immediately.  (I build from QEMU git today.)  I don't know what
options to pass to get -M s390 to do anything useful, and AFAICT it's
considered deprecated.

Help?

--Andy

Re: [PATCH/RFC 0/4] dma ops and virtio

[Christian Borntraeger]

Am 29.10.2015 um 07:22 schrieb Andy Lutomirski:
> On Tue, Oct 27, 2015 at 3:48 PM, Christian Borntraeger
> <borntraeger@de.ibm.com> wrote:
>> This is an RFC to check if I am on the right track.  There
>> are some attempts to unify the dma ops (Christoph) as well
>> as some attempts to make virtio use the dma API (Andy).
>>
>> At todays discussion at the kernel summit, we concluded that
>> we want to use the same code on all platforms, whereever
>> possible, so having a dummy dma_op might be the easiest
>> solution to keep virtio-ccw as similar as possible to
>> virtio-pci. Andy Lutomirski will rework his patchset to
>> unconditionally use the dma ops.  We will also need a
>> compatibility quirk for powerpc to bypass the iommu mappings
>> on older QEMU version (which translates to all versions as
>> of today) and per device, e.g.  via device tree.  Ben
>> Herrenschmidt will look into that.
> 
> The combination of these patches plus my series don't link for me
> unless I enable PCI.  Would s390 need to select HAS_DMA from VIRTIO or
> something similar?

Well, actually this is a potential improvement for series. I could just 
make the noop dma ops default for _all_ devices unless it has a per 
device dma_ops (e.g. s390 pci) and the unconditionally set HAS_DMA.
> 
> Also, is it possible to use QEMU to emulate an s390x?  Even just:
> 
> qemu-system-s390x -M s390-ccw-virtio

Yes, we have no interactive bios and if no boot device is there is bios
will load a disabled wait, which will exit qemu.

Make sure to compile your kernel for z900  (processor type) as qemu does not
handle all things of newer processors.
You can then do 
qemu-system-s390x -nographic -m 256 -kernel vmlinux -initrd <something>

Re: [PATCH/RFC 0/4] dma ops and virtio

[Andy Lutomirski]

On Wed, Oct 28, 2015 at 5:04 PM, Christian Borntraeger
<borntraeger@de.ibm.com> wrote:
> Am 29.10.2015 um 07:22 schrieb Andy Lutomirski:
>> On Tue, Oct 27, 2015 at 3:48 PM, Christian Borntraeger
>> <borntraeger@de.ibm.com> wrote:
>>> This is an RFC to check if I am on the right track.  There
>>> are some attempts to unify the dma ops (Christoph) as well
>>> as some attempts to make virtio use the dma API (Andy).
>>>
>>> At todays discussion at the kernel summit, we concluded that
>>> we want to use the same code on all platforms, whereever
>>> possible, so having a dummy dma_op might be the easiest
>>> solution to keep virtio-ccw as similar as possible to
>>> virtio-pci. Andy Lutomirski will rework his patchset to
>>> unconditionally use the dma ops.  We will also need a
>>> compatibility quirk for powerpc to bypass the iommu mappings
>>> on older QEMU version (which translates to all versions as
>>> of today) and per device, e.g.  via device tree.  Ben
>>> Herrenschmidt will look into that.
>>
>> The combination of these patches plus my series don't link for me
>> unless I enable PCI.  Would s390 need to select HAS_DMA from VIRTIO or
>> something similar?
>
> Well, actually this is a potential improvement for series. I could just
> make the noop dma ops default for _all_ devices unless it has a per
> device dma_ops (e.g. s390 pci) and the unconditionally set HAS_DMA.
>>
>> Also, is it possible to use QEMU to emulate an s390x?  Even just:
>>
>> qemu-system-s390x -M s390-ccw-virtio
>
> Yes, we have no interactive bios and if no boot device is there is bios
> will load a disabled wait, which will exit qemu.
>
> Make sure to compile your kernel for z900  (processor type) as qemu does not
> handle all things of newer processors.
> You can then do
> qemu-system-s390x -nographic -m 256 -kernel vmlinux -initrd <something>
>

Progress!  After getting that sort-of-working, I figured out what was
wrong with my earlier command, and I got that working, too.  Now I
get:

qemu-system-s390x -fsdev
local,id=virtfs1,path=/,security_model=none,readonly -device
virtio-9p-ccw,fsdev=virtfs1,mount_tag=/dev/root -M s390-ccw-virtio
-nodefaults -device sclpconsole,chardev=console -parallel none -net
none -echr 1 -serial none -chardev stdio,id=console,signal=off,mux=on
-serial chardev:console -mon chardev=console -vga none -display none
-kernel arch/s390/boot/bzImage -append
'init=/home/luto/devel/virtme/virtme/guest/virtme-init
psmouse.proto=exps "virtme_stty_con=rows 24 cols 150 iutf8"
TERM=xterm-256color rootfstype=9p
rootflags=ro,version=9p2000.L,trans=virtio,access=any
raid=noautodetect debug'
Initializing cgroup subsys cpuset
Initializing cgroup subsys cpu
Initializing cgroup subsys cpuacct
Linux version 4.3.0-rc7-00403-ga2b5cd810259-dirty
(luto@amaluto.corp.amacapital.net) (gcc version 5.1.1 20150618 (Red
Hat Cross 5.1.1-3) (GCC) ) #328 SMP Thu Oct 29 15:46:05 PDT 2015
setup: Linux is running under KVM in 64-bit mode
setup: Max memory size: 128MB
Zone ranges:
  DMA      [mem 0x0000000000000000-0x000000007fffffff]
  Normal   empty
Movable zone start for each node
Early memory node ranges
  node   0: [mem 0x0000000000000000-0x0000000007ffffff]
Initmem setup node 0 [mem 0x0000000000000000-0x0000000007ffffff]
On node 0 totalpages: 32768
  DMA zone: 512 pages used for memmap
  DMA zone: 0 pages reserved
  DMA zone: 32768 pages, LIFO batch:7
PERCPU: Embedded 466 pages/cpu @0000000007605000 s1868032 r8192 d32512 u1908736
pcpu-alloc: s1868032 r8192 d32512 u1908736 alloc=466*4096
pcpu-alloc: [0] 0 [0] 1
Built 1 zonelists in Zone order, mobility grouping on.  Total pages: 32256
Kernel command line:
init=/home/luto/devel/virtme/virtme/guest/virtme-init
psmouse.proto=exps "virtme_stty_con=rows 24 cols 150 iutf8"
TERM=xterm-256color rootfstype=9p
rootflags=ro,version=9p2000.L,trans=virtio,access=any
raid=noautodetect debug
PID hash table entries: 512 (order: 0, 4096 bytes)
Dentry cache hash table entries: 16384 (order: 5, 131072 bytes)
Inode-cache hash table entries: 8192 (order: 4, 65536 bytes)
Memory: 92552K/131072K available (8229K kernel code, 798K rwdata,
3856K rodata, 2384K init, 14382K bss, 38520K reserved, 0K
cma-reserved)
Write protected kernel read-only data: 0x100000 - 0xccdfff
SLUB: HWalign=256, Order=0-3, MinObjects=0, CPUs=2, Nodes=1
Running RCU self tests
Hierarchical RCU implementation.
    RCU debugfs-based tracing is enabled.
    RCU lockdep checking is enabled.
    Build-time adjustment of leaf fanout to 64.
    RCU restricting CPUs from NR_CPUS=256 to nr_cpu_ids=2.
RCU: Adjusting geometry for rcu_fanout_leaf=64, nr_cpu_ids=2
NR_IRQS:3
clocksource: tod: mask: 0xffffffffffffffff max_cycles:
0x3b0a9be803b0a9, max_idle_ns: 1805497147909793 ns
console [ttyS1] enabled
Lock dependency validator: Copyright (c) 2006 Red Hat, Inc., Ingo Molnar
... MAX_LOCKDEP_SUBCLASSES:  8
... MAX_LOCK_DEPTH:          48
... MAX_LOCKDEP_KEYS:        8191
... CLASSHASH_SIZE:          4096
... MAX_LOCKDEP_ENTRIES:     32768
... MAX_LOCKDEP_CHAINS:      65536
... CHAINHASH_SIZE:          32768
 memory used by lock dependency info: 8671 kB
 per task-struct memory footprint: 2688 bytes
Calibrating delay loop (skipped)... 13370.00 BogoMIPS preset
pid_max: default: 32768 minimum: 301
Mount-cache hash table entries: 512 (order: 0, 4096 bytes)
Mountpoint-cache hash table entries: 512 (order: 0, 4096 bytes)
Initializing cgroup subsys io
Initializing cgroup subsys memory
ftrace: allocating 20968 entries in 82 pages
cpu: 1 configured CPUs, 0 standby CPUs
Brought up 1 CPUs
devtmpfs: initialized
clocksource: jiffies: mask: 0xffffffff max_cycles: 0xffffffff,
max_idle_ns: 19112604462750000 ns
xor: measuring software checksum speed
   8regs     :   158.800 MB/sec
   8regs_prefetch:   158.000 MB/sec
QEMU: Terminated
[luto@amaluto linux-devel]$
/home/luto/apps/qemu/s390x-softmmu/qemu-system-s390x -fsdev
local,id=virtfs1,path=/,security_model=none,readonly -device
virtio-9p-ccw,fsdev=virtfs1,mount_tag=/dev/root -M s390-ccw-virtio
-nodefaults -device sclpconsole,chardev=console -parallel none -net
none -echr 1 -serial none -chardev stdio,id=console,signal=off,mux=on
-serial chardev:console -mon chardev=console -vga none -display none
-kernel arch/s390/boot/bzImage -append debug
Initializing cgroup subsys cpuset
Initializing cgroup subsys cpu
Initializing cgroup subsys cpuacct
Linux version 4.3.0-rc7-00403-ga2b5cd810259-dirty
(luto@amaluto.corp.amacapital.net) (gcc version 5.1.1 20150618 (Red
Hat Cross 5.1.1-3) (GCC) ) #328 SMP Thu Oct 29 15:46:05 PDT 2015
setup: Linux is running under KVM in 64-bit mode
setup: Max memory size: 128MB
Zone ranges:
  DMA      [mem 0x0000000000000000-0x000000007fffffff]
  Normal   empty
Movable zone start for each node
Early memory node ranges
  node   0: [mem 0x0000000000000000-0x0000000007ffffff]
Initmem setup node 0 [mem 0x0000000000000000-0x0000000007ffffff]
On node 0 totalpages: 32768
  DMA zone: 512 pages used for memmap
  DMA zone: 0 pages reserved
  DMA zone: 32768 pages, LIFO batch:7
PERCPU: Embedded 466 pages/cpu @0000000007605000 s1868032 r8192 d32512 u1908736
pcpu-alloc: s1868032 r8192 d32512 u1908736 alloc=466*4096
pcpu-alloc: [0] 0 [0] 1
Built 1 zonelists in Zone order, mobility grouping on.  Total pages: 32256
Kernel command line: debug
PID hash table entries: 512 (order: 0, 4096 bytes)
Dentry cache hash table entries: 16384 (order: 5, 131072 bytes)
Inode-cache hash table entries: 8192 (order: 4, 65536 bytes)
Memory: 92552K/131072K available (8229K kernel code, 798K rwdata,
3856K rodata, 2384K init, 14382K bss, 38520K reserved, 0K
cma-reserved)
Write protected kernel read-only data: 0x100000 - 0xccdfff
SLUB: HWalign=256, Order=0-3, MinObjects=0, CPUs=2, Nodes=1
Running RCU self tests
Hierarchical RCU implementation.
    RCU debugfs-based tracing is enabled.
    RCU lockdep checking is enabled.
    Build-time adjustment of leaf fanout to 64.
    RCU restricting CPUs from NR_CPUS=256 to nr_cpu_ids=2.
RCU: Adjusting geometry for rcu_fanout_leaf=64, nr_cpu_ids=2
NR_IRQS:3
clocksource: tod: mask: 0xffffffffffffffff max_cycles:
0x3b0a9be803b0a9, max_idle_ns: 1805497147909793 ns
console [ttyS1] enabled
Lock dependency validator: Copyright (c) 2006 Red Hat, Inc., Ingo Molnar
... MAX_LOCKDEP_SUBCLASSES:  8
... MAX_LOCK_DEPTH:          48
... MAX_LOCKDEP_KEYS:        8191
... CLASSHASH_SIZE:          4096
... MAX_LOCKDEP_ENTRIES:     32768
... MAX_LOCKDEP_CHAINS:      65536
... CHAINHASH_SIZE:          32768
 memory used by lock dependency info: 8671 kB
 per task-struct memory footprint: 2688 bytes
Calibrating delay loop (skipped)... 13370.00 BogoMIPS preset
pid_max: default: 32768 minimum: 301
Mount-cache hash table entries: 512 (order: 0, 4096 bytes)
Mountpoint-cache hash table entries: 512 (order: 0, 4096 bytes)
Initializing cgroup subsys io
Initializing cgroup subsys memory
ftrace: allocating 20968 entries in 82 pages
cpu: 1 configured CPUs, 0 standby CPUs
Brought up 1 CPUs
devtmpfs: initialized
clocksource: jiffies: mask: 0xffffffff max_cycles: 0xffffffff,
max_idle_ns: 19112604462750000 ns
xor: measuring software checksum speed
   8regs     :   159.200 MB/sec
   8regs_prefetch:   159.200 MB/sec
   32regs    :   696.800 MB/sec
   32regs_prefetch:   692.000 MB/sec
xor: using function: 32regs (696.800 MB/sec)
NET: Registered protocol family 16
raid6: int64x1  gen()   517 MB/s
raid6: int64x1  xor()   321 MB/s
raid6: int64x2  gen()   610 MB/s
raid6: int64x2  xor()   354 MB/s
raid6: int64x4  gen()   467 MB/s
raid6: int64x4  xor()   241 MB/s
raid6: int64x8  gen()   288 MB/s
raid6: int64x8  xor()   194 MB/s
raid6: using algorithm int64x2 gen() 610 MB/s
raid6: .... xor() 354 MB/s, rmw enabled
raid6: using intx1 recovery algorithm
SCSI subsystem initialized
cio: event: sch 0.0.0000, new
cio: Subchannel 0.0.0000 reports subchannel type 0000
cio: chsc: ssd failed for 0.0.0000 (rc=0004)
cio: Detected device 0000 on subchannel 0.0.0000 - PIM = 80, PAM = 80, POM = FF
cio: event: sch 0.0.0000, process=1, action=2
snsid: device 0.0.0000: rc=0 3832/09 0000/00
cio: cdev_todo: sched cdev=0.0.0000 todo=3
cio: cdev_todo: cdev=0.0.0000 todo=3
cio: chsc: ssd failed for 0.0.0000 (rc=0004)
cio: chsc: scm info failed (rc=0004)
NET: Registered protocol family 2
TCP established hash table entries: 1024 (order: 1, 8192 bytes)
TCP bind hash table entries: 1024 (order: 4, 81920 bytes)
TCP: Hash tables configured (established 1024 bind 1024)
UDP hash table entries: 256 (order: 3, 49152 bytes)
UDP-Lite hash table entries: 256 (order: 3, 49152 bytes)
NET: Registered protocol family 1
hypfs: The hardware system does not support hypfs
hypfs: Initialization of hypfs failed with rc=-61
futex hash table entries: 512 (order: 5, 131072 bytes)
audit: initializing netlink subsys (disabled)
audit: type=2000 audit(1446158888.684:1): initialized
Kprobe smoke test: started
Kprobe smoke test: passed successfully
hugetlbfs: disabling because there are no supported hugepage sizes
fuse init (API version 7.23)
SGI XFS with ACLs, security attributes, realtime, no debug enabled
9p: Installing v9fs 9p2000 file system support
Block layer SCSI generic (bsg) driver version 0.4 loaded (major 253)
io scheduler noop registered
io scheduler deadline registered (default)
io scheduler cfq registered
start plist test
end plist test
hvc_iucv: The z/VM IUCV HVC device driver cannot be used without z/VM
brd: module loaded
st: Version 20101219, fixed bufsize 32768, s/g segs 256
cio: Channel measurement facility initialized using format extended
(mode autodetected)
dasd: debug area created
Discipline DIAG cannot be used without z/VM
qeth: loading core functions
qeth: register layer 2 discipline
qeth: register layer 3 discipline
oprofile: using timer interrupt.
NET: Registered protocol family 10
vrfy: device 0.0.0000: rc=0 pgroup=0 mpath=0 vpm=80
virtio_ccw 0.0.0000: Failed to set online: -5

^^^ bad news!

sit: IPv6 over IPv4 tunneling driver
NET: Registered protocol family 17
NET: Registered protocol family 15
8021q: 802.1Q VLAN Support v1.8
9pnet: Installing 9P2000 support
registered taskstats version 1
Btrfs loaded
List of all partitions:
0100            4096 ram0  (driver?)
0101            4096 ram1  (driver?)
0102            4096 ram2  (driver?)
0103            4096 ram3  (driver?)
0104            4096 ram4  (driver?)
0105            4096 ram5  (driver?)
0106            4096 ram6  (driver?)
0107            4096 ram7  (driver?)
0108            4096 ram8  (driver?)
0109            4096 ram9  (driver?)
010a            4096 ram10  (driver?)
010b            4096 ram11  (driver?)
010c            4096 ram12  (driver?)
010d            4096 ram13  (driver?)
010e            4096 ram14  (driver?)
010f            4096 ram15  (driver?)
No filesystem could mount root, tried:  ext3 ext2 ext4 fuseblk xfs btrfs
Kernel panic - not syncing: VFS: Unable to mount root fs on unknown-block(1,0)

I see this with and without this patch series -- 4.3-rc7 by itself
fails like this.  Is there some further magic incantation I need?

I hacked up debug_sprintf_event to log to the console, hence the extra
messages above.

--Andy

Re: [PATCH/RFC 0/4] dma ops and virtio

[Cornelia Huck]

On Thu, 29 Oct 2015 15:50:38 -0700
Andy Lutomirski <luto@amacapital.net> wrote:

> Progress!  After getting that sort-of-working, I figured out what was
> wrong with my earlier command, and I got that working, too.  Now I
> get:
> 
> qemu-system-s390x -fsdev
> local,id=virtfs1,path=/,security_model=none,readonly -device
> virtio-9p-ccw,fsdev=virtfs1,mount_tag=/dev/root -M s390-ccw-virtio
> -nodefaults -device sclpconsole,chardev=console -parallel none -net
> none -echr 1 -serial none -chardev stdio,id=console,signal=off,mux=on
> -serial chardev:console -mon chardev=console -vga none -display none
> -kernel arch/s390/boot/bzImage -append
> 'init=/home/luto/devel/virtme/virtme/guest/virtme-init
> psmouse.proto=exps "virtme_stty_con=rows 24 cols 150 iutf8"
> TERM=xterm-256color rootfstype=9p
> rootflags=ro,version=9p2000.L,trans=virtio,access=any
> raid=noautodetect debug'

The commandline looks sane AFAICS.

(...)

> vrfy: device 0.0.0000: rc=0 pgroup=0 mpath=0 vpm=80
> virtio_ccw 0.0.0000: Failed to set online: -5
> 
> ^^^ bad news!

I'd like to see where in the onlining process this fails. Could you set
up qemu tracing for css_* and virtio_ccw_* (instructions in
qemu/docs/tracing.txt)?

Which qemu version is this, btw.?

Re: [PATCH 4/4] s390/virtio: use noop dma ops

[Cornelia Huck]

On Tue, 27 Oct 2015 23:48:51 +0100
Christian Borntraeger <borntraeger@de.ibm.com> wrote:

> With all infrastructure in place, lets provide dma_ops for virtio
> devices on s390.
> 
> Signed-off-by: Christian Borntraeger <borntraeger@de.ibm.com>
> ---
>  drivers/s390/virtio/kvm_virtio.c | 2 ++
>  drivers/s390/virtio/virtio_ccw.c | 2 ++
>  2 files changed, 4 insertions(+)
> 
> diff --git a/drivers/s390/virtio/kvm_virtio.c b/drivers/s390/virtio/kvm_virtio.c
> index 53fb975..05adaa9 100644
> --- a/drivers/s390/virtio/kvm_virtio.c
> +++ b/drivers/s390/virtio/kvm_virtio.c
> @@ -13,6 +13,7 @@
>  #include <linux/kernel_stat.h>
>  #include <linux/init.h>
>  #include <linux/bootmem.h>
> +#include <linux/dma-mapping.h>
>  #include <linux/err.h>
>  #include <linux/virtio.h>
>  #include <linux/virtio_config.h>
> @@ -318,6 +319,7 @@ static void add_kvm_device(struct kvm_device_desc *d, unsigned int offset)
>  		return;
>  	}
> 
> +	kdev->vdev.dev.archdata.dma_ops = &dma_noop_ops;

This provides dma_ops for the vdev, while Andy's virtio code looks for
dma_ops in the vdev's parent (in the ccw and pci cases, the proxy
device; in this case, it would be our root device).

With

diff --git a/drivers/s390/virtio/kvm_virtio.c b/drivers/s390/virtio/kvm_virtio.c
index 05adaa9..5f79c52 100644
--- a/drivers/s390/virtio/kvm_virtio.c
+++ b/drivers/s390/virtio/kvm_virtio.c
@@ -319,7 +319,6 @@ static void add_kvm_device(struct kvm_device_desc *d, unsigned int offset)
 		return;
 	}
 
-	kdev->vdev.dev.archdata.dma_ops = &dma_noop_ops;
 	kdev->vdev.dev.parent = kvm_root;
 	kdev->vdev.id.device = d->type;
 	kdev->vdev.config = &kvm_vq_configspace_ops;
@@ -473,6 +472,7 @@ static int __init kvm_devices_init(void)
 		vmem_remove_mapping(total_memory_size, PAGE_SIZE);
 		return rc;
 	}
+	kvm_root->archdata.dma_ops = &dma_noop_ops;
 
 	INIT_WORK(&hotplug_work, hotplug_devices);
 
applied (and the endianness fix in the virtio code), I can boot a
s390-virtio guest as well.

>  	kdev->vdev.dev.parent = kvm_root;
>  	kdev->vdev.id.device = d->type;
>  	kdev->vdev.config = &kvm_vq_configspace_ops;

Re: [PATCH 4/4] s390/virtio: use noop dma ops

[Christian Borntraeger]

Am 30.10.2015 um 13:17 schrieb Cornelia Huck:
> On Tue, 27 Oct 2015 23:48:51 +0100
> Christian Borntraeger <borntraeger@de.ibm.com> wrote:
> 
>> With all infrastructure in place, lets provide dma_ops for virtio
>> devices on s390.
>>
>> Signed-off-by: Christian Borntraeger <borntraeger@de.ibm.com>
>> ---
>>  drivers/s390/virtio/kvm_virtio.c | 2 ++
>>  drivers/s390/virtio/virtio_ccw.c | 2 ++
>>  2 files changed, 4 insertions(+)
>>
>> diff --git a/drivers/s390/virtio/kvm_virtio.c b/drivers/s390/virtio/kvm_virtio.c
>> index 53fb975..05adaa9 100644
>> --- a/drivers/s390/virtio/kvm_virtio.c
>> +++ b/drivers/s390/virtio/kvm_virtio.c
>> @@ -13,6 +13,7 @@
>>  #include <linux/kernel_stat.h>
>>  #include <linux/init.h>
>>  #include <linux/bootmem.h>
>> +#include <linux/dma-mapping.h>
>>  #include <linux/err.h>
>>  #include <linux/virtio.h>
>>  #include <linux/virtio_config.h>
>> @@ -318,6 +319,7 @@ static void add_kvm_device(struct kvm_device_desc *d, unsigned int offset)
>>  		return;
>>  	}
>>
>> +	kdev->vdev.dev.archdata.dma_ops = &dma_noop_ops;
> 
> This provides dma_ops for the vdev, while Andy's virtio code looks for
> dma_ops in the vdev's parent (in the ccw and pci cases, the proxy
> device; in this case, it would be our root device).
> 
> With
> 
> diff --git a/drivers/s390/virtio/kvm_virtio.c b/drivers/s390/virtio/kvm_virtio.c
> index 05adaa9..5f79c52 100644
> --- a/drivers/s390/virtio/kvm_virtio.c
> +++ b/drivers/s390/virtio/kvm_virtio.c
> @@ -319,7 +319,6 @@ static void add_kvm_device(struct kvm_device_desc *d, unsigned int offset)
>  		return;
>  	}
> 
> -	kdev->vdev.dev.archdata.dma_ops = &dma_noop_ops;
>  	kdev->vdev.dev.parent = kvm_root;
>  	kdev->vdev.id.device = d->type;
>  	kdev->vdev.config = &kvm_vq_configspace_ops;
> @@ -473,6 +472,7 @@ static int __init kvm_devices_init(void)
>  		vmem_remove_mapping(total_memory_size, PAGE_SIZE);
>  		return rc;
>  	}
> +	kvm_root->archdata.dma_ops = &dma_noop_ops;
> 
>  	INIT_WORK(&hotplug_work, hotplug_devices);
> 
> applied (and the endianness fix in the virtio code), I can boot a
> s390-virtio guest as well.

I am currently reworking this to 

 static inline struct dma_map_ops *get_dma_ops(struct device *dev)
 {
	if (dev && dev->archdata.dma_ops)
		return dev->archdata.dma_ops;
	return &dma_noop_ops;
 }


Which uses the dma_noop_ops for everything unless the device overrides (PCI does)

Re: [PATCH 4/4] s390/virtio: use noop dma ops

[Cornelia Huck]

On Fri, 30 Oct 2015 13:26:09 +0100
Christian Borntraeger <borntraeger@de.ibm.com> wrote:

> I am currently reworking this to 
> 
>  static inline struct dma_map_ops *get_dma_ops(struct device *dev)
>  {
> 	if (dev && dev->archdata.dma_ops)
> 		return dev->archdata.dma_ops;
> 	return &dma_noop_ops;
>  }
> 
> 
> Which uses the dma_noop_ops for everything unless the device overrides (PCI does)

Yes, opt-in seems less error-prone here.

Re: [PATCH 1/4] Provide simple noop dma ops

[Christian Borntraeger]

Am 28.10.2015 um 01:41 schrieb Joerg Roedel:
> Hi Christian,
> 
> On Tue, Oct 27, 2015 at 11:48:48PM +0100, Christian Borntraeger wrote:
>> +static dma_addr_t dma_noop_map_page(struct device *dev, struct page *page,
>> +				      unsigned long offset, size_t size,
>> +				      enum dma_data_direction dir,
>> +				      struct dma_attrs *attrs)
>> +{
>> +	return page_to_phys(page) + offset;
>> +}
> 
> X86 also has its own version of these noop dma_ops, see
> arch/x86/kernel/pci-nommu.c. This one also checks the dma_mask and
> prints a warning if the physical address doesn't fit into the mask.
> 
> I think this would make sense here too, and that we can also make x86
> use the same generic noop-dma-ops your are introducing.

It not trivial without understanding the dma mask details. Do I read the x86
implementation right, that it limits the dma to 32 bit? Then we cannot collapse
both implementations. Or maybe we can hide this in dma_capable. Dont know

So I would prefer to keep it as is and let someone with x86 test environment do
the unification. Christoph, I think you wanted to do that anyway, are you willing
to do that?


Christian

Re: [PATCH 4/4] s390/virtio: use noop dma ops

[Christian Borntraeger]

Am 28.10.2015 um 01:43 schrieb Joerg Roedel:
> On Tue, Oct 27, 2015 at 11:48:51PM +0100, Christian Borntraeger wrote:
>> @@ -1093,6 +1094,7 @@ static void virtio_ccw_auto_online(void *data, async_cookie_t cookie)
>>  	struct ccw_device *cdev = data;
>>  	int ret;
>>  
>> +	cdev->dev.archdata.dma_ops = &dma_noop_ops;
>>  	ret = ccw_device_set_online(cdev);
>>  	if (ret)
>>  		dev_warn(&cdev->dev, "Failed to set online: %d\n", ret);
> 
> Hmm, drivers usually don't deal with setting the dma_ops for their
> devices, as they depend on the platform and not so much on the device
> itself.
> 
> Can you do this special-case handling from device independent platform
> code, where you also setup dma_ops for other devices?
> 

Yes, fixed in v2.

Re: [PATCH 1/4] Provide simple noop dma ops

[Joerg Roedel]

On Fri, Oct 30, 2015 at 01:55:56PM +0100, Christian Borntraeger wrote:
> It not trivial without understanding the dma mask details. Do I read
> the x86 implementation right, that it limits the dma to 32 bit? Then
> we cannot collapse both implementations. Or maybe we can hide this in
> dma_capable. Dont know

No, DMA is not limited to 32bit on x86. Each device has its own
dma_mask, and the requested address+size is checked against it. The
DMA_BIT_MASK(32) check is only to there to print a warning when a 32bit
capable device trys to access memory above 4GB.



	Joerg

Re: [PATCH/RFC 0/4] dma ops and virtio

[Andy Lutomirski]

On Fri, Oct 30, 2015 at 1:25 AM, Cornelia Huck <cornelia.huck@de.ibm.com> wrote:
> On Thu, 29 Oct 2015 15:50:38 -0700
> Andy Lutomirski <luto@amacapital.net> wrote:
>
>> Progress!  After getting that sort-of-working, I figured out what was
>> wrong with my earlier command, and I got that working, too.  Now I
>> get:
>>
>> qemu-system-s390x -fsdev
>> local,id=virtfs1,path=/,security_model=none,readonly -device
>> virtio-9p-ccw,fsdev=virtfs1,mount_tag=/dev/root -M s390-ccw-virtio
>> -nodefaults -device sclpconsole,chardev=console -parallel none -net
>> none -echr 1 -serial none -chardev stdio,id=console,signal=off,mux=on
>> -serial chardev:console -mon chardev=console -vga none -display none
>> -kernel arch/s390/boot/bzImage -append
>> 'init=/home/luto/devel/virtme/virtme/guest/virtme-init
>> psmouse.proto=exps "virtme_stty_con=rows 24 cols 150 iutf8"
>> TERM=xterm-256color rootfstype=9p
>> rootflags=ro,version=9p2000.L,trans=virtio,access=any
>> raid=noautodetect debug'
>
> The commandline looks sane AFAICS.
>
> (...)
>
>> vrfy: device 0.0.0000: rc=0 pgroup=0 mpath=0 vpm=80
>> virtio_ccw 0.0.0000: Failed to set online: -5
>>
>> ^^^ bad news!
>
> I'd like to see where in the onlining process this fails. Could you set
> up qemu tracing for css_* and virtio_ccw_* (instructions in
> qemu/docs/tracing.txt)?

I have a file called events that contains:

css_*
virtio_ccw_*

pointing -trace events= at it results in a trace-<pid> file that's 549
bytes long and contains nothing.  Are wildcards not as well-supported
as the docs suggest?

>
> Which qemu version is this, btw.?
>

git from yesterday.

--Andy



-- 
Andy Lutomirski
AMA Capital Management, LLC

Re: [PATCH/RFC 0/4] dma ops and virtio

[Cornelia Huck]

On Fri, 30 Oct 2015 13:33:07 -0700
Andy Lutomirski <luto@amacapital.net> wrote:

> On Fri, Oct 30, 2015 at 1:25 AM, Cornelia Huck <cornelia.huck@de.ibm.com> wrote:
> > On Thu, 29 Oct 2015 15:50:38 -0700
> > Andy Lutomirski <luto@amacapital.net> wrote:
> >
> >> Progress!  After getting that sort-of-working, I figured out what was
> >> wrong with my earlier command, and I got that working, too.  Now I
> >> get:
> >>
> >> qemu-system-s390x -fsdev
> >> local,id=virtfs1,path=/,security_model=none,readonly -device
> >> virtio-9p-ccw,fsdev=virtfs1,mount_tag=/dev/root -M s390-ccw-virtio
> >> -nodefaults -device sclpconsole,chardev=console -parallel none -net
> >> none -echr 1 -serial none -chardev stdio,id=console,signal=off,mux=on
> >> -serial chardev:console -mon chardev=console -vga none -display none
> >> -kernel arch/s390/boot/bzImage -append
> >> 'init=/home/luto/devel/virtme/virtme/guest/virtme-init
> >> psmouse.proto=exps "virtme_stty_con=rows 24 cols 150 iutf8"
> >> TERM=xterm-256color rootfstype=9p
> >> rootflags=ro,version=9p2000.L,trans=virtio,access=any
> >> raid=noautodetect debug'
> >
> > The commandline looks sane AFAICS.
> >
> > (...)
> >
> >> vrfy: device 0.0.0000: rc=0 pgroup=0 mpath=0 vpm=80
> >> virtio_ccw 0.0.0000: Failed to set online: -5
> >>
> >> ^^^ bad news!
> >
> > I'd like to see where in the onlining process this fails. Could you set
> > up qemu tracing for css_* and virtio_ccw_* (instructions in
> > qemu/docs/tracing.txt)?
> 
> I have a file called events that contains:
> 
> css_*
> virtio_ccw_*
> 
> pointing -trace events= at it results in a trace-<pid> file that's 549
> bytes long and contains nothing.  Are wildcards not as well-supported
> as the docs suggest?

Just tried it, seemed to work for me as expected. And as your messages
indicate, at least some of the css tracepoints are guaranteed to be
hit. Odd.

Can you try the following sophisticated printf debug patch?

diff --git a/hw/s390x/css.c b/hw/s390x/css.c
index c033612..6a87bd6 100644
--- a/hw/s390x/css.c
+++ b/hw/s390x/css.c
@@ -308,6 +308,8 @@ static int css_interpret_ccw(SubchDev *sch, hwaddr ccw_addr)
         sch->ccw_no_data_cnt++;
     }
 
+    fprintf(stderr, "CH DBG: %s: cmd_code=%x\n", __func__, ccw.cmd_code);
+
     /* Look at the command. */
     switch (ccw.cmd_code) {
     case CCW_CMD_NOOP:
@@ -375,6 +377,7 @@ static int css_interpret_ccw(SubchDev *sch, hwaddr ccw_addr)
         }
         break;
     }
+    fprintf(stderr, "CH DBG: %s: ret=%d\n", __func__, ret);
     sch->last_cmd = ccw;
     sch->last_cmd_valid = true;
     if (ret == 0) {


> > Which qemu version is this, btw.?
> >
> 
> git from yesterday.

Hm. Might be worth trying the s390-ccw-virtio-2.4 machine instead.

Re: [PATCH/RFC 0/4] dma ops and virtio

[Andy Lutomirski]

On Mon, Nov 2, 2015 at 3:16 AM, Cornelia Huck <cornelia.huck@de.ibm.com> wrote:
> On Fri, 30 Oct 2015 13:33:07 -0700
> Andy Lutomirski <luto@amacapital.net> wrote:
>
>> On Fri, Oct 30, 2015 at 1:25 AM, Cornelia Huck <cornelia.huck@de.ibm.com> wrote:
>> > On Thu, 29 Oct 2015 15:50:38 -0700
>> > Andy Lutomirski <luto@amacapital.net> wrote:
>> >
>> >> Progress!  After getting that sort-of-working, I figured out what was
>> >> wrong with my earlier command, and I got that working, too.  Now I
>> >> get:
>> >>
>> >> qemu-system-s390x -fsdev
>> >> local,id=virtfs1,path=/,security_model=none,readonly -device
>> >> virtio-9p-ccw,fsdev=virtfs1,mount_tag=/dev/root -M s390-ccw-virtio
>> >> -nodefaults -device sclpconsole,chardev=console -parallel none -net
>> >> none -echr 1 -serial none -chardev stdio,id=console,signal=off,mux=on
>> >> -serial chardev:console -mon chardev=console -vga none -display none
>> >> -kernel arch/s390/boot/bzImage -append
>> >> 'init=/home/luto/devel/virtme/virtme/guest/virtme-init
>> >> psmouse.proto=exps "virtme_stty_con=rows 24 cols 150 iutf8"
>> >> TERM=xterm-256color rootfstype=9p
>> >> rootflags=ro,version=9p2000.L,trans=virtio,access=any
>> >> raid=noautodetect debug'
>> >
>> > The commandline looks sane AFAICS.
>> >
>> > (...)
>> >
>> >> vrfy: device 0.0.0000: rc=0 pgroup=0 mpath=0 vpm=80
>> >> virtio_ccw 0.0.0000: Failed to set online: -5
>> >>
>> >> ^^^ bad news!
>> >
>> > I'd like to see where in the onlining process this fails. Could you set
>> > up qemu tracing for css_* and virtio_ccw_* (instructions in
>> > qemu/docs/tracing.txt)?
>>
>> I have a file called events that contains:
>>
>> css_*
>> virtio_ccw_*
>>
>> pointing -trace events= at it results in a trace-<pid> file that's 549
>> bytes long and contains nothing.  Are wildcards not as well-supported
>> as the docs suggest?
>
> Just tried it, seemed to work for me as expected. And as your messages
> indicate, at least some of the css tracepoints are guaranteed to be
> hit. Odd.
>
> Can you try the following sophisticated printf debug patch?
>
> diff --git a/hw/s390x/css.c b/hw/s390x/css.c
> index c033612..6a87bd6 100644
> --- a/hw/s390x/css.c
> +++ b/hw/s390x/css.c
> @@ -308,6 +308,8 @@ static int css_interpret_ccw(SubchDev *sch, hwaddr ccw_addr)
>          sch->ccw_no_data_cnt++;
>      }
>
> +    fprintf(stderr, "CH DBG: %s: cmd_code=%x\n", __func__, ccw.cmd_code);
> +
>      /* Look at the command. */
>      switch (ccw.cmd_code) {
>      case CCW_CMD_NOOP:
> @@ -375,6 +377,7 @@ static int css_interpret_ccw(SubchDev *sch, hwaddr ccw_addr)
>          }
>          break;
>      }
> +    fprintf(stderr, "CH DBG: %s: ret=%d\n", __func__, ret);
>      sch->last_cmd = ccw;
>      sch->last_cmd_valid = true;
>      if (ret == 0) {
>
>
>> > Which qemu version is this, btw.?
>> >
>>
>> git from yesterday.
>
> Hm. Might be worth trying the s390-ccw-virtio-2.4 machine instead.
>

No change.

With s390-ccw-virtio-2.4, I get:

Initializing cgroup subsys cpuset
Initializing cgroup subsys cpu
Initializing cgroup subsys cpuacct
Linux version 4.3.0-rc7-00008-gff230d6ec6b2
(luto@amaluto.corp.amacapital.net) (gcc version 5.1.1 20150618 (Red
Hat Cross 5.1.1-3) (GCC) ) #344 SMP Fri Oct 30 13:16:13 PDT 2015
setup: Linux is running under KVM in 64-bit mode
setup: Max memory size: 128MB
Zone ranges:
  DMA      [mem 0x0000000000000000-0x000000007fffffff]
  Normal   empty
Movable zone start for each node
Early memory node ranges
  node   0: [mem 0x0000000000000000-0x0000000007ffffff]
Initmem setup node 0 [mem 0x0000000000000000-0x0000000007ffffff]
On node 0 totalpages: 32768
  DMA zone: 512 pages used for memmap
  DMA zone: 0 pages reserved
  DMA zone: 32768 pages, LIFO batch:7
PERCPU: Embedded 466 pages/cpu @0000000007605000 s1868032 r8192 d32512 u1908736
pcpu-alloc: s1868032 r8192 d32512 u1908736 alloc=466*4096
pcpu-alloc: [0] 0 [0] 1
Built 1 zonelists in Zone order, mobility grouping on.  Total pages: 32256
Kernel command line:
init=/home/luto/devel/virtme/virtme/guest/virtme-init
psmouse.proto=exps "virtme_stty_con=rows 45 cols 150 iutf8"
TERM=xterm-256color rootfstype=9p
rootflags=version=9p2000.L,trans=virtio,access=any raid=noautodetect
ro debug
PID hash table entries: 512 (order: 0, 4096 bytes)
Dentry cache hash table entries: 16384 (order: 5, 131072 bytes)
Inode-cache hash table entries: 8192 (order: 4, 65536 bytes)
Memory: 92520K/131072K available (8255K kernel code, 802K rwdata,
3860K rodata, 2384K init, 14382K bss, 38552K reserved, 0K
cma-reserved)
Write protected kernel read-only data: 0x100000 - 0xcd4fff
SLUB: HWalign=256, Order=0-3, MinObjects=0, CPUs=2, Nodes=1
Running RCU self tests
Hierarchical RCU implementation.
    RCU debugfs-based tracing is enabled.
    RCU lockdep checking is enabled.
    Build-time adjustment of leaf fanout to 64.
    RCU restricting CPUs from NR_CPUS=256 to nr_cpu_ids=2.
RCU: Adjusting geometry for rcu_fanout_leaf=64, nr_cpu_ids=2
NR_IRQS:3
clocksource: tod: mask: 0xffffffffffffffff max_cycles:
0x3b0a9be803b0a9, max_idle_ns: 1805497147909793 ns
console [ttyS1] enabled
Lock dependency validator: Copyright (c) 2006 Red Hat, Inc., Ingo Molnar
... MAX_LOCKDEP_SUBCLASSES:  8
... MAX_LOCK_DEPTH:          48
... MAX_LOCKDEP_KEYS:        8191
... CLASSHASH_SIZE:          4096
... MAX_LOCKDEP_ENTRIES:     32768
... MAX_LOCKDEP_CHAINS:      65536
... CHAINHASH_SIZE:          32768
 memory used by lock dependency info: 8671 kB
 per task-struct memory footprint: 2688 bytes
Calibrating delay loop (skipped)... 13370.00 BogoMIPS preset
pid_max: default: 32768 minimum: 301
Mount-cache hash table entries: 512 (order: 0, 4096 bytes)
Mountpoint-cache hash table entries: 512 (order: 0, 4096 bytes)
Initializing cgroup subsys io
Initializing cgroup subsys memory
ftrace: allocating 21000 entries in 83 pages
cpu: 1 configured CPUs, 0 standby CPUs
Brought up 1 CPUs
devtmpfs: initialized
clocksource: jiffies: mask: 0xffffffff max_cycles: 0xffffffff,
max_idle_ns: 19112604462750000 ns
xor: measuring software checksum speed
   8regs     :   152.800 MB/sec
   8regs_prefetch:   152.400 MB/sec
   32regs    :   695.200 MB/sec
   32regs_prefetch:   680.000 MB/sec
xor: using function: 32regs (695.200 MB/sec)
NET: Registered protocol family 16
raid6: int64x1  gen()   519 MB/s
raid6: int64x1  xor()   321 MB/s
raid6: int64x2  gen()   611 MB/s
raid6: int64x2  xor()   357 MB/s
raid6: int64x4  gen()   301 MB/s
raid6: int64x4  xor()   337 MB/s
raid6: int64x8  gen()   290 MB/s
raid6: int64x8  xor()   189 MB/s
raid6: using algorithm int64x2 gen() 611 MB/s
raid6: .... xor() 357 MB/s, rmw enabled
raid6: using intx1 recovery algorithm
SCSI subsystem initialized
CH DBG: css_interpret_ccw: cmd_code=e4
CH DBG: css_interpret_ccw: ret=0
NET: Registered protocol family 2
TCP established hash table entries: 1024 (order: 1, 8192 bytes)
TCP bind hash table entries: 1024 (order: 4, 81920 bytes)
TCP: Hash tables configured (established 1024 bind 1024)
UDP hash table entries: 256 (order: 3, 49152 bytes)
UDP-Lite hash table entries: 256 (order: 3, 49152 bytes)
NET: Registered protocol family 1
hypfs: The hardware system does not support hypfs
hypfs: Initialization of hypfs failed with rc=-61
futex hash table entries: 512 (order: 5, 131072 bytes)
audit: initializing netlink subsys (disabled)
audit: type=2000 audit(1446495676.052:1): initialized
Kprobe smoke test: started
Kprobe smoke test: passed successfully
hugetlbfs: disabling because there are no supported hugepage sizes
fuse init (API version 7.23)
SGI XFS with ACLs, security attributes, realtime, no debug enabled
9p: Installing v9fs 9p2000 file system support
Block layer SCSI generic (bsg) driver version 0.4 loaded (major 253)
io scheduler noop registered
io scheduler deadline registered (default)
io scheduler cfq registered
start plist test
end plist test
hvc_iucv: The z/VM IUCV HVC device driver cannot be used without z/VM
brd: module loaded
st: Version 20101219, fixed bufsize 32768, s/g segs 256
cio: Channel measurement facility initialized using format extended
(mode autodetected)
Discipline DIAG cannot be used without z/VM
qeth: loading core functions
CH DBG: css_interpret_ccw: cmd_code=3
CH DBG: css_interpret_ccw: ret=0
CH DBG: css_interpret_ccw: cmd_code=83
CH DBG: css_interpret_ccw: ret=-38
qeth: register layer 2 discipline
qeth: register layer 3 discipline
oprofile: using timer interrupt.
NET: Registered protocol family 10
virtio_ccw 0.0.0000: Failed to set online: -5
sit: IPv6 over IPv4 tunneling driver
NET: Registered protocol family 17
NET: Registered protocol family 15
8021q: 802.1Q VLAN Support v1.8
9pnet: Installing 9P2000 support
registered taskstats version 1
Btrfs loaded
9pnet_virtio: no channels available

The lack of much interesting output makes me think that maybe I
misconfigured something.

--Andy

Re: [PATCH/RFC 0/4] dma ops and virtio

[Christian Borntraeger]

Am 02.11.2015 um 21:23 schrieb Andy Lutomirski:
> On Mon, Nov 2, 2015 at 3:16 AM, Cornelia Huck <cornelia.huck@de.ibm.com> wrote:
>> On Fri, 30 Oct 2015 13:33:07 -0700
>> Andy Lutomirski <luto@amacapital.net> wrote:
>>
>>> On Fri, Oct 30, 2015 at 1:25 AM, Cornelia Huck <cornelia.huck@de.ibm.com> wrote:
>>>> On Thu, 29 Oct 2015 15:50:38 -0700
>>>> Andy Lutomirski <luto@amacapital.net> wrote:
>>>>
>>>>> Progress!  After getting that sort-of-working, I figured out what was
>>>>> wrong with my earlier command, and I got that working, too.  Now I
>>>>> get:
>>>>>
>>>>> qemu-system-s390x -fsdev
>>>>> local,id=virtfs1,path=/,security_model=none,readonly -device
>>>>> virtio-9p-ccw,fsdev=virtfs1,mount_tag=/dev/root -M s390-ccw-virtio
>>>>> -nodefaults -device sclpconsole,chardev=console -parallel none -net
>>>>> none -echr 1 -serial none -chardev stdio,id=console,signal=off,mux=on
>>>>> -serial chardev:console -mon chardev=console -vga none -display none
>>>>> -kernel arch/s390/boot/bzImage -append
>>>>> 'init=/home/luto/devel/virtme/virtme/guest/virtme-init
>>>>> psmouse.proto=exps "virtme_stty_con=rows 24 cols 150 iutf8"
>>>>> TERM=xterm-256color rootfstype=9p
>>>>> rootflags=ro,version=9p2000.L,trans=virtio,access=any
>>>>> raid=noautodetect debug'
>>>>
>>>> The commandline looks sane AFAICS.
>>>>
>>>> (...)
>>>>
>>>>> vrfy: device 0.0.0000: rc=0 pgroup=0 mpath=0 vpm=80
>>>>> virtio_ccw 0.0.0000: Failed to set online: -5
>>>>>
>>>>> ^^^ bad news!
>>>>
>>>> I'd like to see where in the onlining process this fails. Could you set
>>>> up qemu tracing for css_* and virtio_ccw_* (instructions in
>>>> qemu/docs/tracing.txt)?
>>>
>>> I have a file called events that contains:
>>>
>>> css_*
>>> virtio_ccw_*
>>>
>>> pointing -trace events= at it results in a trace-<pid> file that's 549
>>> bytes long and contains nothing.  Are wildcards not as well-supported
>>> as the docs suggest?
>>
>> Just tried it, seemed to work for me as expected. And as your messages
>> indicate, at least some of the css tracepoints are guaranteed to be
>> hit. Odd.
>>
>> Can you try the following sophisticated printf debug patch?
>>
>> diff --git a/hw/s390x/css.c b/hw/s390x/css.c
>> index c033612..6a87bd6 100644
>> --- a/hw/s390x/css.c
>> +++ b/hw/s390x/css.c
>> @@ -308,6 +308,8 @@ static int css_interpret_ccw(SubchDev *sch, hwaddr ccw_addr)
>>          sch->ccw_no_data_cnt++;
>>      }
>>
>> +    fprintf(stderr, "CH DBG: %s: cmd_code=%x\n", __func__, ccw.cmd_code);
>> +
>>      /* Look at the command. */
>>      switch (ccw.cmd_code) {
>>      case CCW_CMD_NOOP:
>> @@ -375,6 +377,7 @@ static int css_interpret_ccw(SubchDev *sch, hwaddr ccw_addr)
>>          }
>>          break;
>>      }
>> +    fprintf(stderr, "CH DBG: %s: ret=%d\n", __func__, ret);
>>      sch->last_cmd = ccw;
>>      sch->last_cmd_valid = true;
>>      if (ret == 0) {
>>
>>
>>>> Which qemu version is this, btw.?
>>>>
>>>
>>> git from yesterday.
>>
>> Hm. Might be worth trying the s390-ccw-virtio-2.4 machine instead.
>>
> 
> No change.
> 
> With s390-ccw-virtio-2.4, I get:
> 
> Initializing cgroup subsys cpuset
> Initializing cgroup subsys cpu
> Initializing cgroup subsys cpuacct
> Linux version 4.3.0-rc7-00008-gff230d6ec6b2
> (luto@amaluto.corp.amacapital.net) (gcc version 5.1.1 20150618 (Red
> Hat Cross 5.1.1-3) (GCC) ) #344 SMP Fri Oct 30 13:16:13 PDT 2015
> setup: Linux is running under KVM in 64-bit mode
> setup: Max memory size: 128MB
> Zone ranges:
>   DMA      [mem 0x0000000000000000-0x000000007fffffff]
>   Normal   empty
> Movable zone start for each node
> Early memory node ranges
>   node   0: [mem 0x0000000000000000-0x0000000007ffffff]
> Initmem setup node 0 [mem 0x0000000000000000-0x0000000007ffffff]
> On node 0 totalpages: 32768
>   DMA zone: 512 pages used for memmap
>   DMA zone: 0 pages reserved
>   DMA zone: 32768 pages, LIFO batch:7
> PERCPU: Embedded 466 pages/cpu @0000000007605000 s1868032 r8192 d32512 u1908736
> pcpu-alloc: s1868032 r8192 d32512 u1908736 alloc=466*4096
> pcpu-alloc: [0] 0 [0] 1
> Built 1 zonelists in Zone order, mobility grouping on.  Total pages: 32256
> Kernel command line:
> init=/home/luto/devel/virtme/virtme/guest/virtme-init
> psmouse.proto=exps "virtme_stty_con=rows 45 cols 150 iutf8"
> TERM=xterm-256color rootfstype=9p
> rootflags=version=9p2000.L,trans=virtio,access=any raid=noautodetect
> ro debug
> PID hash table entries: 512 (order: 0, 4096 bytes)
> Dentry cache hash table entries: 16384 (order: 5, 131072 bytes)
> Inode-cache hash table entries: 8192 (order: 4, 65536 bytes)
> Memory: 92520K/131072K available (8255K kernel code, 802K rwdata,


can you send your kernel config?

Re: [PATCH/RFC 0/4] dma ops and virtio

[Andy Lutomirski]

[-- Attachment #1: Type: text/plain, Size: 5599 bytes --]

On Tue, Nov 3, 2015 at 12:14 AM, Christian Borntraeger
<borntraeger@de.ibm.com> wrote:
> Am 02.11.2015 um 21:23 schrieb Andy Lutomirski:
>> On Mon, Nov 2, 2015 at 3:16 AM, Cornelia Huck <cornelia.huck@de.ibm.com> wrote:
>>> On Fri, 30 Oct 2015 13:33:07 -0700
>>> Andy Lutomirski <luto@amacapital.net> wrote:
>>>
>>>> On Fri, Oct 30, 2015 at 1:25 AM, Cornelia Huck <cornelia.huck@de.ibm.com> wrote:
>>>>> On Thu, 29 Oct 2015 15:50:38 -0700
>>>>> Andy Lutomirski <luto@amacapital.net> wrote:
>>>>>
>>>>>> Progress!  After getting that sort-of-working, I figured out what was
>>>>>> wrong with my earlier command, and I got that working, too.  Now I
>>>>>> get:
>>>>>>
>>>>>> qemu-system-s390x -fsdev
>>>>>> local,id=virtfs1,path=/,security_model=none,readonly -device
>>>>>> virtio-9p-ccw,fsdev=virtfs1,mount_tag=/dev/root -M s390-ccw-virtio
>>>>>> -nodefaults -device sclpconsole,chardev=console -parallel none -net
>>>>>> none -echr 1 -serial none -chardev stdio,id=console,signal=off,mux=on
>>>>>> -serial chardev:console -mon chardev=console -vga none -display none
>>>>>> -kernel arch/s390/boot/bzImage -append
>>>>>> 'init=/home/luto/devel/virtme/virtme/guest/virtme-init
>>>>>> psmouse.proto=exps "virtme_stty_con=rows 24 cols 150 iutf8"
>>>>>> TERM=xterm-256color rootfstype=9p
>>>>>> rootflags=ro,version=9p2000.L,trans=virtio,access=any
>>>>>> raid=noautodetect debug'
>>>>>
>>>>> The commandline looks sane AFAICS.
>>>>>
>>>>> (...)
>>>>>
>>>>>> vrfy: device 0.0.0000: rc=0 pgroup=0 mpath=0 vpm=80
>>>>>> virtio_ccw 0.0.0000: Failed to set online: -5
>>>>>>
>>>>>> ^^^ bad news!
>>>>>
>>>>> I'd like to see where in the onlining process this fails. Could you set
>>>>> up qemu tracing for css_* and virtio_ccw_* (instructions in
>>>>> qemu/docs/tracing.txt)?
>>>>
>>>> I have a file called events that contains:
>>>>
>>>> css_*
>>>> virtio_ccw_*
>>>>
>>>> pointing -trace events= at it results in a trace-<pid> file that's 549
>>>> bytes long and contains nothing.  Are wildcards not as well-supported
>>>> as the docs suggest?
>>>
>>> Just tried it, seemed to work for me as expected. And as your messages
>>> indicate, at least some of the css tracepoints are guaranteed to be
>>> hit. Odd.
>>>
>>> Can you try the following sophisticated printf debug patch?
>>>
>>> diff --git a/hw/s390x/css.c b/hw/s390x/css.c
>>> index c033612..6a87bd6 100644
>>> --- a/hw/s390x/css.c
>>> +++ b/hw/s390x/css.c
>>> @@ -308,6 +308,8 @@ static int css_interpret_ccw(SubchDev *sch, hwaddr ccw_addr)
>>>          sch->ccw_no_data_cnt++;
>>>      }
>>>
>>> +    fprintf(stderr, "CH DBG: %s: cmd_code=%x\n", __func__, ccw.cmd_code);
>>> +
>>>      /* Look at the command. */
>>>      switch (ccw.cmd_code) {
>>>      case CCW_CMD_NOOP:
>>> @@ -375,6 +377,7 @@ static int css_interpret_ccw(SubchDev *sch, hwaddr ccw_addr)
>>>          }
>>>          break;
>>>      }
>>> +    fprintf(stderr, "CH DBG: %s: ret=%d\n", __func__, ret);
>>>      sch->last_cmd = ccw;
>>>      sch->last_cmd_valid = true;
>>>      if (ret == 0) {
>>>
>>>
>>>>> Which qemu version is this, btw.?
>>>>>
>>>>
>>>> git from yesterday.
>>>
>>> Hm. Might be worth trying the s390-ccw-virtio-2.4 machine instead.
>>>
>>
>> No change.
>>
>> With s390-ccw-virtio-2.4, I get:
>>
>> Initializing cgroup subsys cpuset
>> Initializing cgroup subsys cpu
>> Initializing cgroup subsys cpuacct
>> Linux version 4.3.0-rc7-00008-gff230d6ec6b2
>> (luto@amaluto.corp.amacapital.net) (gcc version 5.1.1 20150618 (Red
>> Hat Cross 5.1.1-3) (GCC) ) #344 SMP Fri Oct 30 13:16:13 PDT 2015
>> setup: Linux is running under KVM in 64-bit mode
>> setup: Max memory size: 128MB
>> Zone ranges:
>>   DMA      [mem 0x0000000000000000-0x000000007fffffff]
>>   Normal   empty
>> Movable zone start for each node
>> Early memory node ranges
>>   node   0: [mem 0x0000000000000000-0x0000000007ffffff]
>> Initmem setup node 0 [mem 0x0000000000000000-0x0000000007ffffff]
>> On node 0 totalpages: 32768
>>   DMA zone: 512 pages used for memmap
>>   DMA zone: 0 pages reserved
>>   DMA zone: 32768 pages, LIFO batch:7
>> PERCPU: Embedded 466 pages/cpu @0000000007605000 s1868032 r8192 d32512 u1908736
>> pcpu-alloc: s1868032 r8192 d32512 u1908736 alloc=466*4096
>> pcpu-alloc: [0] 0 [0] 1
>> Built 1 zonelists in Zone order, mobility grouping on.  Total pages: 32256
>> Kernel command line:
>> init=/home/luto/devel/virtme/virtme/guest/virtme-init
>> psmouse.proto=exps "virtme_stty_con=rows 45 cols 150 iutf8"
>> TERM=xterm-256color rootfstype=9p
>> rootflags=version=9p2000.L,trans=virtio,access=any raid=noautodetect
>> ro debug
>> PID hash table entries: 512 (order: 0, 4096 bytes)
>> Dentry cache hash table entries: 16384 (order: 5, 131072 bytes)
>> Inode-cache hash table entries: 8192 (order: 4, 65536 bytes)
>> Memory: 92520K/131072K available (8255K kernel code, 802K rwdata,
>
>
> can you send your kernel config?
>

Attached.

A failing command looks like:

qemu-system-s390x -fsdev
local,id=virtfs1,path=/,security_model=none,readonly -device
virtio-9p-ccw,fsdev=virtfs1,mount_tag=/dev/root -M s390-ccw-virtio
-nodefaults -device sclpconsole,chardev=console -parallel none -net
none -echr 1 -serial none -chardev stdio,id=console,signal=off,mux=on
-serial chardev:console -mon chardev=console -vga none -display none
-kernel arch/s390/boot/bzImage -append
'init=/home/luto/devel/virtme/virtme/guest/virtme-init
psmouse.proto=exps "virtme_stty_con=rows 24 cols 80 iutf8"
TERM=xterm-256color rootfstype=9p
rootflags=version=9p2000.L,trans=virtio,access=any raid=noautodetect
ro debug'

I'm testing that from an x86_64 host, so this is TCG and not KVM.

--Andy

[-- Attachment #2: s390-config.xz --]
[-- Type: application/x-xz, Size: 8652 bytes --]

Re: [PATCH/RFC 0/4] dma ops and virtio

[Cornelia Huck]

On Mon, 2 Nov 2015 12:23:25 -0800
Andy Lutomirski <luto@amacapital.net> wrote:

> No change.

I'm stumped :(

Here's what I see:

(...)

> CH DBG: css_interpret_ccw: cmd_code=e4
> CH DBG: css_interpret_ccw: ret=0

sense id -> works

(...)

> CH DBG: css_interpret_ccw: cmd_code=3
> CH DBG: css_interpret_ccw: ret=0

nop (path verification) -> works

> CH DBG: css_interpret_ccw: cmd_code=83
> CH DBG: css_interpret_ccw: ret=-38

set revision -> -ENOSYS

This is fine; the virtio device is in legacy mode and the kernel will
try revision 1; qemu will reject this. The code should end up
generating a unit check with command reject, however...

> qeth: register layer 2 discipline
> qeth: register layer 3 discipline
> oprofile: using timer interrupt.
> NET: Registered protocol family 10
> virtio_ccw 0.0.0000: Failed to set online: -5

...this shows the kernel driver somehow did not end up with that
command reject (it would have triggered a retry with revision 0, and
the return code shows that no unit check/command reject was detected,
but some other error.)

> The lack of much interesting output makes me think that maybe I
> misconfigured something.

It's just failing very early in the setup phase. As it works for me
with a kvm setup, I'm suspecting some error in qemu's emulation code,
which is unfortunately not my turf.

Some more poke-around-in-the-dark ideas:

- Do you get more debug out put when you switch back to s390-ccw-virtio
(virtio-1), i.e. does cmd 83 work and is it followed by further
commands?

- Can you try with the following qemu logging patch

-----8<----------8<-----

diff --git a/hw/s390x/css.c b/hw/s390x/css.c
index c033612..80853a6 100644
--- a/hw/s390x/css.c
+++ b/hw/s390x/css.c
@@ -868,6 +868,7 @@ int css_do_tsch_get_irb(SubchDev *sch, IRB *target_irb, int *irb_len)
     PMCW *p = &sch->curr_status.pmcw;
     uint16_t stctl;
     IRB irb;
+    int i;
 
     if (!(p->flags & (PMCW_FLAGS_MASK_DNV | PMCW_FLAGS_MASK_ENA))) {
         return 3;
@@ -898,6 +899,14 @@ int css_do_tsch_get_irb(SubchDev *sch, IRB *target_irb, int *irb_len)
         }
     }
     /* Store the irb to the guest. */
+    fprintf(stderr, "CH DBG: %s: flags=%04x ctrl=%04x cpa=%08x\n",
+            __func__, irb.scsw.flags, irb.scsw.ctrl, irb.scsw.cpa);
+    fprintf(stderr, "CH DBG: %s: dstat=%02x cstat=%02x count=%04x\n",
+            __func__, irb.scsw.dstat, irb.scsw.cstat, irb.scsw.count);
+    for (i = 0; i < ARRAY_SIZE(irb.ecw); i++) {
+        fprintf(stderr, "CH DBG: %s: ecw[%d]=%08x\n", __func__,
+                i, irb.ecw[i]);
+    }
     copy_irb_to_guest(target_irb, &irb, p, irb_len);
 
     return ((stctl & SCSW_STCTL_STATUS_PEND) == 0);

-----8<----------8<-----

and the following kernel patch

-----8<----------8<-----

diff --git a/drivers/s390/cio/device_fsm.c b/drivers/s390/cio/device_fsm.c
index 83da53c..ea4db09 100644
--- a/drivers/s390/cio/device_fsm.c
+++ b/drivers/s390/cio/device_fsm.c
@@ -540,6 +540,9 @@ callback:
 			create_fake_irb(&cdev->private->irb,
 					cdev->private->flags.fake_irb);
 			cdev->private->flags.fake_irb = 0;
+			CIO_TRACE_EVENT(0, "fake_irb");
+			CIO_HEX_EVENT(0, &cdev->private->irb,
+				      sizeof(struct irb));
 			if (cdev->handler)
 				cdev->handler(cdev, cdev->private->intparm,
 					      &cdev->private->irb);
diff --git a/drivers/s390/cio/device_ops.c b/drivers/s390/cio/device_ops.c
index 6acd0b5..e9bf357 100644
--- a/drivers/s390/cio/device_ops.c
+++ b/drivers/s390/cio/device_ops.c
@@ -446,6 +446,8 @@ ccw_device_call_handler(struct ccw_device *cdev)
 	/*
 	 * Now we are ready to call the device driver interrupt handler.
 	 */
+	CIO_TRACE_EVENT(0, "irb");
+	CIO_HEX_EVENT(0, &cdev->private->irb, sizeof(struct irb));
 	if (cdev->handler)
 		cdev->handler(cdev, cdev->private->intparm,
 			      &cdev->private->irb);

-----8<----------8<-----

Just to verify that qemu will produce and the kernel end up with the
irb I'd expect. I'd rather prefer us getting the dma stuff right
instead of chasing qemu issues :/

Re: [PATCH/RFC 0/4] dma ops and virtio

[Andy Lutomirski]

On Tue, Nov 3, 2015 at 9:59 AM, Cornelia Huck <cornelia.huck@de.ibm.com> wrote:
> It's just failing very early in the setup phase. As it works for me
> with a kvm setup, I'm suspecting some error in qemu's emulation code,
> which is unfortunately not my turf.
>

That should be easy to rule out.  Can you try with -machine accel=tcg?
 I can't test with kvm for obvious reasons.

> Some more poke-around-in-the-dark ideas:
>
> - Do you get more debug out put when you switch back to s390-ccw-virtio
> (virtio-1), i.e. does cmd 83 work and is it followed by further
> commands?

I'll play with this stuff later today.

--Andy

Re: [PATCH/RFC 0/4] dma ops and virtio

[Cornelia Huck]

On Tue, 3 Nov 2015 10:45:12 -0800
Andy Lutomirski <luto@amacapital.net> wrote:

> On Tue, Nov 3, 2015 at 9:59 AM, Cornelia Huck <cornelia.huck@de.ibm.com> wrote:
> > It's just failing very early in the setup phase. As it works for me
> > with a kvm setup, I'm suspecting some error in qemu's emulation code,
> > which is unfortunately not my turf.
> >
> 
> That should be easy to rule out.  Can you try with -machine accel=tcg?
>  I can't test with kvm for obvious reasons.

Well, s390-on-s390 works (at least with
https://patchwork.ozlabs.org/patch/538882/ applied).

I'm currently suspecting some endianness issues, probably with the ecw
accesses, which is why I'd be interested in your trace information (as
I currently don't have a LE test setup at hand.)

Re: [PATCH/RFC 0/4] dma ops and virtio

[Cornelia Huck]

On Wed, 4 Nov 2015 15:29:23 +0100
Cornelia Huck <cornelia.huck@de.ibm.com> wrote:

> I'm currently suspecting some endianness issues, probably with the ecw
> accesses, which is why I'd be interested in your trace information (as
> I currently don't have a LE test setup at hand.)

I think I've got it. We have sense_data as a byte array, which
implicitly makes it BE already. When we copy to the ecws while building
the irb, the data ends up in 32 bit values. The conversion from host
endianness to BE now treats them as LE on your system...

Could you please give the following qemu patch a try?

diff --git a/hw/s390x/css.c b/hw/s390x/css.c
index c033612..a13494e 100644
--- a/hw/s390x/css.c
+++ b/hw/s390x/css.c
@@ -892,8 +892,16 @@ int css_do_tsch_get_irb(SubchDev *sch, IRB *target_irb, int *irb_len)
         /* If a unit check is pending, copy sense data. */
         if ((s->dstat & SCSW_DSTAT_UNIT_CHECK) &&
             (p->chars & PMCW_CHARS_MASK_CSENSE)) {
+            uint32_t ecw[8];
+            int i;
+
             irb.scsw.flags |= SCSW_FLAGS_MASK_ESWF | SCSW_FLAGS_MASK_ECTL;
-            memcpy(irb.ecw, sch->sense_data, sizeof(sch->sense_data));
+            /* Attention: sense_data is already BE! */
+            memset(ecw, 0, sizeof(ecw));
+            memcpy(ecw, sch->sense_data, sizeof(sch->sense_data));
+            for (i = 0; i < ARRAY_SIZE(ecw); i++) {
+                irb.ecw[i] = be32_to_cpu(ecw[i]);
+            }
             irb.esw[1] = 0x01000000 | (sizeof(sch->sense_data) << 8);
         }
     }

Re: [PATCH/RFC 0/4] dma ops and virtio

[Andy Lutomirski]

On Wed, Nov 4, 2015 at 9:52 AM, Cornelia Huck <cornelia.huck@de.ibm.com> wrote:
> On Wed, 4 Nov 2015 15:29:23 +0100
> Cornelia Huck <cornelia.huck@de.ibm.com> wrote:
>
>> I'm currently suspecting some endianness issues, probably with the ecw
>> accesses, which is why I'd be interested in your trace information (as
>> I currently don't have a LE test setup at hand.)
>
> I think I've got it. We have sense_data as a byte array, which
> implicitly makes it BE already. When we copy to the ecws while building
> the irb, the data ends up in 32 bit values. The conversion from host
> endianness to BE now treats them as LE on your system...
>
> Could you please give the following qemu patch a try?

Tested-by: Andy Lutomirski <luto@kernel.org>

Now my test script panics for the right reason (init isn't actually an
s390 binary).  Thanks!

I'll update my branch with the latest s390 DMA stuff, and now I can
give it at least basic regression testing.  Of course, I have to find
a root filesystem, too. :)

--Andy

Re: [PATCH/RFC 0/4] dma ops and virtio

[Cornelia Huck]

On Wed, 4 Nov 2015 10:14:11 -0800
Andy Lutomirski <luto@amacapital.net> wrote:

> On Wed, Nov 4, 2015 at 9:52 AM, Cornelia Huck <cornelia.huck@de.ibm.com> wrote:
> > On Wed, 4 Nov 2015 15:29:23 +0100
> > Cornelia Huck <cornelia.huck@de.ibm.com> wrote:
> >
> >> I'm currently suspecting some endianness issues, probably with the ecw
> >> accesses, which is why I'd be interested in your trace information (as
> >> I currently don't have a LE test setup at hand.)
> >
> > I think I've got it. We have sense_data as a byte array, which
> > implicitly makes it BE already. When we copy to the ecws while building
> > the irb, the data ends up in 32 bit values. The conversion from host
> > endianness to BE now treats them as LE on your system...
> >
> > Could you please give the following qemu patch a try?
> 
> Tested-by: Andy Lutomirski <luto@kernel.org>
> 
> Now my test script panics for the right reason (init isn't actually an
> s390 binary).  Thanks!

Cool, thanks for testing! I'll get this into qemu as a proper patch.

Re: [PATCH/RFC 0/4] dma ops and virtio

[Cornelia Huck]

On Thu, 5 Nov 2015 09:11:06 +0100
Cornelia Huck <cornelia.huck@de.ibm.com> wrote:

> On Wed, 4 Nov 2015 10:14:11 -0800
> Andy Lutomirski <luto@amacapital.net> wrote:
> 
> > On Wed, Nov 4, 2015 at 9:52 AM, Cornelia Huck <cornelia.huck@de.ibm.com> wrote:
> > > On Wed, 4 Nov 2015 15:29:23 +0100
> > > Cornelia Huck <cornelia.huck@de.ibm.com> wrote:
> > >
> > >> I'm currently suspecting some endianness issues, probably with the ecw
> > >> accesses, which is why I'd be interested in your trace information (as
> > >> I currently don't have a LE test setup at hand.)
> > >
> > > I think I've got it. We have sense_data as a byte array, which
> > > implicitly makes it BE already. When we copy to the ecws while building
> > > the irb, the data ends up in 32 bit values. The conversion from host
> > > endianness to BE now treats them as LE on your system...
> > >
> > > Could you please give the following qemu patch a try?
> > 
> > Tested-by: Andy Lutomirski <luto@kernel.org>
> > 
> > Now my test script panics for the right reason (init isn't actually an
> > s390 binary).  Thanks!
> 
> Cool, thanks for testing! I'll get this into qemu as a proper patch.

FYI: The patch has been included into the qemu git tree.