[PATCH RFC 0/3] xen: add hypfs per-domain abi-features

[PATCH RFC 0/3] xen: add hypfs per-domain abi-features

[Juergen Gross]

This small series is meant as an example how to add further dynamical
directories to hypfs. It can be used to replace Paul's current approach
to specify ABI-features via domain create flags and replace those by
hypfs nodes.

The related libxl part could just use libxenhypfs to set the values.

This series is meant to be applied on top of my series:

  xen: support per-cpupool scheduling granularity

Juergen Gross (3):
  xen/hypfs: add support for bool leafs in dynamic directories
  xen/domain: add domain hypfs directories
  xen/domain: add per-domain hypfs directory abi-features

 docs/misc/hypfs-paths.pandoc |  10 ++
 xen/common/Makefile          |   1 +
 xen/common/hypfs.c           |  53 +++++++++--
 xen/common/hypfs_dom.c       | 176 +++++++++++++++++++++++++++++++++++
 xen/include/xen/hypfs.h      |   7 ++
 xen/include/xen/sched.h      |   4 +
 6 files changed, 244 insertions(+), 7 deletions(-)
 create mode 100644 xen/common/hypfs_dom.c

-- 
2.26.2

[PATCH RFC 1/3] xen/hypfs: add support for bool leafs in dynamic directories

[Juergen Gross]

Add support for reading fixed sized leafs and writing bool leafs in
dynamic directories with the backing variable being a member of the
structure anchored in struct hypfs_dyndir->data.

This adds the related leaf read and write functions and a helper
macro HYPFS_STRUCT_ELEM() for referencing the element.

Signed-off-by: Juergen Gross <jgross@suse.com>
---
 xen/common/hypfs.c      | 53 +++++++++++++++++++++++++++++++++++------
 xen/include/xen/hypfs.h |  7 ++++++
 2 files changed, 53 insertions(+), 7 deletions(-)

diff --git a/xen/common/hypfs.c b/xen/common/hypfs.c
index 087c63b92f..caee48cc97 100644
--- a/xen/common/hypfs.c
+++ b/xen/common/hypfs.c
@@ -501,17 +501,26 @@ int hypfs_read_dir(const struct hypfs_entry *entry,
     return 0;
 }
 
-int hypfs_read_leaf(const struct hypfs_entry *entry,
-                    XEN_GUEST_HANDLE_PARAM(void) uaddr)
+static int hypfs_read_leaf_off(const struct hypfs_entry *entry,
+                               XEN_GUEST_HANDLE_PARAM(void) uaddr,
+                               void *off)
 {
     const struct hypfs_entry_leaf *l;
     unsigned int size = entry->funcs->getsize(entry);
+    const void *ptr;
 
     ASSERT(this_cpu(hypfs_locked) != hypfs_unlocked);
 
     l = container_of(entry, const struct hypfs_entry_leaf, e);
+    ptr = off ? off + (unsigned long)l->u.content : l->u.content;
+
+    return copy_to_guest(uaddr, ptr, size) ? -EFAULT : 0;
+}
 
-    return copy_to_guest(uaddr, l->u.content, size) ?  -EFAULT : 0;
+int hypfs_read_leaf(const struct hypfs_entry *entry,
+                    XEN_GUEST_HANDLE_PARAM(void) uaddr)
+{
+    return hypfs_read_leaf_off(entry, uaddr, NULL);
 }
 
 static int hypfs_read(const struct hypfs_entry *entry,
@@ -587,11 +596,12 @@ int hypfs_write_leaf(struct hypfs_entry_leaf *leaf,
     return ret;
 }
 
-int hypfs_write_bool(struct hypfs_entry_leaf *leaf,
-                     XEN_GUEST_HANDLE_PARAM(const_void) uaddr,
-                     unsigned int ulen)
+static int hypfs_write_bool_off(struct hypfs_entry_leaf *leaf,
+                                XEN_GUEST_HANDLE_PARAM(const_void) uaddr,
+                                unsigned int ulen, void *off)
 {
     bool buf;
+    bool *ptr;
 
     ASSERT(this_cpu(hypfs_locked) == hypfs_write_locked);
     ASSERT(leaf->e.type == XEN_HYPFS_TYPE_BOOL &&
@@ -604,11 +614,19 @@ int hypfs_write_bool(struct hypfs_entry_leaf *leaf,
     if ( copy_from_guest(&buf, uaddr, ulen) )
         return -EFAULT;
 
-    *(bool *)leaf->u.write_ptr = buf;
+    ptr = off ? off + (unsigned long)leaf->u.write_ptr : leaf->u.write_ptr;
+    *ptr = buf;
 
     return 0;
 }
 
+int hypfs_write_bool(struct hypfs_entry_leaf *leaf,
+                     XEN_GUEST_HANDLE_PARAM(const_void) uaddr,
+                     unsigned int ulen)
+{
+    return hypfs_write_bool_off(leaf, uaddr, ulen, NULL);
+}
+
 int hypfs_write_custom(struct hypfs_entry_leaf *leaf,
                        XEN_GUEST_HANDLE_PARAM(const_void) uaddr,
                        unsigned int ulen)
@@ -644,6 +662,27 @@ int hypfs_write_custom(struct hypfs_entry_leaf *leaf,
     return ret;
 }
 
+int hypfs_dyndir_id_read_leaf(const struct hypfs_entry *entry,
+                              XEN_GUEST_HANDLE_PARAM(void) uaddr)
+{
+    struct hypfs_dyndir_id *dyndata;
+
+    dyndata = hypfs_get_dyndata();
+
+    return hypfs_read_leaf_off(entry, uaddr, dyndata->data);
+}
+
+int hypfs_dyndir_id_write_bool(struct hypfs_entry_leaf *leaf,
+                               XEN_GUEST_HANDLE_PARAM(const_void) uaddr,
+                               unsigned int ulen)
+{
+    struct hypfs_dyndir_id *dyndata;
+
+    dyndata = hypfs_get_dyndata();
+
+    return hypfs_write_bool_off(leaf, uaddr, ulen, dyndata->data);
+}
+
 int hypfs_write_deny(struct hypfs_entry_leaf *leaf,
                      XEN_GUEST_HANDLE_PARAM(const_void) uaddr,
                      unsigned int ulen)
diff --git a/xen/include/xen/hypfs.h b/xen/include/xen/hypfs.h
index 34073faff8..670dc68b48 100644
--- a/xen/include/xen/hypfs.h
+++ b/xen/include/xen/hypfs.h
@@ -160,6 +160,8 @@ static inline void hypfs_string_set_reference(struct hypfs_entry_leaf *leaf,
     HYPFS_FIXEDSIZE_INIT(var, XEN_HYPFS_TYPE_BOOL, nam, contvar, \
                          &hypfs_bool_wr_funcs, 1)
 
+#define HYPFS_STRUCT_ELEM(type, elem)    (((type *)NULL)->elem)
+
 extern struct hypfs_entry_dir hypfs_root;
 
 int hypfs_add_dir(struct hypfs_entry_dir *parent,
@@ -204,6 +206,11 @@ struct hypfs_entry *hypfs_gen_dyndir_id_entry(
     const struct hypfs_entry_dir *template, unsigned int id, void *data);
 unsigned int hypfs_dynid_entry_size(const struct hypfs_entry *template,
                                     unsigned int id);
+int hypfs_dyndir_id_read_leaf(const struct hypfs_entry *entry,
+                              XEN_GUEST_HANDLE_PARAM(void) uaddr);
+int hypfs_dyndir_id_write_bool(struct hypfs_entry_leaf *leaf,
+                               XEN_GUEST_HANDLE_PARAM(const_void) uaddr,
+                               unsigned int ulen);
 #endif
 
 #endif /* __XEN_HYPFS_H__ */
-- 
2.26.2

[PATCH RFC 2/3] xen/domain: add domain hypfs directories

[Juergen Gross]

Add /domain/<domid> directories to hypfs. Those are completely
dynamic, so the related hypfs access functions need to be implemented.

Signed-off-by: Juergen Gross <jgross@suse.com>
---
V3:
- new patch
---
 docs/misc/hypfs-paths.pandoc |  10 +++
 xen/common/Makefile          |   1 +
 xen/common/hypfs_dom.c       | 137 +++++++++++++++++++++++++++++++++++
 3 files changed, 148 insertions(+)
 create mode 100644 xen/common/hypfs_dom.c

diff --git a/docs/misc/hypfs-paths.pandoc b/docs/misc/hypfs-paths.pandoc
index e86f7d0dbe..116642e367 100644
--- a/docs/misc/hypfs-paths.pandoc
+++ b/docs/misc/hypfs-paths.pandoc
@@ -34,6 +34,7 @@ not containing any '/' character. The names "." and ".." are reserved
 for file system internal use.
 
 VALUES are strings and can take the following forms (note that this represents
+>>>>>>> patched
 only the syntax used in this document):
 
 * STRING -- an arbitrary 0-delimited byte string.
@@ -191,6 +192,15 @@ The scheduling granularity of a cpupool.
 Writing a value is allowed only for cpupools with no cpu assigned and if the
 architecture is supporting different scheduling granularities.
 
+#### /domain/
+
+A directory of all current domains.
+
+#### /domain/*/
+
+The individual domains. Each entry is a directory with the name being the
+domain-id (e.g. /domain/0/).
+
 #### /params/
 
 A directory of runtime parameters.
diff --git a/xen/common/Makefile b/xen/common/Makefile
index d109f279a4..e88a9ee91e 100644
--- a/xen/common/Makefile
+++ b/xen/common/Makefile
@@ -15,6 +15,7 @@ obj-$(CONFIG_GRANT_TABLE) += grant_table.o
 obj-y += guestcopy.o
 obj-bin-y += gunzip.init.o
 obj-$(CONFIG_HYPFS) += hypfs.o
+obj-$(CONFIG_HYPFS) += hypfs_dom.o
 obj-y += irq.o
 obj-y += kernel.o
 obj-y += keyhandler.o
diff --git a/xen/common/hypfs_dom.c b/xen/common/hypfs_dom.c
new file mode 100644
index 0000000000..241e379b24
--- /dev/null
+++ b/xen/common/hypfs_dom.c
@@ -0,0 +1,137 @@
+/******************************************************************************
+ *
+ * hypfs_dom.c
+ *
+ * Per domain hypfs nodes.
+ */
+
+#include <xen/err.h>
+#include <xen/hypfs.h>
+#include <xen/lib.h>
+#include <xen/sched.h>
+
+static const struct hypfs_entry *domain_domdir_enter(
+    const struct hypfs_entry *entry)
+{
+    struct hypfs_dyndir_id *data;
+    struct domain *d;
+
+    data = hypfs_get_dyndata();
+    d = get_domain_by_id(data->id);
+    data->data = d;
+    if ( !d )
+        return ERR_PTR(-ENOENT);
+
+    return entry;
+}
+
+static void domain_domdir_exit(const struct hypfs_entry *entry)
+{
+    struct hypfs_dyndir_id *data;
+    struct domain *d;
+
+    data = hypfs_get_dyndata();
+    d = data->data;
+    put_domain(d);
+}
+
+static const struct hypfs_funcs domain_domdir_funcs = {
+    .enter = domain_domdir_enter,
+    .exit = domain_domdir_exit,
+    .read = hypfs_read_dir,
+    .write = hypfs_write_deny,
+    .getsize = hypfs_getsize,
+    .findentry = hypfs_dir_findentry,
+};
+
+static HYPFS_DIR_INIT_FUNC(domain_domdir, "%u", &domain_domdir_funcs);
+
+static int domain_dir_read(const struct hypfs_entry *entry,
+                           XEN_GUEST_HANDLE_PARAM(void) uaddr)
+{
+    int ret = 0;
+    const struct domain *d;
+
+    for_each_domain ( d )
+    {
+        ret = hypfs_read_dyndir_id_entry(&domain_domdir, d->domain_id,
+                                         !d->next_in_list, &uaddr);
+        if ( ret )
+            break;
+    }
+
+    return ret;
+}
+
+static unsigned int domain_dir_getsize(const struct hypfs_entry *entry)
+{
+    const struct domain *d;
+    unsigned int size = 0;
+
+    for_each_domain ( d )
+        size += hypfs_dynid_entry_size(entry, d->domain_id);
+
+    return size;
+}
+
+static const struct hypfs_entry *domain_dir_enter(
+    const struct hypfs_entry *entry)
+{
+    struct hypfs_dyndir_id *data;
+
+    data = hypfs_alloc_dyndata(struct hypfs_dyndir_id);
+    if ( !data )
+        return ERR_PTR(-ENOMEM);
+    data->id = DOMID_SELF;
+
+    rcu_read_lock(&domlist_read_lock);
+
+    return entry;
+}
+
+static void domain_dir_exit(const struct hypfs_entry *entry)
+{
+    rcu_read_unlock(&domlist_read_lock);
+
+    hypfs_free_dyndata();
+}
+
+static struct hypfs_entry *domain_dir_findentry(
+    const struct hypfs_entry_dir *dir, const char *name, unsigned int name_len)
+{
+    unsigned long id;
+    const char *end;
+    struct domain *d;
+
+    id = simple_strtoul(name, &end, 10);
+    if ( end != name + name_len )
+        return ERR_PTR(-ENOENT);
+
+    d = rcu_lock_domain_by_id(id);
+    if ( !d )
+        return ERR_PTR(-ENOENT);
+
+    rcu_unlock_domain(d);
+
+    return hypfs_gen_dyndir_id_entry(&domain_domdir, id, d);
+}
+
+static const struct hypfs_funcs domain_dir_funcs = {
+    .enter = domain_dir_enter,
+    .exit = domain_dir_exit,
+    .read = domain_dir_read,
+    .write = hypfs_write_deny,
+    .getsize = domain_dir_getsize,
+    .findentry = domain_dir_findentry,
+};
+
+static HYPFS_DIR_INIT_FUNC(domain_dir, "domain", &domain_dir_funcs);
+
+static int __init domhypfs_init(void)
+{
+    hypfs_add_dir(&hypfs_root, &domain_dir, true);
+    hypfs_add_dyndir(&domain_dir, &domain_domdir);
+
+    return 0;
+}
+__initcall(domhypfs_init);
-- 
2.26.2

[PATCH RFC 3/3] xen/domain: add per-domain hypfs directory abi-features

[Juergen Gross]

Add a new per-domain hypfs directory "abi-features" used to control
some feature availability. Changing the availability of a feature is
allowed only before the first activation of the domain.

The related leafs right now are "event-channel-upcall" and
"fifo-event-channels". For those bool elements are added to struct
domain, but for now without any further handling.

Signed-off-by: Juergen Gross <jgross@suse.com>
---
 xen/common/hypfs_dom.c  | 39 +++++++++++++++++++++++++++++++++++++++
 xen/include/xen/sched.h |  4 ++++
 2 files changed, 43 insertions(+)

diff --git a/xen/common/hypfs_dom.c b/xen/common/hypfs_dom.c
index 241e379b24..b54e246ad6 100644
--- a/xen/common/hypfs_dom.c
+++ b/xen/common/hypfs_dom.c
@@ -10,6 +10,42 @@
 #include <xen/lib.h>
 #include <xen/sched.h>
 
+static int domain_abifeat_write(struct hypfs_entry_leaf *leaf,
+                                XEN_GUEST_HANDLE_PARAM(const_void) uaddr,
+                                unsigned int ulen)
+{
+    struct hypfs_dyndir_id *data;
+    struct domain *d;
+
+    data = hypfs_get_dyndata();
+    d = data->data;
+
+    if ( d->creation_finished )
+        return -EBUSY;
+
+    return hypfs_dyndir_id_write_bool(leaf, uaddr, ulen);
+}
+
+static const struct hypfs_funcs abifeat_funcs = {
+    .enter = hypfs_node_enter,
+    .exit = hypfs_node_exit,
+    .read = hypfs_dyndir_id_read_leaf,
+    .write = domain_abifeat_write,
+    .getsize = hypfs_getsize,
+    .findentry = hypfs_leaf_findentry,
+};
+
+static HYPFS_FIXEDSIZE_INIT(abifeat_evtupcall, XEN_HYPFS_TYPE_BOOL,
+                            "event-channel-upcall",
+                            HYPFS_STRUCT_ELEM(struct domain, abi_evt_upcall),
+                            &abifeat_funcs, 1);
+static HYPFS_FIXEDSIZE_INIT(abifeat_fifoevnt, XEN_HYPFS_TYPE_BOOL,
+                            "fifo-event-channels",
+                            HYPFS_STRUCT_ELEM(struct domain, abi_fifo_evt),
+                            &abifeat_funcs, 1);
+
+static HYPFS_DIR_INIT(domain_abifeatdir, "abi-features");
+
 static const struct hypfs_entry *domain_domdir_enter(
     const struct hypfs_entry *entry)
 {
@@ -131,6 +167,9 @@ static int __init domhypfs_init(void)
 {
     hypfs_add_dir(&hypfs_root, &domain_dir, true);
     hypfs_add_dyndir(&domain_dir, &domain_domdir);
+    hypfs_add_dir(&domain_domdir, &domain_abifeatdir, true);
+    hypfs_add_leaf(&domain_abifeatdir, &abifeat_evtupcall, true);
+    hypfs_add_leaf(&domain_abifeatdir, &abifeat_fifoevnt, true);
 
     return 0;
 }
diff --git a/xen/include/xen/sched.h b/xen/include/xen/sched.h
index 31abbe7a99..fb99249743 100644
--- a/xen/include/xen/sched.h
+++ b/xen/include/xen/sched.h
@@ -424,6 +424,10 @@ struct domain
      */
     bool             creation_finished;
 
+    /* ABI-features (can be set via hypfs before first unpause).*/
+    bool             abi_fifo_evt;
+    bool             abi_evt_upcall;
+
     /* Which guest this guest has privileges on */
     struct domain   *target;
 
-- 
2.26.2

Re: [PATCH RFC 0/3] xen: add hypfs per-domain abi-features

[Julien Grall]

Hi,

On 09/12/2020 16:16, Juergen Gross wrote:
> This small series is meant as an example how to add further dynamical
> directories to hypfs. It can be used to replace Paul's current approach
> to specify ABI-features via domain create flags and replace those by
> hypfs nodes.

This can only work if all the ABI-features are not required at the time 
of creating the domain.

Those features should also be set only once. Furthermore, HYPFS is so 
far meant to be optional.

So it feels to me Paul's approach is leaner and better for the 
ABI-features purpose.

Cheers,

-- 
Julien Grall

Re: [PATCH RFC 2/3] xen/domain: add domain hypfs directories

[Julien Grall]

Hi Juergen,

On 09/12/2020 16:16, Juergen Gross wrote:
> Add /domain/<domid> directories to hypfs. Those are completely
> dynamic, so the related hypfs access functions need to be implemented.
> 
> Signed-off-by: Juergen Gross <jgross@suse.com>
> ---
> V3:
> - new patch
> ---
>   docs/misc/hypfs-paths.pandoc |  10 +++
>   xen/common/Makefile          |   1 +
>   xen/common/hypfs_dom.c       | 137 +++++++++++++++++++++++++++++++++++
>   3 files changed, 148 insertions(+)
>   create mode 100644 xen/common/hypfs_dom.c
> 
> diff --git a/docs/misc/hypfs-paths.pandoc b/docs/misc/hypfs-paths.pandoc
> index e86f7d0dbe..116642e367 100644
> --- a/docs/misc/hypfs-paths.pandoc
> +++ b/docs/misc/hypfs-paths.pandoc
> @@ -34,6 +34,7 @@ not containing any '/' character. The names "." and ".." are reserved
>   for file system internal use.
>   
>   VALUES are strings and can take the following forms (note that this represents
> +>>>>>>> patched

This seems to be a left-over of a merge.

>   only the syntax used in this document):
>   
>   * STRING -- an arbitrary 0-delimited byte string.
> @@ -191,6 +192,15 @@ The scheduling granularity of a cpupool.
>   Writing a value is allowed only for cpupools with no cpu assigned and if the
>   architecture is supporting different scheduling granularities.
>   

[...]

> +
> +static int domain_dir_read(const struct hypfs_entry *entry,
> +                           XEN_GUEST_HANDLE_PARAM(void) uaddr)
> +{
> +    int ret = 0;
> +    const struct domain *d;
> +
> +    for_each_domain ( d )

This is definitely going to be an issue if you have a lot of domain 
running as Xen is not preemptible.

I think the first step is to make sure that HYPFS can scale without 
hogging a pCPU for a long time.

Cheers,

-- 
Julien Grall

Re: [PATCH RFC 0/3] xen: add hypfs per-domain abi-features

[Jürgen Groß]

[-- Attachment #1.1.1: Type: text/plain, Size: 1160 bytes --]

On 09.12.20 17:24, Julien Grall wrote:
> Hi,
> 
> On 09/12/2020 16:16, Juergen Gross wrote:
>> This small series is meant as an example how to add further dynamical
>> directories to hypfs. It can be used to replace Paul's current approach
>> to specify ABI-features via domain create flags and replace those by
>> hypfs nodes.
> 
> This can only work if all the ABI-features are not required at the time 
> of creating the domain.

Yes. In case there is some further initialization needed this has to be
done later depending on the setting.

> Those features should also be set only once. Furthermore, HYPFS is so 
> far meant to be optional.

"set once" isn't the point. They should not be able to change after the
domain has been started, and that is covered.

> 
> So it feels to me Paul's approach is leaner and better for the 
> ABI-features purpose.

Depends.

My approach doesn't need any tools side changes after first
implementation when adding new abi-features. And it isn't expanding an
unstable interface.

In the end this is the reason I marked this series as RFC. If using
flags is preferred, fine.


Juergen

[-- Attachment #1.1.2: OpenPGP_0xB0DE9DD628BF132F.asc --]
[-- Type: application/pgp-keys, Size: 3135 bytes --]

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 495 bytes --]

Re: [PATCH RFC 2/3] xen/domain: add domain hypfs directories

[Jürgen Groß]

[-- Attachment #1.1.1: Type: text/plain, Size: 2378 bytes --]

On 09.12.20 17:37, Julien Grall wrote:
> Hi Juergen,
> 
> On 09/12/2020 16:16, Juergen Gross wrote:
>> Add /domain/<domid> directories to hypfs. Those are completely
>> dynamic, so the related hypfs access functions need to be implemented.
>>
>> Signed-off-by: Juergen Gross <jgross@suse.com>
>> ---
>> V3:
>> - new patch
>> ---
>>   docs/misc/hypfs-paths.pandoc |  10 +++
>>   xen/common/Makefile          |   1 +
>>   xen/common/hypfs_dom.c       | 137 +++++++++++++++++++++++++++++++++++
>>   3 files changed, 148 insertions(+)
>>   create mode 100644 xen/common/hypfs_dom.c
>>
>> diff --git a/docs/misc/hypfs-paths.pandoc b/docs/misc/hypfs-paths.pandoc
>> index e86f7d0dbe..116642e367 100644
>> --- a/docs/misc/hypfs-paths.pandoc
>> +++ b/docs/misc/hypfs-paths.pandoc
>> @@ -34,6 +34,7 @@ not containing any '/' character. The names "." and 
>> ".." are reserved
>>   for file system internal use.
>>   VALUES are strings and can take the following forms (note that this 
>> represents
>> +>>>>>>> patched
> 
> This seems to be a left-over of a merge.

Oh, interesting that I wasn't warned about that.

> 
>>   only the syntax used in this document):
>>   * STRING -- an arbitrary 0-delimited byte string.
>> @@ -191,6 +192,15 @@ The scheduling granularity of a cpupool.
>>   Writing a value is allowed only for cpupools with no cpu assigned 
>> and if the
>>   architecture is supporting different scheduling granularities.
> 
> [...]
> 
>> +
>> +static int domain_dir_read(const struct hypfs_entry *entry,
>> +                           XEN_GUEST_HANDLE_PARAM(void) uaddr)
>> +{
>> +    int ret = 0;
>> +    const struct domain *d;
>> +
>> +    for_each_domain ( d )
> 
> This is definitely going to be an issue if you have a lot of domain 
> running as Xen is not preemptible.

In general this is correct, but in this case I don't think it will
be a problem. The execution time for each loop iteration should be
rather short (in the microsecond range?), so even with 32000 guests
we would stay way below one second. And on rather slow cpus I don't
think we'd have thousands of guests anyway.

> I think the first step is to make sure that HYPFS can scale without 
> hogging a pCPU for a long time.

I agree this would be best.


Juergen


[-- Attachment #1.1.2: OpenPGP_0xB0DE9DD628BF132F.asc --]
[-- Type: application/pgp-keys, Size: 3135 bytes --]

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 495 bytes --]

Re: [PATCH RFC 2/3] xen/domain: add domain hypfs directories

[Julien Grall]

On 10/12/2020 07:54, Jürgen Groß wrote:
> On 09.12.20 17:37, Julien Grall wrote:
>>>   only the syntax used in this document):
>>>   * STRING -- an arbitrary 0-delimited byte string.
>>> @@ -191,6 +192,15 @@ The scheduling granularity of a cpupool.
>>>   Writing a value is allowed only for cpupools with no cpu assigned 
>>> and if the
>>>   architecture is supporting different scheduling granularities.
>>
>> [...]
>>
>>> +
>>> +static int domain_dir_read(const struct hypfs_entry *entry,
>>> +                           XEN_GUEST_HANDLE_PARAM(void) uaddr)
>>> +{
>>> +    int ret = 0;
>>> +    const struct domain *d;
>>> +
>>> +    for_each_domain ( d )
>>
>> This is definitely going to be an issue if you have a lot of domain 
>> running as Xen is not preemptible.
> 
> In general this is correct, but in this case I don't think it will
> be a problem. The execution time for each loop iteration should be
> rather short (in the microsecond range?), so even with 32000 guests
> we would stay way below one second.

The scheduling slice are usually in ms and not second (yet this will 
depend on your scheduler). It would be unacceptable to me if another 
vCPU cannot run for a second because dom0 is trying to list the domain 
via HYPFS.

Cheers,

-- 
Julien Grall

Re: [PATCH RFC 0/3] xen: add hypfs per-domain abi-features

[Julien Grall]

On 10/12/2020 07:49, Jürgen Groß wrote:
> On 09.12.20 17:24, Julien Grall wrote:
>> Hi,
>>
>> On 09/12/2020 16:16, Juergen Gross wrote:
>>> This small series is meant as an example how to add further dynamical
>>> directories to hypfs. It can be used to replace Paul's current approach
>>> to specify ABI-features via domain create flags and replace those by
>>> hypfs nodes.
>>
>> This can only work if all the ABI-features are not required at the 
>> time of creating the domain.
> 
> Yes. In case there is some further initialization needed this has to be
> done later depending on the setting.
We used to allocate vCPUs after the domain has been created. But this 
ended up in a can of worms because this requires a careful ordering of 
the hypercalls.

So I would rather like to avoid introducing similar operation again...

> 
>> Those features should also be set only once. Furthermore, HYPFS is so 
>> far meant to be optional.
> 
> "set once" isn't the point. They should not be able to change after the
> domain has been started, and that is covered.

That really depends on the flag. Imagine there are dependencies between 
flags or memory needs to be allocated.

> 
>>
>> So it feels to me Paul's approach is leaner and better for the 
>> ABI-features purpose.
> 
> Depends.
> 
> My approach doesn't need any tools side changes after first
> implementation when adding new abi-features. And it isn't expanding an
> unstable interface.
> 
> In the end this is the reason I marked this series as RFC. If using
> flags is preferred, fine.

I prefer the flag version. Let see what other thinks.

Cheers,

---
Julien Grall

Re: [PATCH RFC 2/3] xen/domain: add domain hypfs directories

[Jürgen Groß]

[-- Attachment #1.1.1: Type: text/plain, Size: 1737 bytes --]

On 10.12.20 12:51, Julien Grall wrote:
> 
> 
> On 10/12/2020 07:54, Jürgen Groß wrote:
>> On 09.12.20 17:37, Julien Grall wrote:
>>>>   only the syntax used in this document):
>>>>   * STRING -- an arbitrary 0-delimited byte string.
>>>> @@ -191,6 +192,15 @@ The scheduling granularity of a cpupool.
>>>>   Writing a value is allowed only for cpupools with no cpu assigned 
>>>> and if the
>>>>   architecture is supporting different scheduling granularities.
>>>
>>> [...]
>>>
>>>> +
>>>> +static int domain_dir_read(const struct hypfs_entry *entry,
>>>> +                           XEN_GUEST_HANDLE_PARAM(void) uaddr)
>>>> +{
>>>> +    int ret = 0;
>>>> +    const struct domain *d;
>>>> +
>>>> +    for_each_domain ( d )
>>>
>>> This is definitely going to be an issue if you have a lot of domain 
>>> running as Xen is not preemptible.
>>
>> In general this is correct, but in this case I don't think it will
>> be a problem. The execution time for each loop iteration should be
>> rather short (in the microsecond range?), so even with 32000 guests
>> we would stay way below one second.
> 
> The scheduling slice are usually in ms and not second (yet this will 
> depend on your scheduler). It would be unacceptable to me if another 
> vCPU cannot run for a second because dom0 is trying to list the domain 
> via HYPFS.

Okay, I did a test.

The worrying operation is the reading of /domain/ with lots of domains.

"xenhypfs ls /domain" with 500 domains running needed 231 us of real
time for the library call, while "xenhypfs ls /" needed about 70 us.
This makes 3 domains per usec, resulting in about 10 msecs with 30000
domains.


Juergen

[-- Attachment #1.1.2: OpenPGP_0xB0DE9DD628BF132F.asc --]
[-- Type: application/pgp-keys, Size: 3135 bytes --]

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 495 bytes --]

Re: [PATCH RFC 1/3] xen/hypfs: add support for bool leafs in dynamic directories

[Jan Beulich]

On 09.12.2020 17:16, Juergen Gross wrote:
> --- a/xen/common/hypfs.c
> +++ b/xen/common/hypfs.c
> @@ -501,17 +501,26 @@ int hypfs_read_dir(const struct hypfs_entry *entry,
>      return 0;
>  }
>  
> -int hypfs_read_leaf(const struct hypfs_entry *entry,
> -                    XEN_GUEST_HANDLE_PARAM(void) uaddr)
> +static int hypfs_read_leaf_off(const struct hypfs_entry *entry,
> +                               XEN_GUEST_HANDLE_PARAM(void) uaddr,
> +                               void *off)
>  {
>      const struct hypfs_entry_leaf *l;
>      unsigned int size = entry->funcs->getsize(entry);
> +    const void *ptr;
>  
>      ASSERT(this_cpu(hypfs_locked) != hypfs_unlocked);
>  
>      l = container_of(entry, const struct hypfs_entry_leaf, e);
> +    ptr = off ? off + (unsigned long)l->u.content : l->u.content;

This is very irritating - you effectively add together two
pointers. And even if this was correct for some reason, it
would seem better readable as

    ptr = l->u.content + (unsigned long)off;

to me.

> --- a/xen/include/xen/hypfs.h
> +++ b/xen/include/xen/hypfs.h
> @@ -160,6 +160,8 @@ static inline void hypfs_string_set_reference(struct hypfs_entry_leaf *leaf,
>      HYPFS_FIXEDSIZE_INIT(var, XEN_HYPFS_TYPE_BOOL, nam, contvar, \
>                           &hypfs_bool_wr_funcs, 1)
>  
> +#define HYPFS_STRUCT_ELEM(type, elem)    (((type *)NULL)->elem)

Kind of similar here - this very much looks like a NULL
deref, avoidable by a user only if it takes the address of
the construct. If there's really some non-pointer value
to be encoded here, it would be better to avoid misuse by
making the construct safe in a self-contained way.

Jan

Re: [PATCH RFC 0/3] xen: add hypfs per-domain abi-features

[Jan Beulich]

On 09.12.2020 17:16, Juergen Gross wrote:
> This small series is meant as an example how to add further dynamical
> directories to hypfs. It can be used to replace Paul's current approach
> to specify ABI-features via domain create flags and replace those by
> hypfs nodes.

Actually, before I look at further of the patches here: I seem
to vaguely recall agreement was reached to stick to Paul's
approach for the time being? Or am I misremembering?

Jan

Re: [PATCH RFC 0/3] xen: add hypfs per-domain abi-features

[Jürgen Groß]

[-- Attachment #1.1.1: Type: text/plain, Size: 559 bytes --]

On 08.01.21 16:44, Jan Beulich wrote:
> On 09.12.2020 17:16, Juergen Gross wrote:
>> This small series is meant as an example how to add further dynamical
>> directories to hypfs. It can be used to replace Paul's current approach
>> to specify ABI-features via domain create flags and replace those by
>> hypfs nodes.
> 
> Actually, before I look at further of the patches here: I seem
> to vaguely recall agreement was reached to stick to Paul's
> approach for the time being? Or am I misremembering?

This is my understanding, too.


Juergen

[-- Attachment #1.1.2: OpenPGP_0xB0DE9DD628BF132F.asc --]
[-- Type: application/pgp-keys, Size: 3135 bytes --]

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 495 bytes --]