* [PATCH 1/5] ebpf: Added eBPF initialization by fds and map update.
2021-07-13 15:37 [PATCH 0/5] ebpf: Added ebpf helper for libvirtd Andrew Melnychenko
@ 2021-07-13 15:37 ` Andrew Melnychenko
2021-08-20 3:34 ` Jason Wang
2021-07-13 15:37 ` [PATCH 2/5] virtio-net: Added property to load eBPF RSS with fds Andrew Melnychenko
` (5 subsequent siblings)
6 siblings, 1 reply; 60+ messages in thread
From: Andrew Melnychenko @ 2021-07-13 15:37 UTC (permalink / raw)
To: mst, yuri.benditovich, jasowang, armbru, eblake, berrange; +Cc: yan, qemu-devel
RSS maps are combined into one array map.
Changed eBPF map updates through mmaped array.
eBPF RSS context may be initialized by program fd and map fd.
virtio-net may provide fds passed by libvirt.
Signed-off-by: Andrew Melnychenko <andrew@daynix.com>
---
ebpf/ebpf_rss-stub.c | 6 +
ebpf/ebpf_rss.c | 120 +++++----
ebpf/ebpf_rss.h | 8 +-
ebpf/rss.bpf.skeleton.h | 557 +++++++++++++++++++---------------------
tools/ebpf/rss.bpf.c | 67 ++---
5 files changed, 365 insertions(+), 393 deletions(-)
diff --git a/ebpf/ebpf_rss-stub.c b/ebpf/ebpf_rss-stub.c
index e71e229190..8d7fae2ad9 100644
--- a/ebpf/ebpf_rss-stub.c
+++ b/ebpf/ebpf_rss-stub.c
@@ -28,6 +28,12 @@ bool ebpf_rss_load(struct EBPFRSSContext *ctx)
return false;
}
+bool ebpf_rss_load_fds(struct EBPFRSSContext *ctx, int program_fd,
+ int config_fd, int toeplitz_fd, int table_fd)
+{
+ return false;
+}
+
bool ebpf_rss_set_all(struct EBPFRSSContext *ctx, struct EBPFRSSConfig *config,
uint16_t *indirections_table, uint8_t *toeplitz_key)
{
diff --git a/ebpf/ebpf_rss.c b/ebpf/ebpf_rss.c
index 118c68da83..bfed0b446e 100644
--- a/ebpf/ebpf_rss.c
+++ b/ebpf/ebpf_rss.c
@@ -27,19 +27,21 @@ void ebpf_rss_init(struct EBPFRSSContext *ctx)
{
if (ctx != NULL) {
ctx->obj = NULL;
+ ctx->program_fd = -1;
+ ctx->mmap_configuration = NULL;
}
}
bool ebpf_rss_is_loaded(struct EBPFRSSContext *ctx)
{
- return ctx != NULL && ctx->obj != NULL;
+ return ctx != NULL && (ctx->obj != NULL || ctx->program_fd != -1);
}
bool ebpf_rss_load(struct EBPFRSSContext *ctx)
{
struct rss_bpf *rss_bpf_ctx;
- if (ctx == NULL) {
+ if (ctx == NULL || ebpf_rss_is_loaded(ctx)) {
return false;
}
@@ -61,72 +63,43 @@ bool ebpf_rss_load(struct EBPFRSSContext *ctx)
rss_bpf_ctx->progs.tun_rss_steering_prog);
ctx->map_configuration = bpf_map__fd(
rss_bpf_ctx->maps.tap_rss_map_configurations);
- ctx->map_indirections_table = bpf_map__fd(
- rss_bpf_ctx->maps.tap_rss_map_indirection_table);
- ctx->map_toeplitz_key = bpf_map__fd(
- rss_bpf_ctx->maps.tap_rss_map_toeplitz_key);
+
+ ctx->mmap_configuration = mmap(NULL, qemu_real_host_page_size,
+ PROT_READ | PROT_WRITE, MAP_SHARED,
+ ctx->map_configuration, 0);
+ if (ctx->mmap_configuration == MAP_FAILED) {
+ trace_ebpf_error("eBPF RSS", "can not mmap eBPF configuration array");
+ goto error;
+ }
return true;
error:
rss_bpf__destroy(rss_bpf_ctx);
ctx->obj = NULL;
+ ctx->program_fd = -1;
+ ctx->mmap_configuration = NULL;
return false;
}
-static bool ebpf_rss_set_config(struct EBPFRSSContext *ctx,
- struct EBPFRSSConfig *config)
+bool ebpf_rss_load_fds(struct EBPFRSSContext *ctx, int program_fd,
+ int config_fd, int toeplitz_fd, int table_fd)
{
- uint32_t map_key = 0;
-
- if (!ebpf_rss_is_loaded(ctx)) {
+ if (ctx == NULL || ebpf_rss_is_loaded(ctx)) {
return false;
}
- if (bpf_map_update_elem(ctx->map_configuration,
- &map_key, config, 0) < 0) {
- return false;
- }
- return true;
-}
-static bool ebpf_rss_set_indirections_table(struct EBPFRSSContext *ctx,
- uint16_t *indirections_table,
- size_t len)
-{
- uint32_t i = 0;
+ ctx->program_fd = program_fd;
+ ctx->map_configuration = config_fd;
- if (!ebpf_rss_is_loaded(ctx) || indirections_table == NULL ||
- len > VIRTIO_NET_RSS_MAX_TABLE_LEN) {
+ ctx->mmap_configuration = mmap(NULL, qemu_real_host_page_size,
+ PROT_READ | PROT_WRITE, MAP_SHARED,
+ ctx->map_configuration, 0);
+ if (ctx->mmap_configuration == MAP_FAILED) {
+ trace_ebpf_error("eBPF RSS", "can not mmap eBPF configuration array");
return false;
}
- for (; i < len; ++i) {
- if (bpf_map_update_elem(ctx->map_indirections_table, &i,
- indirections_table + i, 0) < 0) {
- return false;
- }
- }
- return true;
-}
-
-static bool ebpf_rss_set_toepliz_key(struct EBPFRSSContext *ctx,
- uint8_t *toeplitz_key)
-{
- uint32_t map_key = 0;
-
- /* prepare toeplitz key */
- uint8_t toe[VIRTIO_NET_RSS_MAX_KEY_SIZE] = {};
-
- if (!ebpf_rss_is_loaded(ctx) || toeplitz_key == NULL) {
- return false;
- }
- memcpy(toe, toeplitz_key, VIRTIO_NET_RSS_MAX_KEY_SIZE);
- *(uint32_t *)toe = ntohl(*(uint32_t *)toe);
-
- if (bpf_map_update_elem(ctx->map_toeplitz_key, &map_key, toe,
- 0) < 0) {
- return false;
- }
return true;
}
@@ -134,21 +107,32 @@ bool ebpf_rss_set_all(struct EBPFRSSContext *ctx, struct EBPFRSSConfig *config,
uint16_t *indirections_table, uint8_t *toeplitz_key)
{
if (!ebpf_rss_is_loaded(ctx) || config == NULL ||
- indirections_table == NULL || toeplitz_key == NULL) {
+ indirections_table == NULL || toeplitz_key == NULL ||
+ config->indirections_len > VIRTIO_NET_RSS_MAX_TABLE_LEN) {
return false;
}
- if (!ebpf_rss_set_config(ctx, config)) {
- return false;
- }
+ struct {
+ struct EBPFRSSConfig config;
+ uint8_t toeplitz_key[VIRTIO_NET_RSS_MAX_KEY_SIZE];
+ uint16_t indirections_table[VIRTIO_NET_RSS_MAX_TABLE_LEN];
+ } __attribute__((packed)) ebpf_config;
- if (!ebpf_rss_set_indirections_table(ctx, indirections_table,
- config->indirections_len)) {
- return false;
- }
+ /* Setting up configurations */
+ memcpy(&ebpf_config.config, config, sizeof(*config));
- if (!ebpf_rss_set_toepliz_key(ctx, toeplitz_key)) {
- return false;
+ /* Setting up toeplitz key data */
+ memcpy(&ebpf_config.toeplitz_key, toeplitz_key,
+ VIRTIO_NET_RSS_MAX_KEY_SIZE);
+ *(uint32_t *)ebpf_config.toeplitz_key =
+ ntohl(*(uint32_t *)ebpf_config.toeplitz_key);
+
+ /* Setting up indirections table */
+ memcpy(&ebpf_config.indirections_table, indirections_table,
+ config->indirections_len * sizeof(*indirections_table));
+
+ if (ctx->mmap_configuration != NULL) {
+ memcpy(ctx->mmap_configuration, &ebpf_config, sizeof(ebpf_config));
}
return true;
@@ -160,6 +144,18 @@ void ebpf_rss_unload(struct EBPFRSSContext *ctx)
return;
}
- rss_bpf__destroy(ctx->obj);
+ if (ctx->mmap_configuration) {
+ munmap(ctx->mmap_configuration, qemu_real_host_page_size);
+ }
+
+ if (ctx->obj != NULL) {
+ rss_bpf__destroy(ctx->obj);
+ } else {
+ close(ctx->program_fd);
+ close(ctx->map_configuration);
+ }
+
ctx->obj = NULL;
+ ctx->program_fd = -1;
+ ctx->mmap_configuration = NULL;
}
diff --git a/ebpf/ebpf_rss.h b/ebpf/ebpf_rss.h
index bf3f2572c7..40b1c35d5f 100644
--- a/ebpf/ebpf_rss.h
+++ b/ebpf/ebpf_rss.h
@@ -14,12 +14,13 @@
#ifndef QEMU_EBPF_RSS_H
#define QEMU_EBPF_RSS_H
+#define EBPF_RSS_MAX_FDS 2
+
struct EBPFRSSContext {
void *obj;
int program_fd;
int map_configuration;
- int map_toeplitz_key;
- int map_indirections_table;
+ void *mmap_configuration;
};
struct EBPFRSSConfig {
@@ -36,6 +37,9 @@ bool ebpf_rss_is_loaded(struct EBPFRSSContext *ctx);
bool ebpf_rss_load(struct EBPFRSSContext *ctx);
+bool ebpf_rss_load_fds(struct EBPFRSSContext *ctx, int program_fd,
+ int config_fd, int toeplitz_fd, int table_fd);
+
bool ebpf_rss_set_all(struct EBPFRSSContext *ctx, struct EBPFRSSConfig *config,
uint16_t *indirections_table, uint8_t *toeplitz_key);
diff --git a/ebpf/rss.bpf.skeleton.h b/ebpf/rss.bpf.skeleton.h
index 126683eb87..52fb49daa8 100644
--- a/ebpf/rss.bpf.skeleton.h
+++ b/ebpf/rss.bpf.skeleton.h
@@ -12,8 +12,6 @@ struct rss_bpf {
struct bpf_object *obj;
struct {
struct bpf_map *tap_rss_map_configurations;
- struct bpf_map *tap_rss_map_indirection_table;
- struct bpf_map *tap_rss_map_toeplitz_key;
} maps;
struct {
struct bpf_program *tun_rss_steering_prog;
@@ -109,7 +107,7 @@ rss_bpf__create_skeleton(struct rss_bpf *obj)
s->obj = &obj->obj;
/* maps */
- s->map_cnt = 3;
+ s->map_cnt = 1;
s->map_skel_sz = sizeof(*s->maps);
s->maps = (struct bpf_map_skeleton *)calloc(s->map_cnt, s->map_skel_sz);
if (!s->maps)
@@ -118,12 +116,6 @@ rss_bpf__create_skeleton(struct rss_bpf *obj)
s->maps[0].name = "tap_rss_map_configurations";
s->maps[0].map = &obj->maps.tap_rss_map_configurations;
- s->maps[1].name = "tap_rss_map_indirection_table";
- s->maps[1].map = &obj->maps.tap_rss_map_indirection_table;
-
- s->maps[2].name = "tap_rss_map_toeplitz_key";
- s->maps[2].map = &obj->maps.tap_rss_map_toeplitz_key;
-
/* programs */
s->prog_cnt = 1;
s->prog_skel_sz = sizeof(*s->progs);
@@ -135,292 +127,281 @@ rss_bpf__create_skeleton(struct rss_bpf *obj)
s->progs[0].prog = &obj->progs.tun_rss_steering_prog;
s->progs[0].link = &obj->links.tun_rss_steering_prog;
- s->data_sz = 8088;
+ s->data_sz = 7768;
s->data = (void *)"\
\x7f\x45\x4c\x46\x02\x01\x01\0\0\0\0\0\0\0\0\0\x01\0\xf7\0\x01\0\0\0\0\0\0\0\0\
-\0\0\0\0\0\0\0\0\0\0\0\x18\x1d\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\0\x40\0\x0a\0\
-\x01\0\xbf\x18\0\0\0\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\x4c\xff\0\0\0\0\xbf\xa7\
-\0\0\0\0\0\0\x07\x07\0\0\x4c\xff\xff\xff\x18\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
-\xbf\x72\0\0\0\0\0\0\x85\0\0\0\x01\0\0\0\xbf\x06\0\0\0\0\0\0\x18\x01\0\0\0\0\0\
-\0\0\0\0\0\0\0\0\0\xbf\x72\0\0\0\0\0\0\x85\0\0\0\x01\0\0\0\xbf\x07\0\0\0\0\0\0\
-\x18\0\0\0\xff\xff\xff\xff\0\0\0\0\0\0\0\0\x15\x06\x66\x02\0\0\0\0\xbf\x79\0\0\
-\0\0\0\0\x15\x09\x64\x02\0\0\0\0\x71\x61\0\0\0\0\0\0\x55\x01\x01\0\0\0\0\0\x05\
-\0\x5d\x02\0\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\xc0\xff\0\0\0\0\x7b\x1a\xb8\xff\
-\0\0\0\0\x7b\x1a\xb0\xff\0\0\0\0\x7b\x1a\xa8\xff\0\0\0\0\x7b\x1a\xa0\xff\0\0\0\
-\0\x63\x1a\x98\xff\0\0\0\0\x7b\x1a\x90\xff\0\0\0\0\x7b\x1a\x88\xff\0\0\0\0\x7b\
-\x1a\x80\xff\0\0\0\0\x7b\x1a\x78\xff\0\0\0\0\x7b\x1a\x70\xff\0\0\0\0\x7b\x1a\
-\x68\xff\0\0\0\0\x7b\x1a\x60\xff\0\0\0\0\x7b\x1a\x58\xff\0\0\0\0\x7b\x1a\x50\
-\xff\0\0\0\0\x15\x08\x4c\x02\0\0\0\0\x6b\x1a\xd0\xff\0\0\0\0\xbf\xa3\0\0\0\0\0\
-\0\x07\x03\0\0\xd0\xff\xff\xff\xbf\x81\0\0\0\0\0\0\xb7\x02\0\0\x0c\0\0\0\xb7\
-\x04\0\0\x02\0\0\0\xb7\x05\0\0\0\0\0\0\x85\0\0\0\x44\0\0\0\x67\0\0\0\x20\0\0\0\
-\x77\0\0\0\x20\0\0\0\x55\0\x11\0\0\0\0\0\xb7\x02\0\0\x10\0\0\0\x69\xa1\xd0\xff\
-\0\0\0\0\xbf\x13\0\0\0\0\0\0\xdc\x03\0\0\x10\0\0\0\x15\x03\x02\0\0\x81\0\0\x55\
-\x03\x0c\0\xa8\x88\0\0\xb7\x02\0\0\x14\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\
-\xd0\xff\xff\xff\xbf\x81\0\0\0\0\0\0\xb7\x04\0\0\x02\0\0\0\xb7\x05\0\0\0\0\0\0\
-\x85\0\0\0\x44\0\0\0\x69\xa1\xd0\xff\0\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\
-\0\0\0\x15\0\x01\0\0\0\0\0\x05\0\x2f\x02\0\0\0\0\x15\x01\x2e\x02\0\0\0\0\x7b\
-\x9a\x30\xff\0\0\0\0\x15\x01\x57\0\x86\xdd\0\0\x55\x01\x3b\0\x08\0\0\0\x7b\x7a\
-\x20\xff\0\0\0\0\xb7\x07\0\0\x01\0\0\0\x73\x7a\x50\xff\0\0\0\0\xb7\x01\0\0\0\0\
-\0\0\x63\x1a\xe0\xff\0\0\0\0\x7b\x1a\xd8\xff\0\0\0\0\x7b\x1a\xd0\xff\0\0\0\0\
-\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\xd0\xff\xff\xff\xbf\x81\0\0\0\0\0\0\xb7\x02\0\
-\0\0\0\0\0\xb7\x04\0\0\x14\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\x67\
-\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\x55\0\x1a\x02\0\0\0\0\x69\xa1\xd6\xff\0\0\
-\0\0\x55\x01\x01\0\0\0\0\0\xb7\x07\0\0\0\0\0\0\x61\xa1\xdc\xff\0\0\0\0\x63\x1a\
-\x5c\xff\0\0\0\0\x61\xa1\xe0\xff\0\0\0\0\x63\x1a\x60\xff\0\0\0\0\x73\x7a\x56\
-\xff\0\0\0\0\x71\xa9\xd9\xff\0\0\0\0\x71\xa1\xd0\xff\0\0\0\0\x67\x01\0\0\x02\0\
-\0\0\x57\x01\0\0\x3c\0\0\0\x7b\x1a\x40\xff\0\0\0\0\x79\xa7\x20\xff\0\0\0\0\xbf\
+\0\0\0\0\0\0\0\0\0\0\0\xd8\x1b\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\0\x40\0\x0a\0\
+\x01\0\xbf\x18\0\0\0\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\x4c\xff\0\0\0\0\xbf\xa2\
+\0\0\0\0\0\0\x07\x02\0\0\x4c\xff\xff\xff\x18\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\x85\0\0\0\x01\0\0\0\xbf\x06\0\0\0\0\0\0\x18\0\0\0\xff\xff\xff\xff\0\0\0\0\0\0\
+\0\0\x15\x06\x35\0\0\0\0\0\x71\x61\0\0\0\0\0\0\x55\x01\x01\0\0\0\0\0\x05\0\x2d\
+\0\0\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\xc0\xff\0\0\0\0\x7b\x1a\xb8\xff\0\0\0\0\
+\x7b\x1a\xb0\xff\0\0\0\0\x7b\x1a\xa8\xff\0\0\0\0\x7b\x1a\xa0\xff\0\0\0\0\x63\
+\x1a\x98\xff\0\0\0\0\x7b\x1a\x90\xff\0\0\0\0\x7b\x1a\x88\xff\0\0\0\0\x7b\x1a\
+\x80\xff\0\0\0\0\x7b\x1a\x78\xff\0\0\0\0\x7b\x1a\x70\xff\0\0\0\0\x7b\x1a\x68\
+\xff\0\0\0\0\x7b\x1a\x60\xff\0\0\0\0\x7b\x1a\x58\xff\0\0\0\0\x7b\x1a\x50\xff\0\
+\0\0\0\x15\x08\xee\xff\0\0\0\0\x6b\x1a\xd0\xff\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\
+\x03\0\0\xd0\xff\xff\xff\xbf\x81\0\0\0\0\0\0\xb7\x02\0\0\x0c\0\0\0\xb7\x04\0\0\
+\x02\0\0\0\xb7\x05\0\0\0\0\0\0\x85\0\0\0\x44\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\
+\0\x20\0\0\0\x55\0\x11\0\0\0\0\0\xb7\x02\0\0\x10\0\0\0\x69\xa1\xd0\xff\0\0\0\0\
+\xbf\x13\0\0\0\0\0\0\xdc\x03\0\0\x10\0\0\0\x15\x03\x02\0\0\x81\0\0\x55\x03\x11\
+\0\xa8\x88\0\0\xb7\x02\0\0\x14\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\xd0\xff\
+\xff\xff\xbf\x81\0\0\0\0\0\0\xb7\x04\0\0\x02\0\0\0\xb7\x05\0\0\0\0\0\0\x85\0\0\
+\0\x44\0\0\0\x69\xa1\xd0\xff\0\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\
+\x15\0\x06\0\0\0\0\0\x07\x06\0\0\x08\0\0\0\x71\x61\0\0\0\0\0\0\x71\x60\x01\0\0\
+\0\0\0\x67\0\0\0\x08\0\0\0\x4f\x10\0\0\0\0\0\0\x95\0\0\0\0\0\0\0\x15\x01\xcb\
+\xff\0\0\0\0\x15\x01\x53\0\x86\xdd\0\0\x55\x01\x38\0\x08\0\0\0\xb7\x07\0\0\x01\
+\0\0\0\x73\x7a\x50\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\xe0\xff\0\0\0\0\x7b\
+\x1a\xd8\xff\0\0\0\0\x7b\x1a\xd0\xff\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\
+\xd0\xff\xff\xff\xbf\x81\0\0\0\0\0\0\xb7\x02\0\0\0\0\0\0\xb7\x04\0\0\x14\0\0\0\
+\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\
+\0\0\x55\0\xb9\xff\0\0\0\0\x69\xa1\xd6\xff\0\0\0\0\x55\x01\x01\0\0\0\0\0\xb7\
+\x07\0\0\0\0\0\0\x61\xa1\xdc\xff\0\0\0\0\x63\x1a\x5c\xff\0\0\0\0\x61\xa1\xe0\
+\xff\0\0\0\0\x63\x1a\x60\xff\0\0\0\0\x73\x7a\x56\xff\0\0\0\0\x71\xa9\xd9\xff\0\
+\0\0\0\x71\xa7\xd0\xff\0\0\0\0\x67\x07\0\0\x02\0\0\0\x57\x07\0\0\x3c\0\0\0\xbf\
\x91\0\0\0\0\0\0\x57\x01\0\0\xff\0\0\0\x15\x01\x19\0\0\0\0\0\x71\xa1\x56\xff\0\
-\0\0\0\x55\x01\x17\0\0\0\0\0\x57\x09\0\0\xff\0\0\0\x15\x09\x7a\x01\x11\0\0\0\
+\0\0\0\x55\x01\x17\0\0\0\0\0\x57\x09\0\0\xff\0\0\0\x15\x09\x76\x01\x11\0\0\0\
\x55\x09\x14\0\x06\0\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\x53\xff\0\0\0\0\xb7\x01\
\0\0\0\0\0\0\x63\x1a\xe0\xff\0\0\0\0\x7b\x1a\xd8\xff\0\0\0\0\x7b\x1a\xd0\xff\0\
-\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\xd0\xff\xff\xff\xbf\x81\0\0\0\0\0\0\x79\
-\xa2\x40\xff\0\0\0\0\xb7\x04\0\0\x14\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\
-\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\x55\0\xf4\x01\0\0\0\0\x69\xa1\
-\xd0\xff\0\0\0\0\x6b\x1a\x58\xff\0\0\0\0\x69\xa1\xd2\xff\0\0\0\0\x6b\x1a\x5a\
-\xff\0\0\0\0\x71\xa1\x50\xff\0\0\0\0\x15\x01\xd4\0\0\0\0\0\x71\x62\x03\0\0\0\0\
-\0\x67\x02\0\0\x08\0\0\0\x71\x61\x02\0\0\0\0\0\x4f\x12\0\0\0\0\0\0\x71\x63\x04\
-\0\0\0\0\0\x71\x61\x05\0\0\0\0\0\x67\x01\0\0\x08\0\0\0\x4f\x31\0\0\0\0\0\0\x67\
-\x01\0\0\x10\0\0\0\x4f\x21\0\0\0\0\0\0\x71\xa2\x53\xff\0\0\0\0\x79\xa0\x30\xff\
-\0\0\0\0\x15\x02\x06\x01\0\0\0\0\xbf\x12\0\0\0\0\0\0\x57\x02\0\0\x02\0\0\0\x15\
-\x02\x03\x01\0\0\0\0\x61\xa1\x5c\xff\0\0\0\0\x63\x1a\xa0\xff\0\0\0\0\x61\xa1\
-\x60\xff\0\0\0\0\x63\x1a\xa4\xff\0\0\0\0\x69\xa1\x58\xff\0\0\0\0\x6b\x1a\xa8\
-\xff\0\0\0\0\x69\xa1\x5a\xff\0\0\0\0\x6b\x1a\xaa\xff\0\0\0\0\x05\0\x65\x01\0\0\
-\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\x51\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x7b\x1a\
-\xf0\xff\0\0\0\0\x7b\x1a\xe8\xff\0\0\0\0\x7b\x1a\xe0\xff\0\0\0\0\x7b\x1a\xd8\
-\xff\0\0\0\0\x7b\x1a\xd0\xff\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\xd0\xff\
-\xff\xff\xb7\x01\0\0\x28\0\0\0\x7b\x1a\x40\xff\0\0\0\0\xbf\x81\0\0\0\0\0\0\xb7\
-\x02\0\0\0\0\0\0\xb7\x04\0\0\x28\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\
-\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\x55\0\x10\x01\0\0\0\0\x79\xa1\xe0\
-\xff\0\0\0\0\x63\x1a\x64\xff\0\0\0\0\x77\x01\0\0\x20\0\0\0\x63\x1a\x68\xff\0\0\
-\0\0\x79\xa1\xd8\xff\0\0\0\0\x63\x1a\x5c\xff\0\0\0\0\x77\x01\0\0\x20\0\0\0\x63\
-\x1a\x60\xff\0\0\0\0\x79\xa1\xe8\xff\0\0\0\0\x63\x1a\x6c\xff\0\0\0\0\x77\x01\0\
-\0\x20\0\0\0\x63\x1a\x70\xff\0\0\0\0\x79\xa1\xf0\xff\0\0\0\0\x63\x1a\x74\xff\0\
-\0\0\0\x77\x01\0\0\x20\0\0\0\x63\x1a\x78\xff\0\0\0\0\x71\xa9\xd6\xff\0\0\0\0\
-\x25\x09\xff\0\x3c\0\0\0\xb7\x01\0\0\x01\0\0\0\x6f\x91\0\0\0\0\0\0\x18\x02\0\0\
-\x01\0\0\0\0\0\0\0\0\x18\0\x1c\x5f\x21\0\0\0\0\0\0\x55\x01\x01\0\0\0\0\0\x05\0\
-\xf8\0\0\0\0\0\xb7\x01\0\0\0\0\0\0\x6b\x1a\xfe\xff\0\0\0\0\xb7\x01\0\0\x28\0\0\
-\0\x7b\x1a\x40\xff\0\0\0\0\xbf\xa1\0\0\0\0\0\0\x07\x01\0\0\x8c\xff\xff\xff\x7b\
-\x1a\x18\xff\0\0\0\0\xbf\xa1\0\0\0\0\0\0\x07\x01\0\0\x7c\xff\xff\xff\x7b\x1a\
-\x10\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x7b\x1a\x28\xff\0\0\0\0\x7b\x7a\x20\xff\0\
-\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\xfe\xff\xff\xff\xbf\x81\0\0\0\0\0\0\x79\
-\xa2\x40\xff\0\0\0\0\xb7\x04\0\0\x02\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\
-\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\x15\0\x01\0\0\0\0\0\x05\0\x90\
-\x01\0\0\0\0\xbf\x91\0\0\0\0\0\0\x15\x01\x23\0\x3c\0\0\0\x15\x01\x59\0\x2c\0\0\
-\0\x55\x01\x5a\0\x2b\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\xf8\xff\0\0\0\0\xbf\xa3\
-\0\0\0\0\0\0\x07\x03\0\0\xf8\xff\xff\xff\xbf\x81\0\0\0\0\0\0\x79\xa2\x40\xff\0\
-\0\0\0\xb7\x04\0\0\x04\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\xbf\x01\
-\0\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\0\0\0\x55\x01\x03\x01\0\0\0\
-\0\x71\xa1\xfa\xff\0\0\0\0\x55\x01\x4b\0\x02\0\0\0\x71\xa1\xf9\xff\0\0\0\0\x55\
-\x01\x49\0\x02\0\0\0\x71\xa1\xfb\xff\0\0\0\0\x55\x01\x47\0\x01\0\0\0\x79\xa2\
-\x40\xff\0\0\0\0\x07\x02\0\0\x08\0\0\0\xbf\x81\0\0\0\0\0\0\x79\xa3\x18\xff\0\0\
-\0\0\xb7\x04\0\0\x10\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\xbf\x01\0\
-\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\0\0\0\x55\x01\xf2\0\0\0\0\0\
-\xb7\x01\0\0\x01\0\0\0\x73\x1a\x55\xff\0\0\0\0\x05\0\x39\0\0\0\0\0\xb7\x01\0\0\
-\0\0\0\0\x6b\x1a\xf8\xff\0\0\0\0\xb7\x09\0\0\x02\0\0\0\xb7\x07\0\0\x1e\0\0\0\
-\x05\0\x0e\0\0\0\0\0\x79\xa2\x38\xff\0\0\0\0\x0f\x29\0\0\0\0\0\0\xbf\x92\0\0\0\
-\0\0\0\x07\x02\0\0\x01\0\0\0\x71\xa3\xff\xff\0\0\0\0\x67\x03\0\0\x03\0\0\0\x2d\
-\x23\x02\0\0\0\0\0\x79\xa7\x20\xff\0\0\0\0\x05\0\x2b\0\0\0\0\0\x07\x07\0\0\xff\
-\xff\xff\xff\xbf\x72\0\0\0\0\0\0\x67\x02\0\0\x20\0\0\0\x77\x02\0\0\x20\0\0\0\
-\x15\x02\xf9\xff\0\0\0\0\x7b\x9a\x38\xff\0\0\0\0\x79\xa1\x40\xff\0\0\0\0\x0f\
-\x19\0\0\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\xf8\xff\xff\xff\xbf\x81\0\0\0\
-\0\0\0\xbf\x92\0\0\0\0\0\0\xb7\x04\0\0\x02\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\
-\0\x44\0\0\0\xbf\x01\0\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\0\0\0\
-\x55\x01\x94\0\0\0\0\0\x71\xa2\xf8\xff\0\0\0\0\x55\x02\x0f\0\xc9\0\0\0\x07\x09\
-\0\0\x02\0\0\0\xbf\x81\0\0\0\0\0\0\xbf\x92\0\0\0\0\0\0\x79\xa3\x10\xff\0\0\0\0\
-\xb7\x04\0\0\x10\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\xbf\x01\0\0\0\
-\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\0\0\0\x55\x01\x87\0\0\0\0\0\xb7\
-\x01\0\0\x01\0\0\0\x73\x1a\x54\xff\0\0\0\0\x79\xa7\x20\xff\0\0\0\0\x05\0\x07\0\
-\0\0\0\0\xb7\x09\0\0\x01\0\0\0\x15\x02\xd1\xff\0\0\0\0\x71\xa9\xf9\xff\0\0\0\0\
-\x07\x09\0\0\x02\0\0\0\x05\0\xce\xff\0\0\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\x56\
-\xff\0\0\0\0\x71\xa1\xff\xff\0\0\0\0\x67\x01\0\0\x03\0\0\0\x79\xa2\x40\xff\0\0\
-\0\0\x0f\x12\0\0\0\0\0\0\x07\x02\0\0\x08\0\0\0\x7b\x2a\x40\xff\0\0\0\0\x71\xa9\
-\xfe\xff\0\0\0\0\x25\x09\x0e\0\x3c\0\0\0\xb7\x01\0\0\x01\0\0\0\x6f\x91\0\0\0\0\
+\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\xd0\xff\xff\xff\xbf\x81\0\0\0\0\0\0\xbf\
+\x72\0\0\0\0\0\0\xb7\x04\0\0\x14\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\
+\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\x55\0\x95\xff\0\0\0\0\x69\xa1\xd0\
+\xff\0\0\0\0\x6b\x1a\x58\xff\0\0\0\0\x69\xa1\xd2\xff\0\0\0\0\x6b\x1a\x5a\xff\0\
+\0\0\0\x71\xa1\x50\xff\0\0\0\0\x15\x01\xd0\0\0\0\0\0\x71\x62\x03\0\0\0\0\0\x67\
+\x02\0\0\x08\0\0\0\x71\x61\x02\0\0\0\0\0\x4f\x12\0\0\0\0\0\0\x71\x63\x04\0\0\0\
+\0\0\x71\x61\x05\0\0\0\0\0\x67\x01\0\0\x08\0\0\0\x4f\x31\0\0\0\0\0\0\x67\x01\0\
+\0\x10\0\0\0\x4f\x21\0\0\0\0\0\0\x71\xa2\x53\xff\0\0\0\0\x15\x02\x02\x01\0\0\0\
+\0\xbf\x12\0\0\0\0\0\0\x57\x02\0\0\x02\0\0\0\x15\x02\xff\0\0\0\0\0\x61\xa1\x5c\
+\xff\0\0\0\0\x63\x1a\xa0\xff\0\0\0\0\x61\xa1\x60\xff\0\0\0\0\x63\x1a\xa4\xff\0\
+\0\0\0\x69\xa1\x58\xff\0\0\0\0\x6b\x1a\xa8\xff\0\0\0\0\x69\xa1\x5a\xff\0\0\0\0\
+\x6b\x1a\xaa\xff\0\0\0\0\x05\0\x62\x01\0\0\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\
+\x51\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x7b\x1a\xf0\xff\0\0\0\0\x7b\x1a\xe8\xff\0\
+\0\0\0\x7b\x1a\xe0\xff\0\0\0\0\x7b\x1a\xd8\xff\0\0\0\0\x7b\x1a\xd0\xff\0\0\0\0\
+\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\xd0\xff\xff\xff\xb7\x07\0\0\x28\0\0\0\xbf\x81\
+\0\0\0\0\0\0\xb7\x02\0\0\0\0\0\0\xb7\x04\0\0\x28\0\0\0\xb7\x05\0\0\x01\0\0\0\
+\x85\0\0\0\x44\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\x55\0\x0e\x01\0\0\
+\0\0\x79\xa1\xe0\xff\0\0\0\0\x63\x1a\x64\xff\0\0\0\0\x77\x01\0\0\x20\0\0\0\x63\
+\x1a\x68\xff\0\0\0\0\x79\xa1\xd8\xff\0\0\0\0\x63\x1a\x5c\xff\0\0\0\0\x77\x01\0\
+\0\x20\0\0\0\x63\x1a\x60\xff\0\0\0\0\x79\xa1\xe8\xff\0\0\0\0\x63\x1a\x6c\xff\0\
+\0\0\0\x77\x01\0\0\x20\0\0\0\x63\x1a\x70\xff\0\0\0\0\x79\xa1\xf0\xff\0\0\0\0\
+\x63\x1a\x74\xff\0\0\0\0\x77\x01\0\0\x20\0\0\0\x63\x1a\x78\xff\0\0\0\0\x71\xa9\
+\xd6\xff\0\0\0\0\x25\x09\xfd\0\x3c\0\0\0\xb7\x01\0\0\x01\0\0\0\x6f\x91\0\0\0\0\
\0\0\x18\x02\0\0\x01\0\0\0\0\0\0\0\0\x18\0\x1c\x5f\x21\0\0\0\0\0\0\x55\x01\x01\
-\0\0\0\0\0\x05\0\x07\0\0\0\0\0\x79\xa1\x28\xff\0\0\0\0\x07\x01\0\0\x01\0\0\0\
-\x7b\x1a\x28\xff\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\0\0\0\x55\x01\
-\x82\xff\x0b\0\0\0\x05\0\x10\xff\0\0\0\0\x15\x09\xf8\xff\x87\0\0\0\x05\0\xfd\
-\xff\0\0\0\0\x71\xa1\x51\xff\0\0\0\0\x79\xa0\x30\xff\0\0\0\0\x15\x01\x17\x01\0\
-\0\0\0\x71\x62\x03\0\0\0\0\0\x67\x02\0\0\x08\0\0\0\x71\x61\x02\0\0\0\0\0\x4f\
-\x12\0\0\0\0\0\0\x71\x63\x04\0\0\0\0\0\x71\x61\x05\0\0\0\0\0\x67\x01\0\0\x08\0\
-\0\0\x4f\x31\0\0\0\0\0\0\x67\x01\0\0\x10\0\0\0\x4f\x21\0\0\0\0\0\0\x71\xa2\x53\
-\xff\0\0\0\0\x15\x02\x3d\0\0\0\0\0\xbf\x12\0\0\0\0\0\0\x57\x02\0\0\x10\0\0\0\
-\x15\x02\x3a\0\0\0\0\0\xbf\xa2\0\0\0\0\0\0\x07\x02\0\0\x5c\xff\xff\xff\x71\xa4\
-\x54\xff\0\0\0\0\xbf\x23\0\0\0\0\0\0\x15\x04\x02\0\0\0\0\0\xbf\xa3\0\0\0\0\0\0\
-\x07\x03\0\0\x7c\xff\xff\xff\x67\x01\0\0\x38\0\0\0\xc7\x01\0\0\x38\0\0\0\x65\
-\x01\x01\0\xff\xff\xff\xff\xbf\x32\0\0\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\
-\x6c\xff\xff\xff\x71\xa5\x55\xff\0\0\0\0\xbf\x34\0\0\0\0\0\0\x15\x05\x02\0\0\0\
-\0\0\xbf\xa4\0\0\0\0\0\0\x07\x04\0\0\x8c\xff\xff\xff\x65\x01\x01\0\xff\xff\xff\
-\xff\xbf\x43\0\0\0\0\0\0\x61\x21\x04\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x61\x24\0\
-\0\0\0\0\0\x4f\x41\0\0\0\0\0\0\x7b\x1a\xa0\xff\0\0\0\0\x61\x21\x08\0\0\0\0\0\
-\x61\x22\x0c\0\0\0\0\0\x67\x02\0\0\x20\0\0\0\x4f\x12\0\0\0\0\0\0\x7b\x2a\xa8\
-\xff\0\0\0\0\x61\x31\0\0\0\0\0\0\x61\x32\x04\0\0\0\0\0\x61\x34\x08\0\0\0\0\0\
-\x61\x33\x0c\0\0\0\0\0\x69\xa5\x5a\xff\0\0\0\0\x6b\x5a\xc2\xff\0\0\0\0\x69\xa5\
-\x58\xff\0\0\0\0\x6b\x5a\xc0\xff\0\0\0\0\x67\x03\0\0\x20\0\0\0\x4f\x43\0\0\0\0\
-\0\0\x7b\x3a\xb8\xff\0\0\0\0\x67\x02\0\0\x20\0\0\0\x4f\x12\0\0\0\0\0\0\x7b\x2a\
-\xb0\xff\0\0\0\0\x05\0\x6b\0\0\0\0\0\x71\xa2\x52\xff\0\0\0\0\x15\x02\x04\0\0\0\
-\0\0\xbf\x12\0\0\0\0\0\0\x57\x02\0\0\x04\0\0\0\x15\x02\x01\0\0\0\0\0\x05\0\xf7\
-\xfe\0\0\0\0\x57\x01\0\0\x01\0\0\0\x15\x01\xd3\0\0\0\0\0\x61\xa1\x5c\xff\0\0\0\
-\0\x63\x1a\xa0\xff\0\0\0\0\x61\xa1\x60\xff\0\0\0\0\x63\x1a\xa4\xff\0\0\0\0\x05\
-\0\x5e\0\0\0\0\0\x71\xa2\x52\xff\0\0\0\0\x15\x02\x1e\0\0\0\0\0\xbf\x12\0\0\0\0\
-\0\0\x57\x02\0\0\x20\0\0\0\x15\x02\x1b\0\0\0\0\0\xbf\xa2\0\0\0\0\0\0\x07\x02\0\
-\0\x5c\xff\xff\xff\x71\xa4\x54\xff\0\0\0\0\xbf\x23\0\0\0\0\0\0\x15\x04\x02\0\0\
-\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\x7c\xff\xff\xff\x57\x01\0\0\0\x01\0\0\
-\x15\x01\x01\0\0\0\0\0\xbf\x32\0\0\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\x6c\
-\xff\xff\xff\x71\xa5\x55\xff\0\0\0\0\xbf\x34\0\0\0\0\0\0\x15\x05\x02\0\0\0\0\0\
-\xbf\xa4\0\0\0\0\0\0\x07\x04\0\0\x8c\xff\xff\xff\x15\x01\xc3\xff\0\0\0\0\x05\0\
-\xc1\xff\0\0\0\0\xb7\x09\0\0\x3c\0\0\0\x79\xa7\x20\xff\0\0\0\0\x67\0\0\0\x20\0\
-\0\0\x77\0\0\0\x20\0\0\0\x15\0\xa5\xfe\0\0\0\0\x05\0\xb0\0\0\0\0\0\x15\x09\x07\
-\xff\x87\0\0\0\x05\0\xa2\xfe\0\0\0\0\xbf\x12\0\0\0\0\0\0\x57\x02\0\0\x08\0\0\0\
-\x15\x02\xab\0\0\0\0\0\xbf\xa2\0\0\0\0\0\0\x07\x02\0\0\x5c\xff\xff\xff\x71\xa4\
-\x54\xff\0\0\0\0\xbf\x23\0\0\0\0\0\0\x15\x04\x02\0\0\0\0\0\xbf\xa3\0\0\0\0\0\0\
-\x07\x03\0\0\x7c\xff\xff\xff\x57\x01\0\0\x40\0\0\0\x15\x01\x01\0\0\0\0\0\xbf\
-\x32\0\0\0\0\0\0\x61\x23\x04\0\0\0\0\0\x67\x03\0\0\x20\0\0\0\x61\x24\0\0\0\0\0\
-\0\x4f\x43\0\0\0\0\0\0\x7b\x3a\xa0\xff\0\0\0\0\x61\x23\x08\0\0\0\0\0\x61\x22\
-\x0c\0\0\0\0\0\x67\x02\0\0\x20\0\0\0\x4f\x32\0\0\0\0\0\0\x7b\x2a\xa8\xff\0\0\0\
-\0\x15\x01\x1c\0\0\0\0\0\x71\xa1\x55\xff\0\0\0\0\x15\x01\x1a\0\0\0\0\0\x61\xa1\
-\x98\xff\0\0\0\0\x67\x01\0\0\x20\0\0\0\x61\xa2\x94\xff\0\0\0\0\x4f\x21\0\0\0\0\
-\0\0\x7b\x1a\xb8\xff\0\0\0\0\x61\xa1\x90\xff\0\0\0\0\x67\x01\0\0\x20\0\0\0\x61\
-\xa2\x8c\xff\0\0\0\0\x05\0\x19\0\0\0\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\x52\xff\
-\0\0\0\0\xb7\x01\0\0\0\0\0\0\x7b\x1a\xd0\xff\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\
-\x03\0\0\xd0\xff\xff\xff\xbf\x81\0\0\0\0\0\0\x79\xa2\x40\xff\0\0\0\0\xb7\x04\0\
-\0\x08\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\x67\0\0\0\x20\0\0\0\x77\
-\0\0\0\x20\0\0\0\x55\0\x7d\0\0\0\0\0\x05\0\x88\xfe\0\0\0\0\xb7\x09\0\0\x2b\0\0\
-\0\x05\0\xc6\xff\0\0\0\0\x61\xa1\x78\xff\0\0\0\0\x67\x01\0\0\x20\0\0\0\x61\xa2\
-\x74\xff\0\0\0\0\x4f\x21\0\0\0\0\0\0\x7b\x1a\xb8\xff\0\0\0\0\x61\xa1\x70\xff\0\
-\0\0\0\x67\x01\0\0\x20\0\0\0\x61\xa2\x6c\xff\0\0\0\0\x4f\x21\0\0\0\0\0\0\x7b\
-\x1a\xb0\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x07\x07\0\0\x04\0\0\0\x61\x03\0\0\0\0\
-\0\0\xb7\x05\0\0\0\0\0\0\x05\0\x4e\0\0\0\0\0\xaf\x52\0\0\0\0\0\0\xbf\x75\0\0\0\
-\0\0\0\x0f\x15\0\0\0\0\0\0\x71\x55\0\0\0\0\0\0\x67\x03\0\0\x01\0\0\0\xbf\x50\0\
-\0\0\0\0\0\x77\0\0\0\x07\0\0\0\x4f\x03\0\0\0\0\0\0\xbf\x40\0\0\0\0\0\0\x67\0\0\
-\0\x39\0\0\0\xc7\0\0\0\x3f\0\0\0\x5f\x30\0\0\0\0\0\0\xaf\x02\0\0\0\0\0\0\xbf\
-\x50\0\0\0\0\0\0\x77\0\0\0\x06\0\0\0\x57\0\0\0\x01\0\0\0\x67\x03\0\0\x01\0\0\0\
-\x4f\x03\0\0\0\0\0\0\xbf\x40\0\0\0\0\0\0\x67\0\0\0\x3a\0\0\0\xc7\0\0\0\x3f\0\0\
-\0\x5f\x30\0\0\0\0\0\0\xaf\x02\0\0\0\0\0\0\x67\x03\0\0\x01\0\0\0\xbf\x50\0\0\0\
-\0\0\0\x77\0\0\0\x05\0\0\0\x57\0\0\0\x01\0\0\0\x4f\x03\0\0\0\0\0\0\xbf\x40\0\0\
-\0\0\0\0\x67\0\0\0\x3b\0\0\0\xc7\0\0\0\x3f\0\0\0\x5f\x30\0\0\0\0\0\0\xaf\x02\0\
-\0\0\0\0\0\x67\x03\0\0\x01\0\0\0\xbf\x50\0\0\0\0\0\0\x77\0\0\0\x04\0\0\0\x57\0\
-\0\0\x01\0\0\0\x4f\x03\0\0\0\0\0\0\xbf\x40\0\0\0\0\0\0\x67\0\0\0\x3c\0\0\0\xc7\
-\0\0\0\x3f\0\0\0\x5f\x30\0\0\0\0\0\0\xaf\x02\0\0\0\0\0\0\xbf\x50\0\0\0\0\0\0\
-\x77\0\0\0\x03\0\0\0\x57\0\0\0\x01\0\0\0\x67\x03\0\0\x01\0\0\0\x4f\x03\0\0\0\0\
-\0\0\xbf\x40\0\0\0\0\0\0\x67\0\0\0\x3d\0\0\0\xc7\0\0\0\x3f\0\0\0\x5f\x30\0\0\0\
-\0\0\0\xaf\x02\0\0\0\0\0\0\xbf\x50\0\0\0\0\0\0\x77\0\0\0\x02\0\0\0\x57\0\0\0\
-\x01\0\0\0\x67\x03\0\0\x01\0\0\0\x4f\x03\0\0\0\0\0\0\xbf\x40\0\0\0\0\0\0\x67\0\
-\0\0\x3e\0\0\0\xc7\0\0\0\x3f\0\0\0\x5f\x30\0\0\0\0\0\0\xaf\x02\0\0\0\0\0\0\xbf\
-\x50\0\0\0\0\0\0\x77\0\0\0\x01\0\0\0\x57\0\0\0\x01\0\0\0\x67\x03\0\0\x01\0\0\0\
-\x4f\x03\0\0\0\0\0\0\x57\x04\0\0\x01\0\0\0\x87\x04\0\0\0\0\0\0\x5f\x34\0\0\0\0\
-\0\0\xaf\x42\0\0\0\0\0\0\x57\x05\0\0\x01\0\0\0\x67\x03\0\0\x01\0\0\0\x4f\x53\0\
-\0\0\0\0\0\x07\x01\0\0\x01\0\0\0\xbf\x25\0\0\0\0\0\0\x15\x01\x0b\0\x24\0\0\0\
-\xbf\xa2\0\0\0\0\0\0\x07\x02\0\0\xa0\xff\xff\xff\x0f\x12\0\0\0\0\0\0\x71\x24\0\
-\0\0\0\0\0\xbf\x40\0\0\0\0\0\0\x67\0\0\0\x38\0\0\0\xc7\0\0\0\x38\0\0\0\xb7\x02\
-\0\0\0\0\0\0\x65\0\xa9\xff\xff\xff\xff\xff\xbf\x32\0\0\0\0\0\0\x05\0\xa7\xff\0\
-\0\0\0\xbf\x21\0\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\0\0\0\x15\x01\
-\x0e\0\0\0\0\0\x71\x63\x06\0\0\0\0\0\x71\x64\x07\0\0\0\0\0\x67\x04\0\0\x08\0\0\
-\0\x4f\x34\0\0\0\0\0\0\x3f\x41\0\0\0\0\0\0\x2f\x41\0\0\0\0\0\0\x1f\x12\0\0\0\0\
-\0\0\x63\x2a\x50\xff\0\0\0\0\xbf\xa2\0\0\0\0\0\0\x07\x02\0\0\x50\xff\xff\xff\
-\x18\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x85\0\0\0\x01\0\0\0\x55\0\x05\0\0\0\0\0\
-\x71\x61\x08\0\0\0\0\0\x71\x60\x09\0\0\0\0\0\x67\0\0\0\x08\0\0\0\x4f\x10\0\0\0\
-\0\0\0\x95\0\0\0\0\0\0\0\x69\0\0\0\0\0\0\0\x05\0\xfd\xff\0\0\0\0\x02\0\0\0\x04\
-\0\0\0\x0a\0\0\0\x01\0\0\0\0\0\0\0\x02\0\0\0\x04\0\0\0\x28\0\0\0\x01\0\0\0\0\0\
-\0\0\x02\0\0\0\x04\0\0\0\x02\0\0\0\x80\0\0\0\0\0\0\0\x47\x50\x4c\x20\x76\x32\0\
-\0\0\0\0\0\x10\0\0\0\0\0\0\0\x01\x7a\x52\0\x08\x7c\x0b\x01\x0c\0\0\0\x18\0\0\0\
-\x18\0\0\0\0\0\0\0\0\0\0\0\xd8\x13\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
-\0\0\0\0\0\0\0\0\0\0\0\0\xa0\0\0\0\x04\0\xf1\xff\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
-\0\x60\x02\0\0\0\0\x03\0\x20\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x3f\x02\0\0\0\0\
-\x03\0\xd0\x0f\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xed\x01\0\0\0\0\x03\0\x10\x10\0\0\0\
-\0\0\0\0\0\0\0\0\0\0\0\xd4\x01\0\0\0\0\x03\0\x20\x10\0\0\0\0\0\0\0\0\0\0\0\0\0\
-\0\xa3\x01\0\0\0\0\x03\0\xb8\x12\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x63\x01\0\0\0\0\
-\x03\0\x48\x10\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x2a\x01\0\0\0\0\x03\0\x10\x13\0\0\0\
-\0\0\0\0\0\0\0\0\0\0\0\xe1\0\0\0\0\0\x03\0\xa0\x13\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
-\x2e\x02\0\0\0\0\x03\0\x28\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x68\x02\0\0\0\0\x03\
-\0\xc0\x13\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x36\x02\0\0\0\0\x03\0\xc8\x13\0\0\0\0\0\
-\0\0\0\0\0\0\0\0\0\x22\x01\0\0\0\0\x03\0\xe8\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
-\x02\x01\0\0\0\0\x03\0\x40\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xd9\0\0\0\0\0\x03\0\
-\xf8\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x26\x02\0\0\0\0\x03\0\x20\x0e\0\0\0\0\0\0\
-\0\0\0\0\0\0\0\0\xcc\x01\0\0\0\0\x03\0\x60\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x9b\
-\x01\0\0\0\0\x03\0\xc8\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x5b\x01\0\0\0\0\x03\0\
-\x20\x07\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x7c\x01\0\0\0\0\x03\0\x48\x08\0\0\0\0\0\0\
-\0\0\0\0\0\0\0\0\x53\x01\0\0\0\0\x03\0\xb8\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x1a\
-\x01\0\0\0\0\x03\0\xe0\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x84\x01\0\0\0\0\x03\0\
-\xb8\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x1e\x02\0\0\0\0\x03\0\xd8\x09\0\0\0\0\0\0\0\
-\0\0\0\0\0\0\0\xc4\x01\0\0\0\0\x03\0\x70\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x93\
-\x01\0\0\0\0\x03\0\xa8\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x74\x01\0\0\0\0\x03\0\
-\xf0\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x4b\x01\0\0\0\0\x03\0\0\x0a\0\0\0\0\0\0\0\
-\0\0\0\0\0\0\0\x12\x01\0\0\0\0\x03\0\x10\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xfa\0\
-\0\0\0\0\x03\0\xc0\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x58\x02\0\0\0\0\x03\0\x88\
-\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x16\x02\0\0\0\0\x03\0\xb8\x0a\0\0\0\0\0\0\0\0\
-\0\0\0\0\0\0\xe5\x01\0\0\0\0\x03\0\xc0\x0f\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xbc\x01\
-\0\0\0\0\x03\0\0\x0e\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x8b\x01\0\0\0\0\x03\0\x18\x0e\
-\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xd1\0\0\0\0\0\x03\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\
-\0\0\x50\x02\0\0\0\0\x03\0\x20\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x0e\x02\0\0\0\0\
-\x03\0\x48\x0f\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x6c\x01\0\0\0\0\x03\0\xb0\x04\0\0\0\
-\0\0\0\0\0\0\0\0\0\0\0\x43\x01\0\0\0\0\x03\0\xc8\x0c\0\0\0\0\0\0\0\0\0\0\0\0\0\
-\0\xc9\0\0\0\0\0\x03\0\xf8\x0c\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x06\x02\0\0\0\0\x03\
-\0\xd0\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x3b\x01\0\0\0\0\x03\0\x98\x0b\0\0\0\0\0\
-\0\0\0\0\0\0\0\0\0\xf2\0\0\0\0\0\x03\0\xb8\x0b\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x48\
-\x02\0\0\0\0\x03\0\xf0\x0b\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xfe\x01\0\0\0\0\x03\0\
-\xf8\x0b\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xdd\x01\0\0\0\0\x03\0\0\x0c\0\0\0\0\0\0\0\
-\0\0\0\0\0\0\0\xb4\x01\0\0\0\0\x03\0\x30\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x0a\
-\x01\0\0\0\0\x03\0\x90\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xc1\0\0\0\0\0\x03\0\xa8\
-\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xba\0\0\0\0\0\x03\0\xd0\x01\0\0\0\0\0\0\0\0\0\
-\0\0\0\0\0\xf6\x01\0\0\0\0\x03\0\xe0\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xac\x01\0\
-\0\0\0\x03\0\x30\x0e\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x33\x01\0\0\0\0\x03\0\x80\x0e\
-\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xea\0\0\0\0\0\x03\0\x98\x0e\0\0\0\0\0\0\0\0\0\0\0\
-\0\0\0\0\0\0\0\x03\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x6b\0\0\0\x11\0\x06\
-\0\0\0\0\0\0\0\0\0\x07\0\0\0\0\0\0\0\x25\0\0\0\x11\0\x05\0\0\0\0\0\0\0\0\0\x14\
-\0\0\0\0\0\0\0\x82\0\0\0\x11\0\x05\0\x28\0\0\0\0\0\0\0\x14\0\0\0\0\0\0\0\x01\0\
-\0\0\x11\0\x05\0\x14\0\0\0\0\0\0\0\x14\0\0\0\0\0\0\0\x40\0\0\0\x12\0\x03\0\0\0\
-\0\0\0\0\0\0\xd8\x13\0\0\0\0\0\0\x28\0\0\0\0\0\0\0\x01\0\0\0\x3a\0\0\0\x50\0\0\
-\0\0\0\0\0\x01\0\0\0\x3c\0\0\0\x80\x13\0\0\0\0\0\0\x01\0\0\0\x3b\0\0\0\x1c\0\0\
-\0\0\0\0\0\x01\0\0\0\x38\0\0\0\0\x74\x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\
-\x5f\x74\x6f\x65\x70\x6c\x69\x74\x7a\x5f\x6b\x65\x79\0\x2e\x74\x65\x78\x74\0\
-\x6d\x61\x70\x73\0\x74\x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\x5f\x63\x6f\x6e\
-\x66\x69\x67\x75\x72\x61\x74\x69\x6f\x6e\x73\0\x74\x75\x6e\x5f\x72\x73\x73\x5f\
-\x73\x74\x65\x65\x72\x69\x6e\x67\x5f\x70\x72\x6f\x67\0\x2e\x72\x65\x6c\x74\x75\
-\x6e\x5f\x72\x73\x73\x5f\x73\x74\x65\x65\x72\x69\x6e\x67\0\x5f\x6c\x69\x63\x65\
-\x6e\x73\x65\0\x2e\x72\x65\x6c\x2e\x65\x68\x5f\x66\x72\x61\x6d\x65\0\x74\x61\
-\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\x5f\x69\x6e\x64\x69\x72\x65\x63\x74\x69\
-\x6f\x6e\x5f\x74\x61\x62\x6c\x65\0\x72\x73\x73\x2e\x62\x70\x66\x2e\x63\0\x2e\
-\x73\x74\x72\x74\x61\x62\0\x2e\x73\x79\x6d\x74\x61\x62\0\x4c\x42\x42\x30\x5f\
-\x39\0\x4c\x42\x42\x30\x5f\x38\x39\0\x4c\x42\x42\x30\x5f\x36\x39\0\x4c\x42\x42\
-\x30\x5f\x35\x39\0\x4c\x42\x42\x30\x5f\x31\x39\0\x4c\x42\x42\x30\x5f\x31\x30\
-\x39\0\x4c\x42\x42\x30\x5f\x39\x38\0\x4c\x42\x42\x30\x5f\x37\x38\0\x4c\x42\x42\
-\x30\x5f\x34\x38\0\x4c\x42\x42\x30\x5f\x31\x38\0\x4c\x42\x42\x30\x5f\x38\x37\0\
-\x4c\x42\x42\x30\x5f\x34\x37\0\x4c\x42\x42\x30\x5f\x33\x37\0\x4c\x42\x42\x30\
-\x5f\x31\x37\0\x4c\x42\x42\x30\x5f\x31\x30\x37\0\x4c\x42\x42\x30\x5f\x39\x36\0\
-\x4c\x42\x42\x30\x5f\x37\x36\0\x4c\x42\x42\x30\x5f\x36\x36\0\x4c\x42\x42\x30\
-\x5f\x34\x36\0\x4c\x42\x42\x30\x5f\x33\x36\0\x4c\x42\x42\x30\x5f\x32\x36\0\x4c\
-\x42\x42\x30\x5f\x31\x30\x36\0\x4c\x42\x42\x30\x5f\x36\x35\0\x4c\x42\x42\x30\
-\x5f\x34\x35\0\x4c\x42\x42\x30\x5f\x33\x35\0\x4c\x42\x42\x30\x5f\x34\0\x4c\x42\
+\0\0\0\0\0\x05\0\xf6\0\0\0\0\0\xb7\x01\0\0\0\0\0\0\x6b\x1a\xfe\xff\0\0\0\0\xb7\
+\x07\0\0\x28\0\0\0\xbf\xa1\0\0\0\0\0\0\x07\x01\0\0\x8c\xff\xff\xff\x7b\x1a\x28\
+\xff\0\0\0\0\xbf\xa1\0\0\0\0\0\0\x07\x01\0\0\x7c\xff\xff\xff\x7b\x1a\x20\xff\0\
+\0\0\0\xb7\x01\0\0\0\0\0\0\x7b\x1a\x30\xff\0\0\0\0\x7b\x8a\x38\xff\0\0\0\0\xbf\
+\xa3\0\0\0\0\0\0\x07\x03\0\0\xfe\xff\xff\xff\xbf\x81\0\0\0\0\0\0\xbf\x72\0\0\0\
+\0\0\0\xb7\x04\0\0\x02\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\x67\0\0\
+\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\x15\0\x01\0\0\0\0\0\x05\0\x62\xff\0\0\0\0\xbf\
+\x91\0\0\0\0\0\0\x15\x01\x23\0\x3c\0\0\0\x15\x01\x5a\0\x2c\0\0\0\x55\x01\x5b\0\
+\x2b\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\xf8\xff\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\
+\x03\0\0\xf8\xff\xff\xff\xbf\x81\0\0\0\0\0\0\xbf\x72\0\0\0\0\0\0\xb7\x04\0\0\
+\x04\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\xbf\x01\0\0\0\0\0\0\x67\
+\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\0\0\0\x55\x01\x02\x01\0\0\0\0\x71\xa1\xfa\
+\xff\0\0\0\0\x55\x01\x4c\0\x02\0\0\0\x71\xa1\xf9\xff\0\0\0\0\x55\x01\x4a\0\x02\
+\0\0\0\x71\xa1\xfb\xff\0\0\0\0\x55\x01\x48\0\x01\0\0\0\xbf\x72\0\0\0\0\0\0\x07\
+\x02\0\0\x08\0\0\0\xbf\x81\0\0\0\0\0\0\x79\xa3\x28\xff\0\0\0\0\xb7\x04\0\0\x10\
+\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\xbf\x01\0\0\0\0\0\0\x67\x01\0\
+\0\x20\0\0\0\x77\x01\0\0\x20\0\0\0\x55\x01\xf1\0\0\0\0\0\xb7\x01\0\0\x01\0\0\0\
+\x73\x1a\x55\xff\0\0\0\0\x05\0\x3a\0\0\0\0\0\x7b\x7a\x40\xff\0\0\0\0\xb7\x01\0\
+\0\0\0\0\0\x6b\x1a\xf8\xff\0\0\0\0\xb7\x07\0\0\x02\0\0\0\xb7\x08\0\0\x1e\0\0\0\
+\x05\0\x0f\0\0\0\0\0\x0f\x71\0\0\0\0\0\0\xbf\x12\0\0\0\0\0\0\x07\x02\0\0\x01\0\
+\0\0\x71\xa3\xff\xff\0\0\0\0\x67\x03\0\0\x03\0\0\0\x2d\x23\x03\0\0\0\0\0\x79\
+\xa8\x38\xff\0\0\0\0\x79\xa7\x40\xff\0\0\0\0\x05\0\x2b\0\0\0\0\0\x07\x08\0\0\
+\xff\xff\xff\xff\xbf\x82\0\0\0\0\0\0\x67\x02\0\0\x20\0\0\0\x77\x02\0\0\x20\0\0\
+\0\xbf\x17\0\0\0\0\0\0\x15\x02\xf7\xff\0\0\0\0\xbf\x79\0\0\0\0\0\0\x79\xa1\x40\
+\xff\0\0\0\0\x0f\x19\0\0\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\xf8\xff\xff\
+\xff\x79\xa1\x38\xff\0\0\0\0\xbf\x92\0\0\0\0\0\0\xb7\x04\0\0\x02\0\0\0\xb7\x05\
+\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\xbf\x01\0\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\
+\x77\x01\0\0\x20\0\0\0\x55\x01\x90\0\0\0\0\0\x71\xa2\xf8\xff\0\0\0\0\x55\x02\
+\x0e\0\xc9\0\0\0\x07\x09\0\0\x02\0\0\0\x79\xa1\x38\xff\0\0\0\0\xbf\x92\0\0\0\0\
+\0\0\x79\xa3\x20\xff\0\0\0\0\xb7\x04\0\0\x10\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\
+\0\0\x44\0\0\0\xbf\x01\0\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\0\0\0\
+\x55\x01\x83\0\0\0\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\x54\xff\0\0\0\0\x05\0\xd9\
+\xff\0\0\0\0\xb7\x01\0\0\x01\0\0\0\x15\x02\xd1\xff\0\0\0\0\x71\xa1\xf9\xff\0\0\
+\0\0\x07\x01\0\0\x02\0\0\0\x05\0\xce\xff\0\0\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\
+\x56\xff\0\0\0\0\x71\xa1\xff\xff\0\0\0\0\x67\x01\0\0\x03\0\0\0\x0f\x17\0\0\0\0\
+\0\0\x07\x07\0\0\x08\0\0\0\x71\xa9\xfe\xff\0\0\0\0\x25\x09\x0e\0\x3c\0\0\0\xb7\
+\x01\0\0\x01\0\0\0\x6f\x91\0\0\0\0\0\0\x18\x02\0\0\x01\0\0\0\0\0\0\0\0\x18\0\
+\x1c\x5f\x21\0\0\0\0\0\0\x55\x01\x01\0\0\0\0\0\x05\0\x07\0\0\0\0\0\x79\xa1\x30\
+\xff\0\0\0\0\x07\x01\0\0\x01\0\0\0\x7b\x1a\x30\xff\0\0\0\0\x67\x01\0\0\x20\0\0\
+\0\x77\x01\0\0\x20\0\0\0\x55\x01\x83\xff\x0b\0\0\0\x05\0\x14\xff\0\0\0\0\x15\
+\x09\xf8\xff\x87\0\0\0\x05\0\xfd\xff\0\0\0\0\x71\xa1\x51\xff\0\0\0\0\x15\x01\
+\xbd\xfe\0\0\0\0\x71\x62\x03\0\0\0\0\0\x67\x02\0\0\x08\0\0\0\x71\x61\x02\0\0\0\
+\0\0\x4f\x12\0\0\0\0\0\0\x71\x63\x04\0\0\0\0\0\x71\x61\x05\0\0\0\0\0\x67\x01\0\
+\0\x08\0\0\0\x4f\x31\0\0\0\0\0\0\x67\x01\0\0\x10\0\0\0\x4f\x21\0\0\0\0\0\0\x71\
+\xa2\x53\xff\0\0\0\0\x15\x02\x3d\0\0\0\0\0\xbf\x12\0\0\0\0\0\0\x57\x02\0\0\x10\
+\0\0\0\x15\x02\x3a\0\0\0\0\0\xbf\xa2\0\0\0\0\0\0\x07\x02\0\0\x5c\xff\xff\xff\
+\x71\xa4\x54\xff\0\0\0\0\xbf\x23\0\0\0\0\0\0\x15\x04\x02\0\0\0\0\0\xbf\xa3\0\0\
+\0\0\0\0\x07\x03\0\0\x7c\xff\xff\xff\x67\x01\0\0\x38\0\0\0\xc7\x01\0\0\x38\0\0\
+\0\x65\x01\x01\0\xff\xff\xff\xff\xbf\x32\0\0\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\
+\x03\0\0\x6c\xff\xff\xff\x71\xa5\x55\xff\0\0\0\0\xbf\x34\0\0\0\0\0\0\x15\x05\
+\x02\0\0\0\0\0\xbf\xa4\0\0\0\0\0\0\x07\x04\0\0\x8c\xff\xff\xff\x65\x01\x01\0\
+\xff\xff\xff\xff\xbf\x43\0\0\0\0\0\0\x61\x21\x04\0\0\0\0\0\x67\x01\0\0\x20\0\0\
+\0\x61\x24\0\0\0\0\0\0\x4f\x41\0\0\0\0\0\0\x7b\x1a\xa0\xff\0\0\0\0\x61\x21\x08\
+\0\0\0\0\0\x61\x22\x0c\0\0\0\0\0\x67\x02\0\0\x20\0\0\0\x4f\x12\0\0\0\0\0\0\x7b\
+\x2a\xa8\xff\0\0\0\0\x61\x31\0\0\0\0\0\0\x61\x32\x04\0\0\0\0\0\x61\x34\x08\0\0\
+\0\0\0\x61\x33\x0c\0\0\0\0\0\x69\xa5\x5a\xff\0\0\0\0\x6b\x5a\xc2\xff\0\0\0\0\
+\x69\xa5\x58\xff\0\0\0\0\x6b\x5a\xc0\xff\0\0\0\0\x67\x03\0\0\x20\0\0\0\x4f\x43\
+\0\0\0\0\0\0\x7b\x3a\xb8\xff\0\0\0\0\x67\x02\0\0\x20\0\0\0\x4f\x12\0\0\0\0\0\0\
+\x7b\x2a\xb0\xff\0\0\0\0\x05\0\x6c\0\0\0\0\0\x71\xa2\x52\xff\0\0\0\0\x15\x02\
+\x04\0\0\0\0\0\xbf\x12\0\0\0\0\0\0\x57\x02\0\0\x04\0\0\0\x15\x02\x01\0\0\0\0\0\
+\x05\0\xfb\xfe\0\0\0\0\x57\x01\0\0\x01\0\0\0\x15\x01\x79\xfe\0\0\0\0\x61\xa1\
+\x5c\xff\0\0\0\0\x63\x1a\xa0\xff\0\0\0\0\x61\xa1\x60\xff\0\0\0\0\x63\x1a\xa4\
+\xff\0\0\0\0\x05\0\x5f\0\0\0\0\0\x71\xa2\x52\xff\0\0\0\0\x15\x02\x1f\0\0\0\0\0\
+\xbf\x12\0\0\0\0\0\0\x57\x02\0\0\x20\0\0\0\x15\x02\x1c\0\0\0\0\0\xbf\xa2\0\0\0\
+\0\0\0\x07\x02\0\0\x5c\xff\xff\xff\x71\xa4\x54\xff\0\0\0\0\xbf\x23\0\0\0\0\0\0\
+\x15\x04\x02\0\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\x7c\xff\xff\xff\x57\x01\
+\0\0\0\x01\0\0\x15\x01\x01\0\0\0\0\0\xbf\x32\0\0\0\0\0\0\xbf\xa3\0\0\0\0\0\0\
+\x07\x03\0\0\x6c\xff\xff\xff\x71\xa5\x55\xff\0\0\0\0\xbf\x34\0\0\0\0\0\0\x15\
+\x05\x02\0\0\0\0\0\xbf\xa4\0\0\0\0\0\0\x07\x04\0\0\x8c\xff\xff\xff\x15\x01\xc3\
+\xff\0\0\0\0\x05\0\xc1\xff\0\0\0\0\xb7\x09\0\0\x3c\0\0\0\x79\xa8\x38\xff\0\0\0\
+\0\x79\xa7\x40\xff\0\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\x15\0\xa9\
+\xfe\0\0\0\0\x05\0\x83\xfe\0\0\0\0\x15\x09\x09\xff\x87\0\0\0\x05\0\xa6\xfe\0\0\
+\0\0\xbf\x12\0\0\0\0\0\0\x57\x02\0\0\x08\0\0\0\x15\x02\x50\xfe\0\0\0\0\xbf\xa2\
+\0\0\0\0\0\0\x07\x02\0\0\x5c\xff\xff\xff\x71\xa4\x54\xff\0\0\0\0\xbf\x23\0\0\0\
+\0\0\0\x15\x04\x02\0\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\x7c\xff\xff\xff\
+\x57\x01\0\0\x40\0\0\0\x15\x01\x01\0\0\0\0\0\xbf\x32\0\0\0\0\0\0\x61\x23\x04\0\
+\0\0\0\0\x67\x03\0\0\x20\0\0\0\x61\x24\0\0\0\0\0\0\x4f\x43\0\0\0\0\0\0\x7b\x3a\
+\xa0\xff\0\0\0\0\x61\x23\x08\0\0\0\0\0\x61\x22\x0c\0\0\0\0\0\x67\x02\0\0\x20\0\
+\0\0\x4f\x32\0\0\0\0\0\0\x7b\x2a\xa8\xff\0\0\0\0\x15\x01\x1c\0\0\0\0\0\x71\xa1\
+\x55\xff\0\0\0\0\x15\x01\x1a\0\0\0\0\0\x61\xa1\x98\xff\0\0\0\0\x67\x01\0\0\x20\
+\0\0\0\x61\xa2\x94\xff\0\0\0\0\x4f\x21\0\0\0\0\0\0\x7b\x1a\xb8\xff\0\0\0\0\x61\
+\xa1\x90\xff\0\0\0\0\x67\x01\0\0\x20\0\0\0\x61\xa2\x8c\xff\0\0\0\0\x05\0\x19\0\
+\0\0\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\x52\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x7b\
+\x1a\xd0\xff\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\xd0\xff\xff\xff\xbf\x81\0\
+\0\0\0\0\0\xbf\x72\0\0\0\0\0\0\xb7\x04\0\0\x08\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\
+\0\0\0\x44\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\x55\0\x22\xfe\0\0\0\0\
+\x05\0\x8c\xfe\0\0\0\0\xb7\x09\0\0\x2b\0\0\0\x05\0\xc6\xff\0\0\0\0\x61\xa1\x78\
+\xff\0\0\0\0\x67\x01\0\0\x20\0\0\0\x61\xa2\x74\xff\0\0\0\0\x4f\x21\0\0\0\0\0\0\
+\x7b\x1a\xb8\xff\0\0\0\0\x61\xa1\x70\xff\0\0\0\0\x67\x01\0\0\x20\0\0\0\x61\xa2\
+\x6c\xff\0\0\0\0\x4f\x21\0\0\0\0\0\0\x7b\x1a\xb0\xff\0\0\0\0\xb7\x02\0\0\0\0\0\
+\0\xbf\x63\0\0\0\0\0\0\x07\x03\0\0\x0e\0\0\0\x61\x64\x0a\0\0\0\0\0\xb7\0\0\0\0\
+\0\0\0\x05\0\x4e\0\0\0\0\0\xaf\x01\0\0\0\0\0\0\xbf\x30\0\0\0\0\0\0\x0f\x20\0\0\
+\0\0\0\0\x71\0\0\0\0\0\0\0\x67\x04\0\0\x01\0\0\0\xbf\x07\0\0\0\0\0\0\x77\x07\0\
+\0\x07\0\0\0\x4f\x74\0\0\0\0\0\0\xbf\x57\0\0\0\0\0\0\x67\x07\0\0\x39\0\0\0\xc7\
+\x07\0\0\x3f\0\0\0\x5f\x47\0\0\0\0\0\0\xaf\x71\0\0\0\0\0\0\xbf\x07\0\0\0\0\0\0\
+\x77\x07\0\0\x06\0\0\0\x57\x07\0\0\x01\0\0\0\x67\x04\0\0\x01\0\0\0\x4f\x74\0\0\
+\0\0\0\0\xbf\x57\0\0\0\0\0\0\x67\x07\0\0\x3a\0\0\0\xc7\x07\0\0\x3f\0\0\0\x5f\
+\x47\0\0\0\0\0\0\xaf\x71\0\0\0\0\0\0\x67\x04\0\0\x01\0\0\0\xbf\x07\0\0\0\0\0\0\
+\x77\x07\0\0\x05\0\0\0\x57\x07\0\0\x01\0\0\0\x4f\x74\0\0\0\0\0\0\xbf\x57\0\0\0\
+\0\0\0\x67\x07\0\0\x3b\0\0\0\xc7\x07\0\0\x3f\0\0\0\x5f\x47\0\0\0\0\0\0\xaf\x71\
+\0\0\0\0\0\0\x67\x04\0\0\x01\0\0\0\xbf\x07\0\0\0\0\0\0\x77\x07\0\0\x04\0\0\0\
+\x57\x07\0\0\x01\0\0\0\x4f\x74\0\0\0\0\0\0\xbf\x57\0\0\0\0\0\0\x67\x07\0\0\x3c\
+\0\0\0\xc7\x07\0\0\x3f\0\0\0\x5f\x47\0\0\0\0\0\0\xaf\x71\0\0\0\0\0\0\xbf\x07\0\
+\0\0\0\0\0\x77\x07\0\0\x03\0\0\0\x57\x07\0\0\x01\0\0\0\x67\x04\0\0\x01\0\0\0\
+\x4f\x74\0\0\0\0\0\0\xbf\x57\0\0\0\0\0\0\x67\x07\0\0\x3d\0\0\0\xc7\x07\0\0\x3f\
+\0\0\0\x5f\x47\0\0\0\0\0\0\xaf\x71\0\0\0\0\0\0\xbf\x07\0\0\0\0\0\0\x77\x07\0\0\
+\x02\0\0\0\x57\x07\0\0\x01\0\0\0\x67\x04\0\0\x01\0\0\0\x4f\x74\0\0\0\0\0\0\xbf\
+\x57\0\0\0\0\0\0\x67\x07\0\0\x3e\0\0\0\xc7\x07\0\0\x3f\0\0\0\x5f\x47\0\0\0\0\0\
+\0\xaf\x71\0\0\0\0\0\0\xbf\x07\0\0\0\0\0\0\x77\x07\0\0\x01\0\0\0\x57\x07\0\0\
+\x01\0\0\0\x67\x04\0\0\x01\0\0\0\x4f\x74\0\0\0\0\0\0\x57\x05\0\0\x01\0\0\0\x87\
+\x05\0\0\0\0\0\0\x5f\x45\0\0\0\0\0\0\xaf\x51\0\0\0\0\0\0\x57\0\0\0\x01\0\0\0\
+\x67\x04\0\0\x01\0\0\0\x4f\x04\0\0\0\0\0\0\x07\x02\0\0\x01\0\0\0\xbf\x10\0\0\0\
+\0\0\0\x15\x02\x0b\0\x24\0\0\0\xbf\xa1\0\0\0\0\0\0\x07\x01\0\0\xa0\xff\xff\xff\
+\x0f\x21\0\0\0\0\0\0\x71\x15\0\0\0\0\0\0\xbf\x57\0\0\0\0\0\0\x67\x07\0\0\x38\0\
+\0\0\xc7\x07\0\0\x38\0\0\0\xb7\x01\0\0\0\0\0\0\x65\x07\xa9\xff\xff\xff\xff\xff\
+\xbf\x41\0\0\0\0\0\0\x05\0\xa7\xff\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\
+\x20\0\0\0\x15\x01\xb3\xfd\0\0\0\0\x71\x62\x06\0\0\0\0\0\x71\x63\x07\0\0\0\0\0\
+\x67\x03\0\0\x08\0\0\0\x4f\x23\0\0\0\0\0\0\xbf\x12\0\0\0\0\0\0\x3f\x32\0\0\0\0\
+\0\0\x2f\x32\0\0\0\0\0\0\x1f\x21\0\0\0\0\0\0\x25\x01\xaa\xfd\x7f\0\0\0\x67\x01\
+\0\0\x20\0\0\0\x77\x01\0\0\x20\0\0\0\x67\x01\0\0\x01\0\0\0\x0f\x16\0\0\0\0\0\0\
+\x07\x06\0\0\x32\0\0\0\x05\0\xd3\xfd\0\0\0\0\x02\0\0\0\x04\0\0\0\x32\x01\0\0\
+\x01\0\0\0\0\x04\0\0\x47\x50\x4c\x20\x76\x32\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x01\
+\x7a\x52\0\x08\x7c\x0b\x01\x0c\0\0\0\x18\0\0\0\x18\0\0\0\0\0\0\0\0\0\0\0\x50\
+\x13\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x69\0\
+\0\0\x04\0\xf1\xff\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x2e\x02\0\0\0\0\x03\0\xe8\
+\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x0d\x02\0\0\0\0\x03\0\x68\x0f\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\xe4\x01\0\0\0\0\x03\0\x78\x0f\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xb4\x01\
+\0\0\0\0\x03\0\xb8\x0f\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x94\x01\0\0\0\0\x03\0\xc8\
+\x0f\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x6b\x01\0\0\0\0\x03\0\xf8\x0f\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\x2a\x01\0\0\0\0\x03\0\x68\x12\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xe9\0\0\
+\0\0\0\x03\0\xc0\x12\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x05\x02\0\0\0\0\x03\0\x08\x02\
+\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xdc\x01\0\0\0\0\x03\0\x10\x02\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\xe1\0\0\0\0\0\x03\0\xc0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xb2\0\0\0\0\0\
+\x03\0\x08\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xbd\x01\0\0\0\0\x03\0\x70\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\x8c\x01\0\0\0\0\x03\0\xc8\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\x63\x01\0\0\0\0\x03\0\xe8\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xd9\0\0\0\0\0\x03\0\
+\x70\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xaa\0\0\0\0\0\x03\0\xb8\x04\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\x9d\x01\0\0\0\0\x03\0\x78\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xd4\x01\
+\0\0\0\0\x03\0\x18\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xac\x01\0\0\0\0\x03\0\x78\
+\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x5b\x01\0\0\0\0\x03\0\xd0\x06\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\x84\x01\0\0\0\0\x03\0\xf8\x07\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x53\x01\
+\0\0\0\0\x03\0\x28\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x22\x01\0\0\0\0\x03\0\x58\
+\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x12\x01\0\0\0\0\x03\0\x70\x08\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\xd1\0\0\0\0\0\x03\0\xa0\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xcc\x01\0\
+\0\0\0\x03\0\x90\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x7c\x01\0\0\0\0\x03\0\xb8\x09\
+\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x4b\x01\0\0\0\0\x03\0\xc8\x09\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\x0a\x01\0\0\0\0\x03\0\x30\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xc9\0\0\0\0\
+\0\x03\0\x60\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xa2\0\0\0\0\0\x03\0\x68\x0a\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\0\x26\x02\0\0\0\0\x03\0\x78\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\0\x43\x01\0\0\0\0\x03\0\x38\x0b\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x02\x01\0\0\0\0\
+\x03\0\x58\x0b\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x9a\0\0\0\0\0\x03\0\x90\x0b\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\x1e\x02\0\0\0\0\x03\0\x98\x0b\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\xfd\x01\0\0\0\0\x03\0\xa0\x0b\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xc4\x01\0\0\0\0\x03\
+\0\x68\x0c\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x3b\x01\0\0\0\0\x03\0\x98\x0c\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\xfa\0\0\0\0\0\x03\0\xd0\x0c\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xba\
+\0\0\0\0\0\x03\0\x90\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xf5\x01\0\0\0\0\x03\0\x30\
+\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xa4\x01\0\0\0\0\x03\0\x48\x0d\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\x33\x01\0\0\0\0\x03\0\x80\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xf2\0\0\
+\0\0\0\x03\0\x90\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xc1\0\0\0\0\0\x03\0\xa8\x0d\0\
+\0\0\0\0\0\0\0\0\0\0\0\0\0\x92\0\0\0\0\0\x03\0\xc0\x0d\0\0\0\0\0\0\0\0\0\0\0\0\
+\0\0\x83\0\0\0\0\0\x03\0\xe0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x16\x02\0\0\0\0\
+\x03\0\xc8\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xed\x01\0\0\0\0\x03\0\xd8\x0d\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\0\x74\x01\0\0\0\0\x03\0\x28\x0e\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\0\x1a\x01\0\0\0\0\x03\0\x40\x0e\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x8a\0\0\0\0\0\x03\
+\0\xf0\x0e\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\0\x03\0\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\x52\0\0\0\x11\0\x06\0\0\0\0\0\0\0\0\0\x07\0\0\0\0\0\0\0\x0c\0\0\0\
+\x11\0\x05\0\0\0\0\0\0\0\0\0\x14\0\0\0\0\0\0\0\x27\0\0\0\x12\0\x03\0\0\0\0\0\0\
+\0\0\0\x50\x13\0\0\0\0\0\0\x28\0\0\0\0\0\0\0\x01\0\0\0\x3a\0\0\0\x1c\0\0\0\0\0\
+\0\0\x01\0\0\0\x38\0\0\0\0\x2e\x74\x65\x78\x74\0\x6d\x61\x70\x73\0\x74\x61\x70\
+\x5f\x72\x73\x73\x5f\x6d\x61\x70\x5f\x63\x6f\x6e\x66\x69\x67\x75\x72\x61\x74\
+\x69\x6f\x6e\x73\0\x74\x75\x6e\x5f\x72\x73\x73\x5f\x73\x74\x65\x65\x72\x69\x6e\
+\x67\x5f\x70\x72\x6f\x67\0\x2e\x72\x65\x6c\x74\x75\x6e\x5f\x72\x73\x73\x5f\x73\
+\x74\x65\x65\x72\x69\x6e\x67\0\x5f\x6c\x69\x63\x65\x6e\x73\x65\0\x2e\x72\x65\
+\x6c\x2e\x65\x68\x5f\x66\x72\x61\x6d\x65\0\x72\x73\x73\x2e\x62\x70\x66\x2e\x63\
+\0\x2e\x73\x74\x72\x74\x61\x62\0\x2e\x73\x79\x6d\x74\x61\x62\0\x4c\x42\x42\x30\
+\x5f\x39\0\x4c\x42\x42\x30\x5f\x39\x39\0\x4c\x42\x42\x30\x5f\x38\x39\0\x4c\x42\
+\x42\x30\x5f\x36\x39\0\x4c\x42\x42\x30\x5f\x35\x39\0\x4c\x42\x42\x30\x5f\x32\
+\x39\0\x4c\x42\x42\x30\x5f\x31\x39\0\x4c\x42\x42\x30\x5f\x38\0\x4c\x42\x42\x30\
+\x5f\x38\x38\0\x4c\x42\x42\x30\x5f\x35\x38\0\x4c\x42\x42\x30\x5f\x34\x38\0\x4c\
+\x42\x42\x30\x5f\x32\x38\0\x4c\x42\x42\x30\x5f\x31\x38\0\x4c\x42\x42\x30\x5f\
+\x31\x30\x38\0\x4c\x42\x42\x30\x5f\x38\x37\0\x4c\x42\x42\x30\x5f\x37\x37\0\x4c\
+\x42\x42\x30\x5f\x36\x37\0\x4c\x42\x42\x30\x5f\x35\x37\0\x4c\x42\x42\x30\x5f\
+\x34\x37\0\x4c\x42\x42\x30\x5f\x39\x36\0\x4c\x42\x42\x30\x5f\x34\x36\0\x4c\x42\
+\x42\x30\x5f\x31\x30\x36\0\x4c\x42\x42\x30\x5f\x38\x35\0\x4c\x42\x42\x30\x5f\
+\x37\x35\0\x4c\x42\x42\x30\x5f\x36\x35\0\x4c\x42\x42\x30\x5f\x35\x35\0\x4c\x42\
+\x42\x30\x5f\x34\x35\0\x4c\x42\x42\x30\x5f\x33\x35\0\x4c\x42\x42\x30\x5f\x32\
+\x35\0\x4c\x42\x42\x30\x5f\x31\x30\x35\0\x4c\x42\x42\x30\x5f\x39\x34\0\x4c\x42\
\x42\x30\x5f\x35\x34\0\x4c\x42\x42\x30\x5f\x34\x34\0\x4c\x42\x42\x30\x5f\x32\
-\x34\0\x4c\x42\x42\x30\x5f\x31\x30\x34\0\x4c\x42\x42\x30\x5f\x39\x33\0\x4c\x42\
-\x42\x30\x5f\x38\x33\0\x4c\x42\x42\x30\x5f\x35\x33\0\x4c\x42\x42\x30\x5f\x34\
-\x33\0\x4c\x42\x42\x30\x5f\x32\x33\0\x4c\x42\x42\x30\x5f\x31\x30\x33\0\x4c\x42\
-\x42\x30\x5f\x38\x32\0\x4c\x42\x42\x30\x5f\x35\x32\0\x4c\x42\x42\x30\x5f\x31\
-\x30\x32\0\x4c\x42\x42\x30\x5f\x39\x31\0\x4c\x42\x42\x30\x5f\x38\x31\0\x4c\x42\
-\x42\x30\x5f\x37\x31\0\x4c\x42\x42\x30\x5f\x36\x31\0\x4c\x42\x42\x30\x5f\x35\
-\x31\0\x4c\x42\x42\x30\x5f\x34\x31\0\x4c\x42\x42\x30\x5f\x32\x31\0\x4c\x42\x42\
-\x30\x5f\x31\x31\0\x4c\x42\x42\x30\x5f\x31\x31\x31\0\x4c\x42\x42\x30\x5f\x31\
-\x30\x31\0\x4c\x42\x42\x30\x5f\x38\x30\0\x4c\x42\x42\x30\x5f\x36\x30\0\x4c\x42\
-\x42\x30\x5f\x35\x30\0\x4c\x42\x42\x30\x5f\x31\x30\0\x4c\x42\x42\x30\x5f\x31\
-\x31\x30\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
-\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xaa\
-\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xa0\x1a\0\0\0\0\0\0\x71\x02\0\
-\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x1a\0\0\0\x01\0\0\
-\0\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
-\0\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x5a\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\
-\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\0\0\0\xd8\x13\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\
-\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x56\0\0\0\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
-\0\x60\x1a\0\0\0\0\0\0\x30\0\0\0\0\0\0\0\x09\0\0\0\x03\0\0\0\x08\0\0\0\0\0\0\0\
-\x10\0\0\0\0\0\0\0\x20\0\0\0\x01\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x18\
-\x14\0\0\0\0\0\0\x3c\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\
-\0\0\0\x6c\0\0\0\x01\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x54\x14\0\0\0\0\0\
-\0\x07\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x78\0\0\
-\0\x01\0\0\0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x60\x14\0\0\0\0\0\0\x30\0\0\0\0\
-\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x74\0\0\0\x09\0\0\0\0\
-\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x90\x1a\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x09\0\0\0\
-\x07\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\xb2\0\0\0\x02\0\0\0\0\0\0\0\0\0\
-\0\0\0\0\0\0\0\0\0\0\x90\x14\0\0\0\0\0\0\xd0\x05\0\0\0\0\0\0\x01\0\0\0\x39\0\0\
-\0\x08\0\0\0\0\0\0\0\x18\0\0\0\0\0\0\0";
+\x34\0\x4c\x42\x42\x30\x5f\x31\x30\x34\0\x4c\x42\x42\x30\x5f\x33\0\x4c\x42\x42\
+\x30\x5f\x38\x33\0\x4c\x42\x42\x30\x5f\x33\x33\0\x4c\x42\x42\x30\x5f\x31\x30\
+\x33\0\x4c\x42\x42\x30\x5f\x32\0\x4c\x42\x42\x30\x5f\x37\x32\0\x4c\x42\x42\x30\
+\x5f\x35\x32\0\x4c\x42\x42\x30\x5f\x33\x32\0\x4c\x42\x42\x30\x5f\x31\x32\0\x4c\
+\x42\x42\x30\x5f\x31\x30\x32\0\x4c\x42\x42\x30\x5f\x39\x31\0\x4c\x42\x42\x30\
+\x5f\x38\x31\0\x4c\x42\x42\x30\x5f\x37\x31\0\x4c\x42\x42\x30\x5f\x31\x31\0\x4c\
+\x42\x42\x30\x5f\x31\x30\x31\0\x4c\x42\x42\x30\x5f\x39\x30\0\x4c\x42\x42\x30\
+\x5f\x37\x30\0\x4c\x42\x42\x30\x5f\x36\x30\0\x4c\x42\x42\x30\x5f\x31\x30\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x73\0\0\0\x03\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\0\0\xa0\x19\0\0\0\0\0\0\x36\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\x41\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x40\0\
+\0\0\0\0\0\0\x50\x13\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\0\0\x3d\0\0\0\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x80\x19\0\0\0\0\0\0\
+\x10\0\0\0\0\0\0\0\x09\0\0\0\x03\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x07\
+\0\0\0\x01\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x90\x13\0\0\0\0\0\0\x14\0\0\
+\0\0\0\0\0\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x53\0\0\0\x01\0\0\
+\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xa4\x13\0\0\0\0\0\0\x07\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x5f\0\0\0\x01\0\0\0\x02\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\0\xb0\x13\0\0\0\0\0\0\x30\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x5b\0\0\0\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\x90\x19\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x09\0\0\0\x07\0\0\0\x08\0\0\0\0\
+\0\0\0\x10\0\0\0\0\0\0\0\x7b\0\0\0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\xe0\x13\0\0\0\0\0\0\xa0\x05\0\0\0\0\0\0\x01\0\0\0\x39\0\0\0\x08\0\0\0\0\0\0\0\
+\x18\0\0\0\0\0\0\0";
return 0;
err:
diff --git a/tools/ebpf/rss.bpf.c b/tools/ebpf/rss.bpf.c
index e85ec55f9b..1fed1d074c 100644
--- a/tools/ebpf/rss.bpf.c
+++ b/tools/ebpf/rss.bpf.c
@@ -36,19 +36,6 @@
#define INDIRECTION_TABLE_SIZE 128
#define HASH_CALCULATION_BUFFER_SIZE 36
-struct rss_config_t {
- __u8 redirect;
- __u8 populate_hash;
- __u32 hash_types;
- __u16 indirections_len;
- __u16 default_queue;
-} __attribute__((packed));
-
-struct toeplitz_key_data_t {
- __u32 leftmost_32_bits;
- __u8 next_byte[HASH_CALCULATION_BUFFER_SIZE];
-};
-
struct packet_hash_info_t {
__u8 is_ipv4;
__u8 is_ipv6;
@@ -76,28 +63,30 @@ struct packet_hash_info_t {
};
};
-struct bpf_map_def SEC("maps")
-tap_rss_map_configurations = {
- .type = BPF_MAP_TYPE_ARRAY,
- .key_size = sizeof(__u32),
- .value_size = sizeof(struct rss_config_t),
- .max_entries = 1,
+struct toeplitz_key_data_t {
+ __u32 leftmost_32_bits;
+ __u8 next_byte[HASH_CALCULATION_BUFFER_SIZE];
};
-struct bpf_map_def SEC("maps")
-tap_rss_map_toeplitz_key = {
- .type = BPF_MAP_TYPE_ARRAY,
- .key_size = sizeof(__u32),
- .value_size = sizeof(struct toeplitz_key_data_t),
- .max_entries = 1,
-};
+struct rss_config_t {
+ __u8 redirect;
+ __u8 populate_hash;
+ __u32 hash_types;
+ __u16 indirections_len;
+ __u16 default_queue;
+
+ struct toeplitz_key_data_t toeplitz_key;
+
+ __u16 indirections_table[INDIRECTION_TABLE_SIZE];
+} __attribute__((packed));
struct bpf_map_def SEC("maps")
-tap_rss_map_indirection_table = {
+tap_rss_map_configurations = {
.type = BPF_MAP_TYPE_ARRAY,
.key_size = sizeof(__u32),
- .value_size = sizeof(__u16),
- .max_entries = INDIRECTION_TABLE_SIZE,
+ .value_size = sizeof(struct rss_config_t),
+ .max_entries = 1,
+ .map_flags = BPF_F_MMAPABLE,
};
static inline void net_rx_rss_add_chunk(__u8 *rss_input, size_t *bytes_written,
@@ -381,7 +370,7 @@ error:
}
static inline __u32 calculate_rss_hash(struct __sk_buff *skb,
- struct rss_config_t *config, struct toeplitz_key_data_t *toe)
+ struct rss_config_t *config)
{
__u8 rss_input[HASH_CALCULATION_BUFFER_SIZE] = {};
size_t bytes_written = 0;
@@ -525,7 +514,8 @@ static inline __u32 calculate_rss_hash(struct __sk_buff *skb,
}
if (bytes_written) {
- net_toeplitz_add(&result, rss_input, bytes_written, toe);
+ net_toeplitz_add(&result, rss_input,
+ bytes_written, &config->toeplitz_key);
}
return result;
@@ -536,29 +526,24 @@ int tun_rss_steering_prog(struct __sk_buff *skb)
{
struct rss_config_t *config;
- struct toeplitz_key_data_t *toe;
__u32 key = 0;
__u32 hash = 0;
config = bpf_map_lookup_elem(&tap_rss_map_configurations, &key);
- toe = bpf_map_lookup_elem(&tap_rss_map_toeplitz_key, &key);
- if (config && toe) {
+ if (config) {
if (!config->redirect) {
return config->default_queue;
}
- hash = calculate_rss_hash(skb, config, toe);
+ hash = calculate_rss_hash(skb, config);
if (hash) {
__u32 table_idx = hash % config->indirections_len;
- __u16 *queue = 0;
-
- queue = bpf_map_lookup_elem(&tap_rss_map_indirection_table,
- &table_idx);
- if (queue) {
- return *queue;
+ if (table_idx < INDIRECTION_TABLE_SIZE
+ && table_idx < config->indirections_len) {
+ return config->indirections_table[table_idx];
}
}
--
2.31.1
^ permalink raw reply related [flat|nested] 60+ messages in thread
* Re: [PATCH 1/5] ebpf: Added eBPF initialization by fds and map update.
2021-07-13 15:37 ` [PATCH 1/5] ebpf: Added eBPF initialization by fds and map update Andrew Melnychenko
@ 2021-08-20 3:34 ` Jason Wang
2021-08-25 18:13 ` Andrew Melnichenko
0 siblings, 1 reply; 60+ messages in thread
From: Jason Wang @ 2021-08-20 3:34 UTC (permalink / raw)
To: Andrew Melnychenko, mst, yuri.benditovich, armbru, eblake, berrange
Cc: yan, qemu-devel
在 2021/7/13 下午11:37, Andrew Melnychenko 写道:
> -static bool ebpf_rss_set_indirections_table(struct EBPFRSSContext *ctx,
> - uint16_t *indirections_table,
> - size_t len)
> -{
> - uint32_t i = 0;
> + ctx->program_fd = program_fd;
> + ctx->map_configuration = config_fd;
>
> - if (!ebpf_rss_is_loaded(ctx) || indirections_table == NULL ||
> - len > VIRTIO_NET_RSS_MAX_TABLE_LEN) {
> + ctx->mmap_configuration = mmap(NULL, qemu_real_host_page_size,
> + PROT_READ | PROT_WRITE, MAP_SHARED,
> + ctx->map_configuration, 0);
> + if (ctx->mmap_configuration == MAP_FAILED) {
> + trace_ebpf_error("eBPF RSS", "can not mmap eBPF configuration array");
> return false;
> }
>
> - for (; i < len; ++i) {
> - if (bpf_map_update_elem(ctx->map_indirections_table, &i,
> - indirections_table + i, 0) < 0) {
> - return false;
> - }
> - }
> - return true;
> -}
> -
It looks to me you want to drop syscall path for map updating
completely. I think we'd better keep that for backward compatibility?
Thanks
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 1/5] ebpf: Added eBPF initialization by fds and map update.
2021-08-20 3:34 ` Jason Wang
@ 2021-08-25 18:13 ` Andrew Melnichenko
0 siblings, 0 replies; 60+ messages in thread
From: Andrew Melnichenko @ 2021-08-25 18:13 UTC (permalink / raw)
To: Jason Wang
Cc: Daniel P. Berrangé,
Michael S. Tsirkin, qemu-devel, Markus Armbruster,
Yuri Benditovich, Yan Vugenfirer, Eric Blake
[-- Attachment #1: Type: text/plain, Size: 1618 bytes --]
Hi,
Yes - to make the bpf() syscall capabilities are required, which libvirt
have no intentions to give.
Does it make any sense to leave syscall if mmap works?
On Fri, Aug 20, 2021 at 6:34 AM Jason Wang <jasowang@redhat.com> wrote:
>
> 在 2021/7/13 下午11:37, Andrew Melnychenko 写道:
> > -static bool ebpf_rss_set_indirections_table(struct EBPFRSSContext *ctx,
> > - uint16_t
> *indirections_table,
> > - size_t len)
> > -{
> > - uint32_t i = 0;
> > + ctx->program_fd = program_fd;
> > + ctx->map_configuration = config_fd;
> >
> > - if (!ebpf_rss_is_loaded(ctx) || indirections_table == NULL ||
> > - len > VIRTIO_NET_RSS_MAX_TABLE_LEN) {
> > + ctx->mmap_configuration = mmap(NULL, qemu_real_host_page_size,
> > + PROT_READ | PROT_WRITE, MAP_SHARED,
> > + ctx->map_configuration, 0);
> > + if (ctx->mmap_configuration == MAP_FAILED) {
> > + trace_ebpf_error("eBPF RSS", "can not mmap eBPF configuration
> array");
> > return false;
> > }
> >
> > - for (; i < len; ++i) {
> > - if (bpf_map_update_elem(ctx->map_indirections_table, &i,
> > - indirections_table + i, 0) < 0) {
> > - return false;
> > - }
> > - }
> > - return true;
> > -}
> > -
>
>
> It looks to me you want to drop syscall path for map updating
> completely. I think we'd better keep that for backward compatibility?
>
> Thanks
>
>
[-- Attachment #2: Type: text/html, Size: 2290 bytes --]
^ permalink raw reply [flat|nested] 60+ messages in thread
* [PATCH 2/5] virtio-net: Added property to load eBPF RSS with fds.
2021-07-13 15:37 [PATCH 0/5] ebpf: Added ebpf helper for libvirtd Andrew Melnychenko
2021-07-13 15:37 ` [PATCH 1/5] ebpf: Added eBPF initialization by fds and map update Andrew Melnychenko
@ 2021-07-13 15:37 ` Andrew Melnychenko
2021-08-20 3:36 ` Jason Wang
2021-07-13 15:37 ` [PATCH 3/5] qmp: Added the helper stamp check Andrew Melnychenko
` (4 subsequent siblings)
6 siblings, 1 reply; 60+ messages in thread
From: Andrew Melnychenko @ 2021-07-13 15:37 UTC (permalink / raw)
To: mst, yuri.benditovich, jasowang, armbru, eblake, berrange; +Cc: yan, qemu-devel
eBPF RSS program and maps now may be passed during initialization.
Initially was implemented for libvirt to launch qemu without permissions.
Signed-off-by: Andrew Melnychenko <andrew@daynix.com>
---
hw/net/virtio-net.c | 77 ++++++++++++++++++++++++++++++++--
include/hw/virtio/virtio-net.h | 1 +
2 files changed, 74 insertions(+), 4 deletions(-)
diff --git a/hw/net/virtio-net.c b/hw/net/virtio-net.c
index bd7958b9f0..0602b1772e 100644
--- a/hw/net/virtio-net.c
+++ b/hw/net/virtio-net.c
@@ -41,6 +41,7 @@
#include "sysemu/sysemu.h"
#include "trace.h"
#include "monitor/qdev.h"
+#include "monitor/monitor.h"
#include "hw/pci/pci.h"
#include "net_rx_pkt.h"
#include "hw/virtio/vhost.h"
@@ -1223,14 +1224,81 @@ static void virtio_net_detach_epbf_rss(VirtIONet *n)
virtio_net_attach_ebpf_to_backend(n->nic, -1);
}
-static bool virtio_net_load_ebpf(VirtIONet *n)
+static int virtio_net_get_ebpf_rss_fds(char *str, char *fds[], int nfds)
{
- if (!virtio_net_attach_ebpf_to_backend(n->nic, -1)) {
- /* backend does't support steering ebpf */
+ char *ptr = str;
+ char *cur = NULL;
+ size_t len = strlen(str);
+ int i = 0;
+
+ for (; i < nfds && ptr < str + len;) {
+ cur = strchr(ptr, ':');
+
+ if (cur == NULL) {
+ fds[i] = g_strdup(ptr);
+ } else {
+ fds[i] = g_strndup(ptr, cur - ptr);
+ }
+
+ i++;
+ if (cur == NULL) {
+ break;
+ } else {
+ ptr = cur + 1;
+ }
+ }
+
+ return i;
+}
+
+static bool virtio_net_load_ebpf_fds(VirtIONet *n)
+{
+ char *fds_strs[EBPF_RSS_MAX_FDS];
+ int fds[EBPF_RSS_MAX_FDS];
+ int nfds;
+ int ret = false;
+ Error *errp;
+ int i = 0;
+
+ if (n == NULL || !n->ebpf_rss_fds) {
return false;
}
- return ebpf_rss_load(&n->ebpf_rss);
+ nfds = virtio_net_get_ebpf_rss_fds(n->ebpf_rss_fds,
+ fds_strs, EBPF_RSS_MAX_FDS);
+ for (i = 0; i < nfds; i++) {
+ fds[i] = monitor_fd_param(monitor_cur(), fds_strs[i], &errp);
+ }
+
+ if (nfds == EBPF_RSS_MAX_FDS) {
+ ret = ebpf_rss_load_fds(&n->ebpf_rss, fds[0], fds[1], fds[2], fds[3]);
+ }
+
+ if (!ret) {
+ for (i = 0; i < nfds; i++) {
+ close(fds[i]);
+ }
+ }
+
+ for (i = 0; i < nfds; i++) {
+ g_free(fds_strs[i]);
+ }
+
+ return ret;
+}
+
+static bool virtio_net_load_ebpf(VirtIONet *n)
+{
+ bool ret = true;
+
+ if (virtio_net_attach_ebpf_to_backend(n->nic, -1)) {
+ if (!(n->ebpf_rss_fds
+ && virtio_net_load_ebpf_fds(n))) {
+ ret = ebpf_rss_load(&n->ebpf_rss);
+ }
+ }
+
+ return ret;
}
static void virtio_net_unload_ebpf(VirtIONet *n)
@@ -3605,6 +3673,7 @@ static Property virtio_net_properties[] = {
VIRTIO_NET_F_RSS, false),
DEFINE_PROP_BIT64("hash", VirtIONet, host_features,
VIRTIO_NET_F_HASH_REPORT, false),
+ DEFINE_PROP_STRING("ebpf_rss_fds", VirtIONet, ebpf_rss_fds),
DEFINE_PROP_BIT64("guest_rsc_ext", VirtIONet, host_features,
VIRTIO_NET_F_RSC_EXT, false),
DEFINE_PROP_UINT32("rsc_interval", VirtIONet, rsc_timeout,
diff --git a/include/hw/virtio/virtio-net.h b/include/hw/virtio/virtio-net.h
index 824a69c23f..993f2f3036 100644
--- a/include/hw/virtio/virtio-net.h
+++ b/include/hw/virtio/virtio-net.h
@@ -213,6 +213,7 @@ struct VirtIONet {
VirtioNetRssData rss_data;
struct NetRxPkt *rx_pkt;
struct EBPFRSSContext ebpf_rss;
+ char *ebpf_rss_fds;
};
void virtio_net_set_netclient_name(VirtIONet *n, const char *name,
--
2.31.1
^ permalink raw reply related [flat|nested] 60+ messages in thread
* Re: [PATCH 2/5] virtio-net: Added property to load eBPF RSS with fds.
2021-07-13 15:37 ` [PATCH 2/5] virtio-net: Added property to load eBPF RSS with fds Andrew Melnychenko
@ 2021-08-20 3:36 ` Jason Wang
2021-08-25 18:18 ` Andrew Melnichenko
0 siblings, 1 reply; 60+ messages in thread
From: Jason Wang @ 2021-08-20 3:36 UTC (permalink / raw)
To: Andrew Melnychenko, mst, yuri.benditovich, armbru, eblake, berrange
Cc: yan, qemu-devel
在 2021/7/13 下午11:37, Andrew Melnychenko 写道:
> eBPF RSS program and maps now may be passed during initialization.
> Initially was implemented for libvirt to launch qemu without permissions.
>
> Signed-off-by: Andrew Melnychenko <andrew@daynix.com>
> ---
> hw/net/virtio-net.c | 77 ++++++++++++++++++++++++++++++++--
> include/hw/virtio/virtio-net.h | 1 +
> 2 files changed, 74 insertions(+), 4 deletions(-)
>
> diff --git a/hw/net/virtio-net.c b/hw/net/virtio-net.c
> index bd7958b9f0..0602b1772e 100644
> --- a/hw/net/virtio-net.c
> +++ b/hw/net/virtio-net.c
> @@ -41,6 +41,7 @@
> #include "sysemu/sysemu.h"
> #include "trace.h"
> #include "monitor/qdev.h"
> +#include "monitor/monitor.h"
> #include "hw/pci/pci.h"
> #include "net_rx_pkt.h"
> #include "hw/virtio/vhost.h"
> @@ -1223,14 +1224,81 @@ static void virtio_net_detach_epbf_rss(VirtIONet *n)
> virtio_net_attach_ebpf_to_backend(n->nic, -1);
> }
>
> -static bool virtio_net_load_ebpf(VirtIONet *n)
> +static int virtio_net_get_ebpf_rss_fds(char *str, char *fds[], int nfds)
> {
> - if (!virtio_net_attach_ebpf_to_backend(n->nic, -1)) {
> - /* backend does't support steering ebpf */
> + char *ptr = str;
> + char *cur = NULL;
> + size_t len = strlen(str);
> + int i = 0;
> +
> + for (; i < nfds && ptr < str + len;) {
> + cur = strchr(ptr, ':');
> +
> + if (cur == NULL) {
> + fds[i] = g_strdup(ptr);
> + } else {
> + fds[i] = g_strndup(ptr, cur - ptr);
> + }
> +
> + i++;
> + if (cur == NULL) {
> + break;
> + } else {
> + ptr = cur + 1;
> + }
> + }
> +
> + return i;
> +}
> +
> +static bool virtio_net_load_ebpf_fds(VirtIONet *n)
> +{
> + char *fds_strs[EBPF_RSS_MAX_FDS];
> + int fds[EBPF_RSS_MAX_FDS];
> + int nfds;
> + int ret = false;
> + Error *errp;
> + int i = 0;
> +
> + if (n == NULL || !n->ebpf_rss_fds) {
> return false;
> }
>
> - return ebpf_rss_load(&n->ebpf_rss);
> + nfds = virtio_net_get_ebpf_rss_fds(n->ebpf_rss_fds,
> + fds_strs, EBPF_RSS_MAX_FDS);
> + for (i = 0; i < nfds; i++) {
> + fds[i] = monitor_fd_param(monitor_cur(), fds_strs[i], &errp);
> + }
> +
> + if (nfds == EBPF_RSS_MAX_FDS) {
> + ret = ebpf_rss_load_fds(&n->ebpf_rss, fds[0], fds[1], fds[2], fds[3]);
> + }
> +
> + if (!ret) {
> + for (i = 0; i < nfds; i++) {
> + close(fds[i]);
> + }
> + }
> +
> + for (i = 0; i < nfds; i++) {
> + g_free(fds_strs[i]);
> + }
> +
> + return ret;
> +}
> +
> +static bool virtio_net_load_ebpf(VirtIONet *n)
> +{
> + bool ret = true;
> +
> + if (virtio_net_attach_ebpf_to_backend(n->nic, -1)) {
> + if (!(n->ebpf_rss_fds
> + && virtio_net_load_ebpf_fds(n))) {
> + ret = ebpf_rss_load(&n->ebpf_rss);
> + }
> + }
> +
> + return ret;
> }
>
> static void virtio_net_unload_ebpf(VirtIONet *n)
> @@ -3605,6 +3673,7 @@ static Property virtio_net_properties[] = {
> VIRTIO_NET_F_RSS, false),
> DEFINE_PROP_BIT64("hash", VirtIONet, host_features,
> VIRTIO_NET_F_HASH_REPORT, false),
> + DEFINE_PROP_STRING("ebpf_rss_fds", VirtIONet, ebpf_rss_fds),
> DEFINE_PROP_BIT64("guest_rsc_ext", VirtIONet, host_features,
> VIRTIO_NET_F_RSC_EXT, false),
> DEFINE_PROP_UINT32("rsc_interval", VirtIONet, rsc_timeout,
> diff --git a/include/hw/virtio/virtio-net.h b/include/hw/virtio/virtio-net.h
> index 824a69c23f..993f2f3036 100644
> --- a/include/hw/virtio/virtio-net.h
> +++ b/include/hw/virtio/virtio-net.h
> @@ -213,6 +213,7 @@ struct VirtIONet {
> VirtioNetRssData rss_data;
> struct NetRxPkt *rx_pkt;
> struct EBPFRSSContext ebpf_rss;
> + char *ebpf_rss_fds;
I wonder if it's better to use separated properties instead of implying
an order here?
E.g "program_fd", "key_map_fd", "indirection_table_map_fd" etc.
Thanks
> };
>
> void virtio_net_set_netclient_name(VirtIONet *n, const char *name,
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 2/5] virtio-net: Added property to load eBPF RSS with fds.
2021-08-20 3:36 ` Jason Wang
@ 2021-08-25 18:18 ` Andrew Melnichenko
0 siblings, 0 replies; 60+ messages in thread
From: Andrew Melnichenko @ 2021-08-25 18:18 UTC (permalink / raw)
To: Jason Wang
Cc: Daniel P. Berrangé,
Michael S. Tsirkin, qemu-devel, Markus Armbruster,
Yuri Benditovich, Yan Vugenfirer, Eric Blake
[-- Attachment #1: Type: text/plain, Size: 5082 bytes --]
Hi,
> I wonder if it's better to use separated properties instead of implying
> an order here?
>
Not really, technically RSS BPF interface may be changed (it's already
changed after RFC).
And libvirt should use something unified, so it's better to use fd array.
If any changes occur - those changes will be applied only for qemu and the
helper.
Also, now all maps are combined in one configuration map.
On Fri, Aug 20, 2021 at 6:36 AM Jason Wang <jasowang@redhat.com> wrote:
>
> 在 2021/7/13 下午11:37, Andrew Melnychenko 写道:
> > eBPF RSS program and maps now may be passed during initialization.
> > Initially was implemented for libvirt to launch qemu without permissions.
> >
> > Signed-off-by: Andrew Melnychenko <andrew@daynix.com>
> > ---
> > hw/net/virtio-net.c | 77 ++++++++++++++++++++++++++++++++--
> > include/hw/virtio/virtio-net.h | 1 +
> > 2 files changed, 74 insertions(+), 4 deletions(-)
> >
> > diff --git a/hw/net/virtio-net.c b/hw/net/virtio-net.c
> > index bd7958b9f0..0602b1772e 100644
> > --- a/hw/net/virtio-net.c
> > +++ b/hw/net/virtio-net.c
> > @@ -41,6 +41,7 @@
> > #include "sysemu/sysemu.h"
> > #include "trace.h"
> > #include "monitor/qdev.h"
> > +#include "monitor/monitor.h"
> > #include "hw/pci/pci.h"
> > #include "net_rx_pkt.h"
> > #include "hw/virtio/vhost.h"
> > @@ -1223,14 +1224,81 @@ static void virtio_net_detach_epbf_rss(VirtIONet
> *n)
> > virtio_net_attach_ebpf_to_backend(n->nic, -1);
> > }
> >
> > -static bool virtio_net_load_ebpf(VirtIONet *n)
> > +static int virtio_net_get_ebpf_rss_fds(char *str, char *fds[], int nfds)
> > {
> > - if (!virtio_net_attach_ebpf_to_backend(n->nic, -1)) {
> > - /* backend does't support steering ebpf */
> > + char *ptr = str;
> > + char *cur = NULL;
> > + size_t len = strlen(str);
> > + int i = 0;
> > +
> > + for (; i < nfds && ptr < str + len;) {
> > + cur = strchr(ptr, ':');
> > +
> > + if (cur == NULL) {
> > + fds[i] = g_strdup(ptr);
> > + } else {
> > + fds[i] = g_strndup(ptr, cur - ptr);
> > + }
> > +
> > + i++;
> > + if (cur == NULL) {
> > + break;
> > + } else {
> > + ptr = cur + 1;
> > + }
> > + }
> > +
> > + return i;
> > +}
> > +
> > +static bool virtio_net_load_ebpf_fds(VirtIONet *n)
> > +{
> > + char *fds_strs[EBPF_RSS_MAX_FDS];
> > + int fds[EBPF_RSS_MAX_FDS];
> > + int nfds;
> > + int ret = false;
> > + Error *errp;
> > + int i = 0;
> > +
> > + if (n == NULL || !n->ebpf_rss_fds) {
> > return false;
> > }
> >
> > - return ebpf_rss_load(&n->ebpf_rss);
> > + nfds = virtio_net_get_ebpf_rss_fds(n->ebpf_rss_fds,
> > + fds_strs, EBPF_RSS_MAX_FDS);
> > + for (i = 0; i < nfds; i++) {
> > + fds[i] = monitor_fd_param(monitor_cur(), fds_strs[i], &errp);
> > + }
> > +
> > + if (nfds == EBPF_RSS_MAX_FDS) {
> > + ret = ebpf_rss_load_fds(&n->ebpf_rss, fds[0], fds[1], fds[2],
> fds[3]);
> > + }
> > +
> > + if (!ret) {
> > + for (i = 0; i < nfds; i++) {
> > + close(fds[i]);
> > + }
> > + }
> > +
> > + for (i = 0; i < nfds; i++) {
> > + g_free(fds_strs[i]);
> > + }
> > +
> > + return ret;
> > +}
> > +
> > +static bool virtio_net_load_ebpf(VirtIONet *n)
> > +{
> > + bool ret = true;
> > +
> > + if (virtio_net_attach_ebpf_to_backend(n->nic, -1)) {
> > + if (!(n->ebpf_rss_fds
> > + && virtio_net_load_ebpf_fds(n))) {
> > + ret = ebpf_rss_load(&n->ebpf_rss);
> > + }
> > + }
> > +
> > + return ret;
> > }
> >
> > static void virtio_net_unload_ebpf(VirtIONet *n)
> > @@ -3605,6 +3673,7 @@ static Property virtio_net_properties[] = {
> > VIRTIO_NET_F_RSS, false),
> > DEFINE_PROP_BIT64("hash", VirtIONet, host_features,
> > VIRTIO_NET_F_HASH_REPORT, false),
> > + DEFINE_PROP_STRING("ebpf_rss_fds", VirtIONet, ebpf_rss_fds),
> > DEFINE_PROP_BIT64("guest_rsc_ext", VirtIONet, host_features,
> > VIRTIO_NET_F_RSC_EXT, false),
> > DEFINE_PROP_UINT32("rsc_interval", VirtIONet, rsc_timeout,
> > diff --git a/include/hw/virtio/virtio-net.h
> b/include/hw/virtio/virtio-net.h
> > index 824a69c23f..993f2f3036 100644
> > --- a/include/hw/virtio/virtio-net.h
> > +++ b/include/hw/virtio/virtio-net.h
> > @@ -213,6 +213,7 @@ struct VirtIONet {
> > VirtioNetRssData rss_data;
> > struct NetRxPkt *rx_pkt;
> > struct EBPFRSSContext ebpf_rss;
> > + char *ebpf_rss_fds;
>
>
> I wonder if it's better to use separated properties instead of implying
> an order here?
>
> E.g "program_fd", "key_map_fd", "indirection_table_map_fd" etc.
>
> Thanks
>
>
> > };
> >
> > void virtio_net_set_netclient_name(VirtIONet *n, const char *name,
>
>
[-- Attachment #2: Type: text/html, Size: 6829 bytes --]
^ permalink raw reply [flat|nested] 60+ messages in thread
* [PATCH 3/5] qmp: Added the helper stamp check.
2021-07-13 15:37 [PATCH 0/5] ebpf: Added ebpf helper for libvirtd Andrew Melnychenko
2021-07-13 15:37 ` [PATCH 1/5] ebpf: Added eBPF initialization by fds and map update Andrew Melnychenko
2021-07-13 15:37 ` [PATCH 2/5] virtio-net: Added property to load eBPF RSS with fds Andrew Melnychenko
@ 2021-07-13 15:37 ` Andrew Melnychenko
2021-07-13 15:37 ` [PATCH 4/5] ebpf_rss_helper: Added helper for eBPF RSS Andrew Melnychenko
` (3 subsequent siblings)
6 siblings, 0 replies; 60+ messages in thread
From: Andrew Melnychenko @ 2021-07-13 15:37 UTC (permalink / raw)
To: mst, yuri.benditovich, jasowang, armbru, eblake, berrange; +Cc: yan, qemu-devel
Added function to check the stamp in the helper.
eBPF helper should have a special symbol that generates during build.
QEMU checks the helper and determinates that it fits, so the helper
will produce proper output.
Signed-off-by: Andrew Melnychenko <andrew@daynix.com>
---
meson.build | 10 +
monitor/meson.build | 1 +
monitor/qemu-helper-stamp-utils.c | 297 ++++++++++++++++++++++++++++++
monitor/qemu-helper-stamp-utils.h | 24 +++
4 files changed, 332 insertions(+)
create mode 100644 monitor/qemu-helper-stamp-utils.c
create mode 100644 monitor/qemu-helper-stamp-utils.h
diff --git a/meson.build b/meson.build
index 626cf932c1..257e51d91b 100644
--- a/meson.build
+++ b/meson.build
@@ -1757,6 +1757,16 @@ foreach d : hx_headers
endforeach
genh += hxdep
+helper_stamp = custom_target(
+ 'qemu-helper-stamp.h',
+ output : 'qemu-helper-stamp.h',
+ input : 'ebpf/rss.bpf.skeleton.h',
+ command : [python, '-c', 'import hashlib; print(\'#define QEMU_HELPER_STAMP qemuHelperStamp_{}\'.format(hashlib.sha1(open(\'@INPUT@\', \'rb\').read()).hexdigest()))'],
+ capture: true,
+)
+
+genh += helper_stamp
+
###################
# Collect sources #
###################
diff --git a/monitor/meson.build b/monitor/meson.build
index 6d00985ace..2b6b39549b 100644
--- a/monitor/meson.build
+++ b/monitor/meson.build
@@ -5,5 +5,6 @@ softmmu_ss.add(files(
'hmp.c',
))
softmmu_ss.add([spice_headers, files('qmp-cmds.c')])
+softmmu_ss.add(files('qemu-helper-stamp-utils.c'))
specific_ss.add(when: 'CONFIG_SOFTMMU', if_true: [files('misc.c'), spice])
diff --git a/monitor/qemu-helper-stamp-utils.c b/monitor/qemu-helper-stamp-utils.c
new file mode 100644
index 0000000000..d34c3b94c5
--- /dev/null
+++ b/monitor/qemu-helper-stamp-utils.c
@@ -0,0 +1,297 @@
+/*
+ * QEMU helper stamp check utils.
+ *
+ * Developed by Daynix Computing LTD (http://www.daynix.com)
+ *
+ * Authors:
+ * Andrew Melnychenko <andrew@daynix.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2. See
+ * the COPYING file in the top-level directory.
+ *
+ * Description: This file mostly implements helper stamp checking.
+ * The stamp is implemented in a similar way as in qemu modules.
+ * The helper should contain a specific symbol.
+ * Not in a similar way is symbol checking - here we parse
+ * the ELF file. For now(10.07.2021), only eBPF helper contains
+ * the stamp, and the stamp is generated from
+ * sha1 ebpf/rss.bpf.skeleton.h (see meson.build).
+ */
+
+#include "qemu/osdep.h"
+#include "elf.h"
+#include "qemu-helper-stamp-utils.h"
+
+#include <glib/gstdio.h>
+
+#ifdef CONFIG_LINUX
+
+static void *file_allocate_and_read(int fd, off_t off, size_t size)
+{
+ void *data;
+ int err;
+
+ if (fd < 0) {
+ return NULL;
+ }
+
+ err = lseek(fd, off, SEEK_SET);
+ if (err < 0) {
+ return NULL;
+ }
+
+ data = g_new0(char, size);
+ if (data == NULL) {
+ return NULL;
+ }
+
+ err = read(fd, data, size);
+ if (err < 0) {
+ g_free(data);
+ return NULL;
+ }
+
+ return data;
+}
+
+static Elf64_Shdr *elf64_get_section_table(int fd, Elf64_Ehdr *elf_header)
+{
+ if (elf_header == NULL) {
+ return NULL;
+ }
+ return (Elf64_Shdr *)file_allocate_and_read(fd, elf_header->e_shoff,
+ elf_header->e_shnum * elf_header->e_shentsize);
+}
+
+static Elf32_Shdr *elf32_get_section_table(int fd, Elf32_Ehdr *elf_header)
+{
+ if (elf_header == NULL) {
+ return NULL;
+ }
+ return (Elf32_Shdr *)file_allocate_and_read(fd, elf_header->e_shoff,
+ elf_header->e_shnum * elf_header->e_shentsize);
+}
+
+static void *elf64_get_section_data(int fd, const Elf64_Shdr* section_header)
+{
+ if (fd < 0 || section_header == NULL) {
+ return NULL;
+ }
+ return file_allocate_and_read(fd, section_header->sh_offset,
+ section_header->sh_size);
+}
+
+static void *elf32_get_section_data(int fd, const Elf32_Shdr* section_header)
+{
+ if (fd < 0 || section_header == NULL) {
+ return NULL;
+ }
+ return file_allocate_and_read(fd, section_header->sh_offset,
+ section_header->sh_size);
+}
+
+static bool elf64_check_symbol_in_symbol_table(int fd,
+ Elf64_Shdr *section_table,
+ Elf64_Shdr *symbol_section,
+ const char *symbol)
+{
+ Elf64_Sym *symbol_table;
+ char *string_table;
+ uint32_t i;
+ bool ret = false;
+
+ symbol_table = (Elf64_Sym *) elf64_get_section_data(fd, symbol_section);
+ if (symbol_table == NULL) {
+ return false;
+ }
+
+ string_table = (char *) elf64_get_section_data(
+ fd, section_table + symbol_section->sh_link);
+ if (string_table == NULL) {
+ g_free(symbol_table);
+ return false;
+ }
+
+ for (i = 0; i < (symbol_section->sh_size / sizeof(Elf64_Sym)); ++i) {
+ if (strncmp((string_table + symbol_table[i].st_name),
+ symbol, strlen(symbol)) == 0)
+ {
+ ret = true;
+ break;
+ }
+ }
+
+ g_free(string_table);
+ g_free(symbol_table);
+ return ret;
+}
+
+static bool elf32_check_symbol_in_symbol_table(int fd,
+ Elf32_Shdr *section_table,
+ Elf32_Shdr *symbol_section,
+ const char *symbol)
+{
+ Elf32_Sym *symbol_table;
+ char *string_table;
+ uint32_t i;
+ bool ret = false;
+
+ symbol_table = (Elf32_Sym *) elf32_get_section_data(fd, symbol_section);
+ if (symbol_table == NULL) {
+ return false;
+ }
+
+ string_table = (char *) elf32_get_section_data(fd,
+ section_table + symbol_section->sh_link);
+ if (string_table == NULL) {
+ g_free(symbol_table);
+ return false;
+ }
+
+ for (i = 0; i < (symbol_section->sh_size / sizeof(Elf32_Sym)); ++i) {
+ if (strncmp((string_table + symbol_table[i].st_name),
+ symbol, strlen(symbol)) == 0)
+ {
+ ret = true;
+ break;
+ }
+ }
+
+ g_free(string_table);
+ g_free(symbol_table);
+ return ret;
+}
+
+static bool elf64_check_stamp(int fd, Elf64_Ehdr *elf_header, const char *stamp)
+{
+ Elf64_Shdr *section_table;
+ size_t i;
+ bool ret = false;
+
+ section_table = elf64_get_section_table(fd, elf_header);
+ if (section_table == NULL) {
+ return false;
+ }
+
+ for (i = 0; i < elf_header->e_shnum; ++i) {
+ if ((section_table[i].sh_type == SHT_SYMTAB)
+ || (section_table[i].sh_type == SHT_DYNSYM)) {
+ if (elf64_check_symbol_in_symbol_table(fd, section_table,
+ section_table + i, stamp)) {
+ ret = true;
+ break;
+ }
+ }
+ }
+
+ g_free(section_table);
+ return ret;
+}
+
+static bool elf32_check_stamp(int fd, Elf32_Ehdr *elf_header, const char *stamp)
+{
+ Elf32_Shdr *section_table;
+ size_t i;
+ bool ret = false;
+
+ section_table = elf32_get_section_table(fd, elf_header);
+ if (section_table == NULL) {
+ return false;
+ }
+
+ for (i = 0; i < elf_header->e_shnum; ++i) {
+ if ((section_table[i].sh_type == SHT_SYMTAB)
+ || (section_table[i].sh_type == SHT_DYNSYM)) {
+ if (elf32_check_symbol_in_symbol_table(fd, section_table,
+ section_table + i, stamp)) {
+ ret = true;
+ break;
+ }
+ }
+ }
+
+ g_free(section_table);
+ return ret;
+}
+
+bool qemu_check_helper_stamp(const char *path, const char *stamp)
+{
+ int fd;
+ bool ret = false;
+ Elf64_Ehdr *elf_header;
+
+ fd = open(path, O_RDONLY | O_SYNC);
+ if (fd < 0) {
+ return false;
+ }
+
+ elf_header = (Elf64_Ehdr *)file_allocate_and_read(
+ fd, 0, sizeof(Elf64_Ehdr));
+ if (elf_header == NULL) {
+ goto error;
+ }
+
+ if (strncmp((char *)elf_header->e_ident, ELFMAG, SELFMAG)) {
+ g_free(elf_header);
+ goto error;
+ }
+
+ if (elf_header->e_ident[EI_CLASS] == ELFCLASS64) {
+ ret = elf64_check_stamp(fd, elf_header, stamp);
+ } else if (elf_header->e_ident[EI_CLASS] == ELFCLASS32) {
+ ret = elf32_check_stamp(fd, (Elf32_Ehdr *)elf_header, stamp);
+ }
+
+ g_free(elf_header);
+error:
+ close(fd);
+ return ret;
+}
+
+#else
+
+bool qemu_check_helper_stamp(const char *path, const char *stamp)
+{
+ return false;
+}
+
+#endif
+
+char *qemu_find_helper(const char *name, bool check_stamp)
+{
+ char *qemu_exec = NULL;
+ char *qemu_dir = NULL;
+ char *helper = NULL;
+
+ if (name == NULL) {
+ return NULL;
+ }
+
+ helper = g_build_filename(CONFIG_QEMU_HELPERDIR, name, NULL);
+ if (g_access(helper, F_OK) == 0
+ && (!check_stamp
+ || qemu_check_helper_stamp(helper, QEMU_HELPER_STAMP_STR))) {
+ return helper;
+ }
+ g_free(helper);
+
+#ifdef CONFIG_LINUX
+ qemu_exec = g_file_read_link("/proc/self/exe", NULL);
+#else
+ qemu_exec = NULL;
+#endif
+ if (qemu_exec != NULL) {
+ qemu_dir = g_path_get_dirname(qemu_exec);
+ g_free(qemu_exec);
+ helper = g_build_filename(qemu_dir, name, NULL);
+ g_free(qemu_dir);
+ if (g_access(helper, F_OK) == 0
+ && (!check_stamp
+ || qemu_check_helper_stamp(helper, QEMU_HELPER_STAMP_STR))) {
+ return helper;
+ }
+ g_free(helper);
+ }
+
+ return NULL;
+}
diff --git a/monitor/qemu-helper-stamp-utils.h b/monitor/qemu-helper-stamp-utils.h
new file mode 100644
index 0000000000..e64cf96aa6
--- /dev/null
+++ b/monitor/qemu-helper-stamp-utils.h
@@ -0,0 +1,24 @@
+/*
+ * QEMU helper stamp check utils.
+ *
+ * Developed by Daynix Computing LTD (http://www.daynix.com)
+ *
+ * Authors:
+ * Andrew Melnychenko <andrew@daynix.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2. See
+ * the COPYING file in the top-level directory.
+ */
+
+#ifndef QEMU_QEMU_HELPER_STAMP_UTILS_H
+#define QEMU_QEMU_HELPER_STAMP_UTILS_H
+
+#include "qemu-helper-stamp.h" /* generated stamp per build */
+
+#define QEMU_HELPER_STAMP_STR stringify(QEMU_HELPER_STAMP)
+
+bool qemu_check_helper_stamp(const char *path, const char *stamp);
+
+char *qemu_find_helper(const char *name, bool check_stamp);
+
+#endif /* QEMU_QEMU_HELPER_STAMP_UTILS_H */
--
2.31.1
^ permalink raw reply related [flat|nested] 60+ messages in thread
* [PATCH 4/5] ebpf_rss_helper: Added helper for eBPF RSS.
2021-07-13 15:37 [PATCH 0/5] ebpf: Added ebpf helper for libvirtd Andrew Melnychenko
` (2 preceding siblings ...)
2021-07-13 15:37 ` [PATCH 3/5] qmp: Added the helper stamp check Andrew Melnychenko
@ 2021-07-13 15:37 ` Andrew Melnychenko
2021-08-20 3:40 ` Jason Wang
2021-07-13 15:37 ` [PATCH 5/5] qmp: Added qemu-ebpf-rss-path command Andrew Melnychenko
` (2 subsequent siblings)
6 siblings, 1 reply; 60+ messages in thread
From: Andrew Melnychenko @ 2021-07-13 15:37 UTC (permalink / raw)
To: mst, yuri.benditovich, jasowang, armbru, eblake, berrange; +Cc: yan, qemu-devel
Helper program. Loads eBPF RSS program and maps and passes them through unix socket.
Libvirt may launch this helper and pass eBPF fds to qemu virtio-net.
Also, libbpf dependency now exclusively for Linux.
Libbpf is used for eBPF RSS steering, which is supported only by Linux TAP.
There is no reason yet to build eBPF loader and helper for non Linux systems,
even if libbpf is present.
Signed-off-by: Andrew Melnychenko <andrew@daynix.com>
---
ebpf/qemu-ebpf-rss-helper.c | 130 ++++++++++++++++++++++++++++++++++++
meson.build | 37 ++++++----
2 files changed, 154 insertions(+), 13 deletions(-)
create mode 100644 ebpf/qemu-ebpf-rss-helper.c
diff --git a/ebpf/qemu-ebpf-rss-helper.c b/ebpf/qemu-ebpf-rss-helper.c
new file mode 100644
index 0000000000..fe68758f57
--- /dev/null
+++ b/ebpf/qemu-ebpf-rss-helper.c
@@ -0,0 +1,130 @@
+/*
+ * eBPF RSS Helper
+ *
+ * Developed by Daynix Computing LTD (http://www.daynix.com)
+ *
+ * Authors:
+ * Andrew Melnychenko <andrew@daynix.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2. See
+ * the COPYING file in the top-level directory.
+ *
+ * Description: This is helper program for libvirtd.
+ * It loads eBPF RSS program and passes fds through unix socket.
+ * Built by meson, target - 'qemu-ebpf-rss-helper'.
+ */
+
+#include <stdio.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdbool.h>
+#include <getopt.h>
+#include <memory.h>
+#include <errno.h>
+#include <sys/socket.h>
+
+#include "ebpf_rss.h"
+
+#include "qemu-helper-stamp.h"
+
+void QEMU_HELPER_STAMP(void) {}
+
+static int send_fds(int socket, int *fds, int n)
+{
+ struct msghdr msg = {};
+ struct cmsghdr *cmsg = NULL;
+ char buf[CMSG_SPACE(n * sizeof(int))];
+ char dummy_buffer = 0;
+ struct iovec io = { .iov_base = &dummy_buffer,
+ .iov_len = sizeof(dummy_buffer) };
+
+ memset(buf, 0, sizeof(buf));
+
+ msg.msg_iov = &io;
+ msg.msg_iovlen = 1;
+ msg.msg_control = buf;
+ msg.msg_controllen = sizeof(buf);
+
+ cmsg = CMSG_FIRSTHDR(&msg);
+ cmsg->cmsg_level = SOL_SOCKET;
+ cmsg->cmsg_type = SCM_RIGHTS;
+ cmsg->cmsg_len = CMSG_LEN(n * sizeof(int));
+
+ memcpy(CMSG_DATA(cmsg), fds, n * sizeof(int));
+
+ return sendmsg(socket, &msg, 0);
+}
+
+static void print_help_and_exit(const char *prog, int exitcode)
+{
+ fprintf(stderr, "%s - load eBPF RSS program for qemu and pass eBPF fds"
+ " through unix socket.\n", prog);
+ fprintf(stderr, "\t--fd <num>, -f <num> - unix socket file descriptor"
+ " used to pass eBPF fds.\n");
+ fprintf(stderr, "\t--help, -h - this help.\n");
+ exit(exitcode);
+}
+
+int main(int argc, char **argv)
+{
+ char *fd_string = NULL;
+ int unix_fd = 0;
+ struct EBPFRSSContext ctx = {};
+ int fds[EBPF_RSS_MAX_FDS] = {};
+ int ret = -1;
+
+ for (;;) {
+ int c;
+ static struct option long_options[] = {
+ {"help", no_argument, 0, 'h'},
+ {"fd", required_argument, 0, 'f'},
+ {0, 0, 0, 0}
+ };
+ c = getopt_long(argc, argv, "hf:",
+ long_options, NULL);
+
+ if (c == -1) {
+ break;
+ }
+
+ switch (c) {
+ case 'f':
+ fd_string = optarg;
+ break;
+ case 'h':
+ default:
+ print_help_and_exit(argv[0],
+ c == 'h' ? EXIT_SUCCESS : EXIT_FAILURE);
+ }
+ }
+
+ if (!fd_string) {
+ fprintf(stderr, "Unix file descriptor not present.\n");
+ print_help_and_exit(argv[0], EXIT_FAILURE);
+ }
+
+ unix_fd = atoi(fd_string);
+
+ if (!unix_fd) {
+ fprintf(stderr, "Unix file descriptor is invalid.\n");
+ return EXIT_FAILURE;
+ }
+
+ ebpf_rss_init(&ctx);
+ if (!ebpf_rss_load(&ctx)) {
+ fprintf(stderr, "Can't load ebpf.\n");
+ return EXIT_FAILURE;
+ }
+ fds[0] = ctx.program_fd;
+ fds[1] = ctx.map_configuration;
+
+ ret = send_fds(unix_fd, fds, EBPF_RSS_MAX_FDS);
+ if (ret < 0) {
+ fprintf(stderr, "Issue while sending fds: %s.\n", strerror(errno));
+ }
+
+ ebpf_rss_unload(&ctx);
+
+ return ret < 0 ? EXIT_FAILURE : EXIT_SUCCESS;
+}
+
diff --git a/meson.build b/meson.build
index 257e51d91b..913aa1fee5 100644
--- a/meson.build
+++ b/meson.build
@@ -1033,19 +1033,22 @@ if not get_option('fuse_lseek').disabled()
endif
# libbpf
-libbpf = dependency('libbpf', required: get_option('bpf'), method: 'pkg-config')
-if libbpf.found() and not cc.links('''
- #include <bpf/libbpf.h>
- int main(void)
- {
- bpf_object__destroy_skeleton(NULL);
- return 0;
- }''', dependencies: libbpf)
- libbpf = not_found
- if get_option('bpf').enabled()
- error('libbpf skeleton test failed')
- else
- warning('libbpf skeleton test failed, disabling')
+libbpf = not_found
+if targetos == 'linux'
+ libbpf = dependency('libbpf', required: get_option('bpf'), method: 'pkg-config')
+ if libbpf.found() and not cc.links('''
+ #include <bpf/libbpf.h>
+ int main(void)
+ {
+ bpf_object__destroy_skeleton(NULL);
+ return 0;
+ }''', dependencies: libbpf)
+ libbpf = not_found
+ if get_option('bpf').enabled()
+ error('libbpf skeleton test failed')
+ else
+ warning('libbpf skeleton test failed, disabling')
+ endif
endif
endif
@@ -2423,6 +2426,14 @@ if have_tools
dependencies: [authz, crypto, io, qom, qemuutil,
libcap_ng, mpathpersist],
install: true)
+
+ if libbpf.found()
+ executable('qemu-ebpf-rss-helper', files(
+ 'ebpf/qemu-ebpf-rss-helper.c', 'ebpf/ebpf_rss.c'),
+ dependencies: [qemuutil, libbpf, glib],
+ install: true,
+ install_dir: get_option('libexecdir'))
+ endif
endif
if 'CONFIG_IVSHMEM' in config_host
--
2.31.1
^ permalink raw reply related [flat|nested] 60+ messages in thread
* Re: [PATCH 4/5] ebpf_rss_helper: Added helper for eBPF RSS.
2021-07-13 15:37 ` [PATCH 4/5] ebpf_rss_helper: Added helper for eBPF RSS Andrew Melnychenko
@ 2021-08-20 3:40 ` Jason Wang
2021-08-25 18:24 ` Andrew Melnichenko
2021-08-30 17:07 ` Yuri Benditovich
0 siblings, 2 replies; 60+ messages in thread
From: Jason Wang @ 2021-08-20 3:40 UTC (permalink / raw)
To: Andrew Melnychenko, mst, yuri.benditovich, armbru, eblake, berrange
Cc: yan, qemu-devel
在 2021/7/13 下午11:37, Andrew Melnychenko 写道:
> Helper program. Loads eBPF RSS program and maps and passes them through unix socket.
> Libvirt may launch this helper and pass eBPF fds to qemu virtio-net.
I wonder if this can be done as helper for TAP/bridge.
E.g it's the qemu to launch those helper with set-uid.
Then libvirt won't even need to care about that?
> Also, libbpf dependency now exclusively for Linux.
> Libbpf is used for eBPF RSS steering, which is supported only by Linux TAP.
> There is no reason yet to build eBPF loader and helper for non Linux systems,
> even if libbpf is present.
>
> Signed-off-by: Andrew Melnychenko <andrew@daynix.com>
> ---
> ebpf/qemu-ebpf-rss-helper.c | 130 ++++++++++++++++++++++++++++++++++++
> meson.build | 37 ++++++----
> 2 files changed, 154 insertions(+), 13 deletions(-)
> create mode 100644 ebpf/qemu-ebpf-rss-helper.c
>
> diff --git a/ebpf/qemu-ebpf-rss-helper.c b/ebpf/qemu-ebpf-rss-helper.c
> new file mode 100644
> index 0000000000..fe68758f57
> --- /dev/null
> +++ b/ebpf/qemu-ebpf-rss-helper.c
> @@ -0,0 +1,130 @@
> +/*
> + * eBPF RSS Helper
> + *
> + * Developed by Daynix Computing LTD (http://www.daynix.com)
> + *
> + * Authors:
> + * Andrew Melnychenko <andrew@daynix.com>
> + *
> + * This work is licensed under the terms of the GNU GPL, version 2. See
> + * the COPYING file in the top-level directory.
> + *
> + * Description: This is helper program for libvirtd.
> + * It loads eBPF RSS program and passes fds through unix socket.
> + * Built by meson, target - 'qemu-ebpf-rss-helper'.
> + */
> +
> +#include <stdio.h>
> +#include <stdint.h>
> +#include <stdlib.h>
> +#include <stdbool.h>
> +#include <getopt.h>
> +#include <memory.h>
> +#include <errno.h>
> +#include <sys/socket.h>
> +
> +#include "ebpf_rss.h"
> +
> +#include "qemu-helper-stamp.h"
> +
> +void QEMU_HELPER_STAMP(void) {}
> +
> +static int send_fds(int socket, int *fds, int n)
> +{
> + struct msghdr msg = {};
> + struct cmsghdr *cmsg = NULL;
> + char buf[CMSG_SPACE(n * sizeof(int))];
> + char dummy_buffer = 0;
> + struct iovec io = { .iov_base = &dummy_buffer,
> + .iov_len = sizeof(dummy_buffer) };
> +
> + memset(buf, 0, sizeof(buf));
> +
> + msg.msg_iov = &io;
> + msg.msg_iovlen = 1;
> + msg.msg_control = buf;
> + msg.msg_controllen = sizeof(buf);
> +
> + cmsg = CMSG_FIRSTHDR(&msg);
> + cmsg->cmsg_level = SOL_SOCKET;
> + cmsg->cmsg_type = SCM_RIGHTS;
> + cmsg->cmsg_len = CMSG_LEN(n * sizeof(int));
> +
> + memcpy(CMSG_DATA(cmsg), fds, n * sizeof(int));
> +
> + return sendmsg(socket, &msg, 0);
> +}
> +
> +static void print_help_and_exit(const char *prog, int exitcode)
> +{
> + fprintf(stderr, "%s - load eBPF RSS program for qemu and pass eBPF fds"
> + " through unix socket.\n", prog);
> + fprintf(stderr, "\t--fd <num>, -f <num> - unix socket file descriptor"
> + " used to pass eBPF fds.\n");
> + fprintf(stderr, "\t--help, -h - this help.\n");
> + exit(exitcode);
> +}
> +
> +int main(int argc, char **argv)
> +{
> + char *fd_string = NULL;
> + int unix_fd = 0;
> + struct EBPFRSSContext ctx = {};
> + int fds[EBPF_RSS_MAX_FDS] = {};
> + int ret = -1;
> +
> + for (;;) {
> + int c;
> + static struct option long_options[] = {
> + {"help", no_argument, 0, 'h'},
> + {"fd", required_argument, 0, 'f'},
> + {0, 0, 0, 0}
> + };
> + c = getopt_long(argc, argv, "hf:",
> + long_options, NULL);
> +
> + if (c == -1) {
> + break;
> + }
> +
> + switch (c) {
> + case 'f':
> + fd_string = optarg;
> + break;
> + case 'h':
> + default:
> + print_help_and_exit(argv[0],
> + c == 'h' ? EXIT_SUCCESS : EXIT_FAILURE);
> + }
> + }
> +
> + if (!fd_string) {
> + fprintf(stderr, "Unix file descriptor not present.\n");
> + print_help_and_exit(argv[0], EXIT_FAILURE);
> + }
> +
> + unix_fd = atoi(fd_string);
> +
> + if (!unix_fd) {
> + fprintf(stderr, "Unix file descriptor is invalid.\n");
> + return EXIT_FAILURE;
> + }
> +
> + ebpf_rss_init(&ctx);
> + if (!ebpf_rss_load(&ctx)) {
> + fprintf(stderr, "Can't load ebpf.\n");
> + return EXIT_FAILURE;
> + }
> + fds[0] = ctx.program_fd;
> + fds[1] = ctx.map_configuration;
> +
> + ret = send_fds(unix_fd, fds, EBPF_RSS_MAX_FDS);
> + if (ret < 0) {
> + fprintf(stderr, "Issue while sending fds: %s.\n", strerror(errno));
> + }
> +
> + ebpf_rss_unload(&ctx);
> +
> + return ret < 0 ? EXIT_FAILURE : EXIT_SUCCESS;
> +}
> +
> diff --git a/meson.build b/meson.build
> index 257e51d91b..913aa1fee5 100644
> --- a/meson.build
> +++ b/meson.build
> @@ -1033,19 +1033,22 @@ if not get_option('fuse_lseek').disabled()
> endif
>
> # libbpf
> -libbpf = dependency('libbpf', required: get_option('bpf'), method: 'pkg-config')
> -if libbpf.found() and not cc.links('''
> - #include <bpf/libbpf.h>
> - int main(void)
> - {
> - bpf_object__destroy_skeleton(NULL);
> - return 0;
> - }''', dependencies: libbpf)
> - libbpf = not_found
> - if get_option('bpf').enabled()
> - error('libbpf skeleton test failed')
> - else
> - warning('libbpf skeleton test failed, disabling')
> +libbpf = not_found
> +if targetos == 'linux'
> + libbpf = dependency('libbpf', required: get_option('bpf'), method: 'pkg-config')
> + if libbpf.found() and not cc.links('''
> + #include <bpf/libbpf.h>
> + int main(void)
> + {
> + bpf_object__destroy_skeleton(NULL);
Do we need to test whether the bpf can do mmap() here?
Thanks
> + return 0;
> + }''', dependencies: libbpf)
> + libbpf = not_found
> + if get_option('bpf').enabled()
> + error('libbpf skeleton test failed')
> + else
> + warning('libbpf skeleton test failed, disabling')
> + endif
> endif
> endif
>
> @@ -2423,6 +2426,14 @@ if have_tools
> dependencies: [authz, crypto, io, qom, qemuutil,
> libcap_ng, mpathpersist],
> install: true)
> +
> + if libbpf.found()
> + executable('qemu-ebpf-rss-helper', files(
> + 'ebpf/qemu-ebpf-rss-helper.c', 'ebpf/ebpf_rss.c'),
> + dependencies: [qemuutil, libbpf, glib],
> + install: true,
> + install_dir: get_option('libexecdir'))
> + endif
> endif
>
> if 'CONFIG_IVSHMEM' in config_host
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 4/5] ebpf_rss_helper: Added helper for eBPF RSS.
2021-08-20 3:40 ` Jason Wang
@ 2021-08-25 18:24 ` Andrew Melnichenko
2021-09-01 6:37 ` Jason Wang
2021-08-30 17:07 ` Yuri Benditovich
1 sibling, 1 reply; 60+ messages in thread
From: Andrew Melnichenko @ 2021-08-25 18:24 UTC (permalink / raw)
To: Jason Wang
Cc: Daniel P. Berrangé,
Michael S. Tsirkin, qemu-devel, Markus Armbruster,
Yuri Benditovich, Yan Vugenfirer, Eric Blake
[-- Attachment #1: Type: text/plain, Size: 7890 bytes --]
Hi,
> I wonder if this can be done as helper for TAP/bridge.
>
Well, it does already, libvirt may create TAP device and pass it in command
line or using getfd qmp command.
E.g it's the qemu to launch those helper with set-uid.
>
Then libvirt won't even need to care about that?
Yea, we may think about this routine in the future as a fallback.
Do we need to test whether the bpf can do mmap() here?
>
I'm not sure that it's required.
On Fri, Aug 20, 2021 at 6:41 AM Jason Wang <jasowang@redhat.com> wrote:
>
> 在 2021/7/13 下午11:37, Andrew Melnychenko 写道:
> > Helper program. Loads eBPF RSS program and maps and passes them through
> unix socket.
> > Libvirt may launch this helper and pass eBPF fds to qemu virtio-net.
>
>
> I wonder if this can be done as helper for TAP/bridge.
>
> E.g it's the qemu to launch those helper with set-uid.
>
> Then libvirt won't even need to care about that?
>
>
> > Also, libbpf dependency now exclusively for Linux.
> > Libbpf is used for eBPF RSS steering, which is supported only by Linux
> TAP.
> > There is no reason yet to build eBPF loader and helper for non Linux
> systems,
> > even if libbpf is present.
> >
> > Signed-off-by: Andrew Melnychenko <andrew@daynix.com>
> > ---
> > ebpf/qemu-ebpf-rss-helper.c | 130 ++++++++++++++++++++++++++++++++++++
> > meson.build | 37 ++++++----
> > 2 files changed, 154 insertions(+), 13 deletions(-)
> > create mode 100644 ebpf/qemu-ebpf-rss-helper.c
> >
> > diff --git a/ebpf/qemu-ebpf-rss-helper.c b/ebpf/qemu-ebpf-rss-helper.c
> > new file mode 100644
> > index 0000000000..fe68758f57
> > --- /dev/null
> > +++ b/ebpf/qemu-ebpf-rss-helper.c
> > @@ -0,0 +1,130 @@
> > +/*
> > + * eBPF RSS Helper
> > + *
> > + * Developed by Daynix Computing LTD (http://www.daynix.com)
> > + *
> > + * Authors:
> > + * Andrew Melnychenko <andrew@daynix.com>
> > + *
> > + * This work is licensed under the terms of the GNU GPL, version 2. See
> > + * the COPYING file in the top-level directory.
> > + *
> > + * Description: This is helper program for libvirtd.
> > + * It loads eBPF RSS program and passes fds through unix
> socket.
> > + * Built by meson, target - 'qemu-ebpf-rss-helper'.
> > + */
> > +
> > +#include <stdio.h>
> > +#include <stdint.h>
> > +#include <stdlib.h>
> > +#include <stdbool.h>
> > +#include <getopt.h>
> > +#include <memory.h>
> > +#include <errno.h>
> > +#include <sys/socket.h>
> > +
> > +#include "ebpf_rss.h"
> > +
> > +#include "qemu-helper-stamp.h"
> > +
> > +void QEMU_HELPER_STAMP(void) {}
> > +
> > +static int send_fds(int socket, int *fds, int n)
> > +{
> > + struct msghdr msg = {};
> > + struct cmsghdr *cmsg = NULL;
> > + char buf[CMSG_SPACE(n * sizeof(int))];
> > + char dummy_buffer = 0;
> > + struct iovec io = { .iov_base = &dummy_buffer,
> > + .iov_len = sizeof(dummy_buffer) };
> > +
> > + memset(buf, 0, sizeof(buf));
> > +
> > + msg.msg_iov = &io;
> > + msg.msg_iovlen = 1;
> > + msg.msg_control = buf;
> > + msg.msg_controllen = sizeof(buf);
> > +
> > + cmsg = CMSG_FIRSTHDR(&msg);
> > + cmsg->cmsg_level = SOL_SOCKET;
> > + cmsg->cmsg_type = SCM_RIGHTS;
> > + cmsg->cmsg_len = CMSG_LEN(n * sizeof(int));
> > +
> > + memcpy(CMSG_DATA(cmsg), fds, n * sizeof(int));
> > +
> > + return sendmsg(socket, &msg, 0);
> > +}
> > +
> > +static void print_help_and_exit(const char *prog, int exitcode)
> > +{
> > + fprintf(stderr, "%s - load eBPF RSS program for qemu and pass eBPF
> fds"
> > + " through unix socket.\n", prog);
> > + fprintf(stderr, "\t--fd <num>, -f <num> - unix socket file
> descriptor"
> > + " used to pass eBPF fds.\n");
> > + fprintf(stderr, "\t--help, -h - this help.\n");
> > + exit(exitcode);
> > +}
> > +
> > +int main(int argc, char **argv)
> > +{
> > + char *fd_string = NULL;
> > + int unix_fd = 0;
> > + struct EBPFRSSContext ctx = {};
> > + int fds[EBPF_RSS_MAX_FDS] = {};
> > + int ret = -1;
> > +
> > + for (;;) {
> > + int c;
> > + static struct option long_options[] = {
> > + {"help", no_argument, 0, 'h'},
> > + {"fd", required_argument, 0, 'f'},
> > + {0, 0, 0, 0}
> > + };
> > + c = getopt_long(argc, argv, "hf:",
> > + long_options, NULL);
> > +
> > + if (c == -1) {
> > + break;
> > + }
> > +
> > + switch (c) {
> > + case 'f':
> > + fd_string = optarg;
> > + break;
> > + case 'h':
> > + default:
> > + print_help_and_exit(argv[0],
> > + c == 'h' ? EXIT_SUCCESS : EXIT_FAILURE);
> > + }
> > + }
> > +
> > + if (!fd_string) {
> > + fprintf(stderr, "Unix file descriptor not present.\n");
> > + print_help_and_exit(argv[0], EXIT_FAILURE);
> > + }
> > +
> > + unix_fd = atoi(fd_string);
> > +
> > + if (!unix_fd) {
> > + fprintf(stderr, "Unix file descriptor is invalid.\n");
> > + return EXIT_FAILURE;
> > + }
> > +
> > + ebpf_rss_init(&ctx);
> > + if (!ebpf_rss_load(&ctx)) {
> > + fprintf(stderr, "Can't load ebpf.\n");
> > + return EXIT_FAILURE;
> > + }
> > + fds[0] = ctx.program_fd;
> > + fds[1] = ctx.map_configuration;
> > +
> > + ret = send_fds(unix_fd, fds, EBPF_RSS_MAX_FDS);
> > + if (ret < 0) {
> > + fprintf(stderr, "Issue while sending fds: %s.\n",
> strerror(errno));
> > + }
> > +
> > + ebpf_rss_unload(&ctx);
> > +
> > + return ret < 0 ? EXIT_FAILURE : EXIT_SUCCESS;
> > +}
> > +
> > diff --git a/meson.build b/meson.build
> > index 257e51d91b..913aa1fee5 100644
> > --- a/meson.build
> > +++ b/meson.build
> > @@ -1033,19 +1033,22 @@ if not get_option('fuse_lseek').disabled()
> > endif
> >
> > # libbpf
> > -libbpf = dependency('libbpf', required: get_option('bpf'), method:
> 'pkg-config')
> > -if libbpf.found() and not cc.links('''
> > - #include <bpf/libbpf.h>
> > - int main(void)
> > - {
> > - bpf_object__destroy_skeleton(NULL);
> > - return 0;
> > - }''', dependencies: libbpf)
> > - libbpf = not_found
> > - if get_option('bpf').enabled()
> > - error('libbpf skeleton test failed')
> > - else
> > - warning('libbpf skeleton test failed, disabling')
> > +libbpf = not_found
> > +if targetos == 'linux'
> > + libbpf = dependency('libbpf', required: get_option('bpf'), method:
> 'pkg-config')
> > + if libbpf.found() and not cc.links('''
> > + #include <bpf/libbpf.h>
> > + int main(void)
> > + {
> > + bpf_object__destroy_skeleton(NULL);
>
>
> Do we need to test whether the bpf can do mmap() here?
>
> Thanks
>
>
> > + return 0;
> > + }''', dependencies: libbpf)
> > + libbpf = not_found
> > + if get_option('bpf').enabled()
> > + error('libbpf skeleton test failed')
> > + else
> > + warning('libbpf skeleton test failed, disabling')
> > + endif
> > endif
> > endif
> >
> > @@ -2423,6 +2426,14 @@ if have_tools
> > dependencies: [authz, crypto, io, qom, qemuutil,
> > libcap_ng, mpathpersist],
> > install: true)
> > +
> > + if libbpf.found()
> > + executable('qemu-ebpf-rss-helper', files(
> > + 'ebpf/qemu-ebpf-rss-helper.c', 'ebpf/ebpf_rss.c'),
> > + dependencies: [qemuutil, libbpf, glib],
> > + install: true,
> > + install_dir: get_option('libexecdir'))
> > + endif
> > endif
> >
> > if 'CONFIG_IVSHMEM' in config_host
>
>
[-- Attachment #2: Type: text/html, Size: 10998 bytes --]
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 4/5] ebpf_rss_helper: Added helper for eBPF RSS.
2021-08-25 18:24 ` Andrew Melnichenko
@ 2021-09-01 6:37 ` Jason Wang
0 siblings, 0 replies; 60+ messages in thread
From: Jason Wang @ 2021-09-01 6:37 UTC (permalink / raw)
To: Andrew Melnichenko
Cc: Daniel P. Berrangé,
Michael S. Tsirkin, qemu-devel, Markus Armbruster,
Yuri Benditovich, Yan Vugenfirer, Eric Blake
在 2021/8/26 上午2:24, Andrew Melnichenko 写道:
> Hi,
>
> I wonder if this can be done as helper for TAP/bridge.
>
> Well, it does already, libvirt may create TAP device and pass it in
> command line or using getfd qmp command.
>
> E.g it's the qemu to launch those helper with set-uid.
>
> Then libvirt won't even need to care about that?
>
> Yea, we may think about this routine in the future as a fallback.
>
> Do we need to test whether the bpf can do mmap() here?
>
> I'm not sure that it's required.
I think it's for back-compatibility.
E.g current codes works without mmap(), and user will surprise that it
wont' work after upgrading their qemu.
Thanks
>
> On Fri, Aug 20, 2021 at 6:41 AM Jason Wang <jasowang@redhat.com
> <mailto:jasowang@redhat.com>> wrote:
>
>
> 在 2021/7/13 下午11:37, Andrew Melnychenko 写道:
> > Helper program. Loads eBPF RSS program and maps and passes them
> through unix socket.
> > Libvirt may launch this helper and pass eBPF fds to qemu virtio-net.
>
>
> I wonder if this can be done as helper for TAP/bridge.
>
> E.g it's the qemu to launch those helper with set-uid.
>
> Then libvirt won't even need to care about that?
>
>
> > Also, libbpf dependency now exclusively for Linux.
> > Libbpf is used for eBPF RSS steering, which is supported only by
> Linux TAP.
> > There is no reason yet to build eBPF loader and helper for non
> Linux systems,
> > even if libbpf is present.
> >
> > Signed-off-by: Andrew Melnychenko <andrew@daynix.com
> <mailto:andrew@daynix.com>>
> > ---
> > ebpf/qemu-ebpf-rss-helper.c | 130
> ++++++++++++++++++++++++++++++++++++
> > meson.build | 37 ++++++----
> > 2 files changed, 154 insertions(+), 13 deletions(-)
> > create mode 100644 ebpf/qemu-ebpf-rss-helper.c
> >
> > diff --git a/ebpf/qemu-ebpf-rss-helper.c
> b/ebpf/qemu-ebpf-rss-helper.c
> > new file mode 100644
> > index 0000000000..fe68758f57
> > --- /dev/null
> > +++ b/ebpf/qemu-ebpf-rss-helper.c
> > @@ -0,0 +1,130 @@
> > +/*
> > + * eBPF RSS Helper
> > + *
> > + * Developed by Daynix Computing LTD (http://www.daynix.com
> <http://www.daynix.com>)
> > + *
> > + * Authors:
> > + * Andrew Melnychenko <andrew@daynix.com
> <mailto:andrew@daynix.com>>
> > + *
> > + * This work is licensed under the terms of the GNU GPL,
> version 2. See
> > + * the COPYING file in the top-level directory.
> > + *
> > + * Description: This is helper program for libvirtd.
> > + * It loads eBPF RSS program and passes fds
> through unix socket.
> > + * Built by meson, target - 'qemu-ebpf-rss-helper'.
> > + */
> > +
> > +#include <stdio.h>
> > +#include <stdint.h>
> > +#include <stdlib.h>
> > +#include <stdbool.h>
> > +#include <getopt.h>
> > +#include <memory.h>
> > +#include <errno.h>
> > +#include <sys/socket.h>
> > +
> > +#include "ebpf_rss.h"
> > +
> > +#include "qemu-helper-stamp.h"
> > +
> > +void QEMU_HELPER_STAMP(void) {}
> > +
> > +static int send_fds(int socket, int *fds, int n)
> > +{
> > + struct msghdr msg = {};
> > + struct cmsghdr *cmsg = NULL;
> > + char buf[CMSG_SPACE(n * sizeof(int))];
> > + char dummy_buffer = 0;
> > + struct iovec io = { .iov_base = &dummy_buffer,
> > + .iov_len = sizeof(dummy_buffer) };
> > +
> > + memset(buf, 0, sizeof(buf));
> > +
> > + msg.msg_iov = &io;
> > + msg.msg_iovlen = 1;
> > + msg.msg_control = buf;
> > + msg.msg_controllen = sizeof(buf);
> > +
> > + cmsg = CMSG_FIRSTHDR(&msg);
> > + cmsg->cmsg_level = SOL_SOCKET;
> > + cmsg->cmsg_type = SCM_RIGHTS;
> > + cmsg->cmsg_len = CMSG_LEN(n * sizeof(int));
> > +
> > + memcpy(CMSG_DATA(cmsg), fds, n * sizeof(int));
> > +
> > + return sendmsg(socket, &msg, 0);
> > +}
> > +
> > +static void print_help_and_exit(const char *prog, int exitcode)
> > +{
> > + fprintf(stderr, "%s - load eBPF RSS program for qemu and
> pass eBPF fds"
> > + " through unix socket.\n", prog);
> > + fprintf(stderr, "\t--fd <num>, -f <num> - unix socket file
> descriptor"
> > + " used to pass eBPF fds.\n");
> > + fprintf(stderr, "\t--help, -h - this help.\n");
> > + exit(exitcode);
> > +}
> > +
> > +int main(int argc, char **argv)
> > +{
> > + char *fd_string = NULL;
> > + int unix_fd = 0;
> > + struct EBPFRSSContext ctx = {};
> > + int fds[EBPF_RSS_MAX_FDS] = {};
> > + int ret = -1;
> > +
> > + for (;;) {
> > + int c;
> > + static struct option long_options[] = {
> > + {"help", no_argument, 0, 'h'},
> > + {"fd", required_argument, 0, 'f'},
> > + {0, 0, 0, 0}
> > + };
> > + c = getopt_long(argc, argv, "hf:",
> > + long_options, NULL);
> > +
> > + if (c == -1) {
> > + break;
> > + }
> > +
> > + switch (c) {
> > + case 'f':
> > + fd_string = optarg;
> > + break;
> > + case 'h':
> > + default:
> > + print_help_and_exit(argv[0],
> > + c == 'h' ? EXIT_SUCCESS : EXIT_FAILURE);
> > + }
> > + }
> > +
> > + if (!fd_string) {
> > + fprintf(stderr, "Unix file descriptor not present.\n");
> > + print_help_and_exit(argv[0], EXIT_FAILURE);
> > + }
> > +
> > + unix_fd = atoi(fd_string);
> > +
> > + if (!unix_fd) {
> > + fprintf(stderr, "Unix file descriptor is invalid.\n");
> > + return EXIT_FAILURE;
> > + }
> > +
> > + ebpf_rss_init(&ctx);
> > + if (!ebpf_rss_load(&ctx)) {
> > + fprintf(stderr, "Can't load ebpf.\n");
> > + return EXIT_FAILURE;
> > + }
> > + fds[0] = ctx.program_fd;
> > + fds[1] = ctx.map_configuration;
> > +
> > + ret = send_fds(unix_fd, fds, EBPF_RSS_MAX_FDS);
> > + if (ret < 0) {
> > + fprintf(stderr, "Issue while sending fds: %s.\n",
> strerror(errno));
> > + }
> > +
> > + ebpf_rss_unload(&ctx);
> > +
> > + return ret < 0 ? EXIT_FAILURE : EXIT_SUCCESS;
> > +}
> > +
> > diff --git a/meson.build b/meson.build
> > index 257e51d91b..913aa1fee5 100644
> > --- a/meson.build
> > +++ b/meson.build
> > @@ -1033,19 +1033,22 @@ if not get_option('fuse_lseek').disabled()
> > endif
> >
> > # libbpf
> > -libbpf = dependency('libbpf', required: get_option('bpf'),
> method: 'pkg-config')
> > -if libbpf.found() and not cc.links('''
> > - #include <bpf/libbpf.h>
> > - int main(void)
> > - {
> > - bpf_object__destroy_skeleton(NULL);
> > - return 0;
> > - }''', dependencies: libbpf)
> > - libbpf = not_found
> > - if get_option('bpf').enabled()
> > - error('libbpf skeleton test failed')
> > - else
> > - warning('libbpf skeleton test failed, disabling')
> > +libbpf = not_found
> > +if targetos == 'linux'
> > + libbpf = dependency('libbpf', required: get_option('bpf'),
> method: 'pkg-config')
> > + if libbpf.found() and not cc.links('''
> > + #include <bpf/libbpf.h>
> > + int main(void)
> > + {
> > + bpf_object__destroy_skeleton(NULL);
>
>
> Do we need to test whether the bpf can do mmap() here?
>
> Thanks
>
>
> > + return 0;
> > + }''', dependencies: libbpf)
> > + libbpf = not_found
> > + if get_option('bpf').enabled()
> > + error('libbpf skeleton test failed')
> > + else
> > + warning('libbpf skeleton test failed, disabling')
> > + endif
> > endif
> > endif
> >
> > @@ -2423,6 +2426,14 @@ if have_tools
> > dependencies: [authz, crypto, io, qom, qemuutil,
> > libcap_ng, mpathpersist],
> > install: true)
> > +
> > + if libbpf.found()
> > + executable('qemu-ebpf-rss-helper', files(
> > + 'ebpf/qemu-ebpf-rss-helper.c',
> 'ebpf/ebpf_rss.c'),
> > + dependencies: [qemuutil, libbpf, glib],
> > + install: true,
> > + install_dir: get_option('libexecdir'))
> > + endif
> > endif
> >
> > if 'CONFIG_IVSHMEM' in config_host
>
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 4/5] ebpf_rss_helper: Added helper for eBPF RSS.
2021-08-20 3:40 ` Jason Wang
2021-08-25 18:24 ` Andrew Melnichenko
@ 2021-08-30 17:07 ` Yuri Benditovich
2021-09-01 6:42 ` Jason Wang
1 sibling, 1 reply; 60+ messages in thread
From: Yuri Benditovich @ 2021-08-30 17:07 UTC (permalink / raw)
To: Jason Wang
Cc: Andrew Melnychenko, Daniel P. Berrangé,
Michael S . Tsirkin, qemu-devel, Markus Armbruster,
Yan Vugenfirer, Eric Blake
On Fri, Aug 20, 2021 at 6:41 AM Jason Wang <jasowang@redhat.com> wrote:
>
>
> 在 2021/7/13 下午11:37, Andrew Melnychenko 写道:
> > Helper program. Loads eBPF RSS program and maps and passes them through unix socket.
> > Libvirt may launch this helper and pass eBPF fds to qemu virtio-net.
>
>
> I wonder if this can be done as helper for TAP/bridge.
>
> E.g it's the qemu to launch those helper with set-uid.
>
> Then libvirt won't even need to care about that?
>
There are pros and cons for such a solution with set-uid.
From my point of view one of the cons is that set-uid is efficient
only at install time so the coexistence of different qemu builds (and
different helpers for each one) is kind of problematic.
With the current solution this does not present any problem: the
developer can have several different builds, each one automatically
has its own helper and there is no conflict between these builds and
between these builds and installed qemu package. Changing the
'emulator' in the libvirt profile automatically brings the proper
helper to work.
>
> > Also, libbpf dependency now exclusively for Linux.
> > Libbpf is used for eBPF RSS steering, which is supported only by Linux TAP.
> > There is no reason yet to build eBPF loader and helper for non Linux systems,
> > even if libbpf is present.
> >
> > Signed-off-by: Andrew Melnychenko <andrew@daynix.com>
> > ---
> > ebpf/qemu-ebpf-rss-helper.c | 130 ++++++++++++++++++++++++++++++++++++
> > meson.build | 37 ++++++----
> > 2 files changed, 154 insertions(+), 13 deletions(-)
> > create mode 100644 ebpf/qemu-ebpf-rss-helper.c
> >
> > diff --git a/ebpf/qemu-ebpf-rss-helper.c b/ebpf/qemu-ebpf-rss-helper.c
> > new file mode 100644
> > index 0000000000..fe68758f57
> > --- /dev/null
> > +++ b/ebpf/qemu-ebpf-rss-helper.c
> > @@ -0,0 +1,130 @@
> > +/*
> > + * eBPF RSS Helper
> > + *
> > + * Developed by Daynix Computing LTD (http://www.daynix.com)
> > + *
> > + * Authors:
> > + * Andrew Melnychenko <andrew@daynix.com>
> > + *
> > + * This work is licensed under the terms of the GNU GPL, version 2. See
> > + * the COPYING file in the top-level directory.
> > + *
> > + * Description: This is helper program for libvirtd.
> > + * It loads eBPF RSS program and passes fds through unix socket.
> > + * Built by meson, target - 'qemu-ebpf-rss-helper'.
> > + */
> > +
> > +#include <stdio.h>
> > +#include <stdint.h>
> > +#include <stdlib.h>
> > +#include <stdbool.h>
> > +#include <getopt.h>
> > +#include <memory.h>
> > +#include <errno.h>
> > +#include <sys/socket.h>
> > +
> > +#include "ebpf_rss.h"
> > +
> > +#include "qemu-helper-stamp.h"
> > +
> > +void QEMU_HELPER_STAMP(void) {}
> > +
> > +static int send_fds(int socket, int *fds, int n)
> > +{
> > + struct msghdr msg = {};
> > + struct cmsghdr *cmsg = NULL;
> > + char buf[CMSG_SPACE(n * sizeof(int))];
> > + char dummy_buffer = 0;
> > + struct iovec io = { .iov_base = &dummy_buffer,
> > + .iov_len = sizeof(dummy_buffer) };
> > +
> > + memset(buf, 0, sizeof(buf));
> > +
> > + msg.msg_iov = &io;
> > + msg.msg_iovlen = 1;
> > + msg.msg_control = buf;
> > + msg.msg_controllen = sizeof(buf);
> > +
> > + cmsg = CMSG_FIRSTHDR(&msg);
> > + cmsg->cmsg_level = SOL_SOCKET;
> > + cmsg->cmsg_type = SCM_RIGHTS;
> > + cmsg->cmsg_len = CMSG_LEN(n * sizeof(int));
> > +
> > + memcpy(CMSG_DATA(cmsg), fds, n * sizeof(int));
> > +
> > + return sendmsg(socket, &msg, 0);
> > +}
> > +
> > +static void print_help_and_exit(const char *prog, int exitcode)
> > +{
> > + fprintf(stderr, "%s - load eBPF RSS program for qemu and pass eBPF fds"
> > + " through unix socket.\n", prog);
> > + fprintf(stderr, "\t--fd <num>, -f <num> - unix socket file descriptor"
> > + " used to pass eBPF fds.\n");
> > + fprintf(stderr, "\t--help, -h - this help.\n");
> > + exit(exitcode);
> > +}
> > +
> > +int main(int argc, char **argv)
> > +{
> > + char *fd_string = NULL;
> > + int unix_fd = 0;
> > + struct EBPFRSSContext ctx = {};
> > + int fds[EBPF_RSS_MAX_FDS] = {};
> > + int ret = -1;
> > +
> > + for (;;) {
> > + int c;
> > + static struct option long_options[] = {
> > + {"help", no_argument, 0, 'h'},
> > + {"fd", required_argument, 0, 'f'},
> > + {0, 0, 0, 0}
> > + };
> > + c = getopt_long(argc, argv, "hf:",
> > + long_options, NULL);
> > +
> > + if (c == -1) {
> > + break;
> > + }
> > +
> > + switch (c) {
> > + case 'f':
> > + fd_string = optarg;
> > + break;
> > + case 'h':
> > + default:
> > + print_help_and_exit(argv[0],
> > + c == 'h' ? EXIT_SUCCESS : EXIT_FAILURE);
> > + }
> > + }
> > +
> > + if (!fd_string) {
> > + fprintf(stderr, "Unix file descriptor not present.\n");
> > + print_help_and_exit(argv[0], EXIT_FAILURE);
> > + }
> > +
> > + unix_fd = atoi(fd_string);
> > +
> > + if (!unix_fd) {
> > + fprintf(stderr, "Unix file descriptor is invalid.\n");
> > + return EXIT_FAILURE;
> > + }
> > +
> > + ebpf_rss_init(&ctx);
> > + if (!ebpf_rss_load(&ctx)) {
> > + fprintf(stderr, "Can't load ebpf.\n");
> > + return EXIT_FAILURE;
> > + }
> > + fds[0] = ctx.program_fd;
> > + fds[1] = ctx.map_configuration;
> > +
> > + ret = send_fds(unix_fd, fds, EBPF_RSS_MAX_FDS);
> > + if (ret < 0) {
> > + fprintf(stderr, "Issue while sending fds: %s.\n", strerror(errno));
> > + }
> > +
> > + ebpf_rss_unload(&ctx);
> > +
> > + return ret < 0 ? EXIT_FAILURE : EXIT_SUCCESS;
> > +}
> > +
> > diff --git a/meson.build b/meson.build
> > index 257e51d91b..913aa1fee5 100644
> > --- a/meson.build
> > +++ b/meson.build
> > @@ -1033,19 +1033,22 @@ if not get_option('fuse_lseek').disabled()
> > endif
> >
> > # libbpf
> > -libbpf = dependency('libbpf', required: get_option('bpf'), method: 'pkg-config')
> > -if libbpf.found() and not cc.links('''
> > - #include <bpf/libbpf.h>
> > - int main(void)
> > - {
> > - bpf_object__destroy_skeleton(NULL);
> > - return 0;
> > - }''', dependencies: libbpf)
> > - libbpf = not_found
> > - if get_option('bpf').enabled()
> > - error('libbpf skeleton test failed')
> > - else
> > - warning('libbpf skeleton test failed, disabling')
> > +libbpf = not_found
> > +if targetos == 'linux'
> > + libbpf = dependency('libbpf', required: get_option('bpf'), method: 'pkg-config')
> > + if libbpf.found() and not cc.links('''
> > + #include <bpf/libbpf.h>
> > + int main(void)
> > + {
> > + bpf_object__destroy_skeleton(NULL);
>
>
> Do we need to test whether the bpf can do mmap() here?
>
> Thanks
>
>
> > + return 0;
> > + }''', dependencies: libbpf)
> > + libbpf = not_found
> > + if get_option('bpf').enabled()
> > + error('libbpf skeleton test failed')
> > + else
> > + warning('libbpf skeleton test failed, disabling')
> > + endif
> > endif
> > endif
> >
> > @@ -2423,6 +2426,14 @@ if have_tools
> > dependencies: [authz, crypto, io, qom, qemuutil,
> > libcap_ng, mpathpersist],
> > install: true)
> > +
> > + if libbpf.found()
> > + executable('qemu-ebpf-rss-helper', files(
> > + 'ebpf/qemu-ebpf-rss-helper.c', 'ebpf/ebpf_rss.c'),
> > + dependencies: [qemuutil, libbpf, glib],
> > + install: true,
> > + install_dir: get_option('libexecdir'))
> > + endif
> > endif
> >
> > if 'CONFIG_IVSHMEM' in config_host
>
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 4/5] ebpf_rss_helper: Added helper for eBPF RSS.
2021-08-30 17:07 ` Yuri Benditovich
@ 2021-09-01 6:42 ` Jason Wang
2021-09-06 15:50 ` Andrew Melnichenko
2021-09-07 10:40 ` Yuri Benditovich
0 siblings, 2 replies; 60+ messages in thread
From: Jason Wang @ 2021-09-01 6:42 UTC (permalink / raw)
To: Yuri Benditovich
Cc: Andrew Melnychenko, Daniel P. Berrangé,
Michael S . Tsirkin, qemu-devel, Markus Armbruster,
Yan Vugenfirer, Eric Blake
在 2021/8/31 上午1:07, Yuri Benditovich 写道:
> On Fri, Aug 20, 2021 at 6:41 AM Jason Wang <jasowang@redhat.com> wrote:
>>
>> 在 2021/7/13 下午11:37, Andrew Melnychenko 写道:
>>> Helper program. Loads eBPF RSS program and maps and passes them through unix socket.
>>> Libvirt may launch this helper and pass eBPF fds to qemu virtio-net.
>>
>> I wonder if this can be done as helper for TAP/bridge.
>>
>> E.g it's the qemu to launch those helper with set-uid.
>>
>> Then libvirt won't even need to care about that?
>>
> There are pros and cons for such a solution with set-uid.
> From my point of view one of the cons is that set-uid is efficient
> only at install time so the coexistence of different qemu builds (and
> different helpers for each one) is kind of problematic.
> With the current solution this does not present any problem: the
> developer can have several different builds, each one automatically
> has its own helper and there is no conflict between these builds and
> between these builds and installed qemu package. Changing the
> 'emulator' in the libvirt profile automatically brings the proper
> helper to work.
I'm not sure I get you here. We can still have default/sample helper to
make sure it works for different builds.
If we can avoid the involvement of libvirt, that would be better.
Thanks
>
>>> Also, libbpf dependency now exclusively for Linux.
>>> Libbpf is used for eBPF RSS steering, which is supported only by Linux TAP.
>>> There is no reason yet to build eBPF loader and helper for non Linux systems,
>>> even if libbpf is present.
>>>
>>> Signed-off-by: Andrew Melnychenko <andrew@daynix.com>
>>> ---
>>> ebpf/qemu-ebpf-rss-helper.c | 130 ++++++++++++++++++++++++++++++++++++
>>> meson.build | 37 ++++++----
>>> 2 files changed, 154 insertions(+), 13 deletions(-)
>>> create mode 100644 ebpf/qemu-ebpf-rss-helper.c
>>>
>>> diff --git a/ebpf/qemu-ebpf-rss-helper.c b/ebpf/qemu-ebpf-rss-helper.c
>>> new file mode 100644
>>> index 0000000000..fe68758f57
>>> --- /dev/null
>>> +++ b/ebpf/qemu-ebpf-rss-helper.c
>>> @@ -0,0 +1,130 @@
>>> +/*
>>> + * eBPF RSS Helper
>>> + *
>>> + * Developed by Daynix Computing LTD (http://www.daynix.com)
>>> + *
>>> + * Authors:
>>> + * Andrew Melnychenko <andrew@daynix.com>
>>> + *
>>> + * This work is licensed under the terms of the GNU GPL, version 2. See
>>> + * the COPYING file in the top-level directory.
>>> + *
>>> + * Description: This is helper program for libvirtd.
>>> + * It loads eBPF RSS program and passes fds through unix socket.
>>> + * Built by meson, target - 'qemu-ebpf-rss-helper'.
>>> + */
>>> +
>>> +#include <stdio.h>
>>> +#include <stdint.h>
>>> +#include <stdlib.h>
>>> +#include <stdbool.h>
>>> +#include <getopt.h>
>>> +#include <memory.h>
>>> +#include <errno.h>
>>> +#include <sys/socket.h>
>>> +
>>> +#include "ebpf_rss.h"
>>> +
>>> +#include "qemu-helper-stamp.h"
>>> +
>>> +void QEMU_HELPER_STAMP(void) {}
>>> +
>>> +static int send_fds(int socket, int *fds, int n)
>>> +{
>>> + struct msghdr msg = {};
>>> + struct cmsghdr *cmsg = NULL;
>>> + char buf[CMSG_SPACE(n * sizeof(int))];
>>> + char dummy_buffer = 0;
>>> + struct iovec io = { .iov_base = &dummy_buffer,
>>> + .iov_len = sizeof(dummy_buffer) };
>>> +
>>> + memset(buf, 0, sizeof(buf));
>>> +
>>> + msg.msg_iov = &io;
>>> + msg.msg_iovlen = 1;
>>> + msg.msg_control = buf;
>>> + msg.msg_controllen = sizeof(buf);
>>> +
>>> + cmsg = CMSG_FIRSTHDR(&msg);
>>> + cmsg->cmsg_level = SOL_SOCKET;
>>> + cmsg->cmsg_type = SCM_RIGHTS;
>>> + cmsg->cmsg_len = CMSG_LEN(n * sizeof(int));
>>> +
>>> + memcpy(CMSG_DATA(cmsg), fds, n * sizeof(int));
>>> +
>>> + return sendmsg(socket, &msg, 0);
>>> +}
>>> +
>>> +static void print_help_and_exit(const char *prog, int exitcode)
>>> +{
>>> + fprintf(stderr, "%s - load eBPF RSS program for qemu and pass eBPF fds"
>>> + " through unix socket.\n", prog);
>>> + fprintf(stderr, "\t--fd <num>, -f <num> - unix socket file descriptor"
>>> + " used to pass eBPF fds.\n");
>>> + fprintf(stderr, "\t--help, -h - this help.\n");
>>> + exit(exitcode);
>>> +}
>>> +
>>> +int main(int argc, char **argv)
>>> +{
>>> + char *fd_string = NULL;
>>> + int unix_fd = 0;
>>> + struct EBPFRSSContext ctx = {};
>>> + int fds[EBPF_RSS_MAX_FDS] = {};
>>> + int ret = -1;
>>> +
>>> + for (;;) {
>>> + int c;
>>> + static struct option long_options[] = {
>>> + {"help", no_argument, 0, 'h'},
>>> + {"fd", required_argument, 0, 'f'},
>>> + {0, 0, 0, 0}
>>> + };
>>> + c = getopt_long(argc, argv, "hf:",
>>> + long_options, NULL);
>>> +
>>> + if (c == -1) {
>>> + break;
>>> + }
>>> +
>>> + switch (c) {
>>> + case 'f':
>>> + fd_string = optarg;
>>> + break;
>>> + case 'h':
>>> + default:
>>> + print_help_and_exit(argv[0],
>>> + c == 'h' ? EXIT_SUCCESS : EXIT_FAILURE);
>>> + }
>>> + }
>>> +
>>> + if (!fd_string) {
>>> + fprintf(stderr, "Unix file descriptor not present.\n");
>>> + print_help_and_exit(argv[0], EXIT_FAILURE);
>>> + }
>>> +
>>> + unix_fd = atoi(fd_string);
>>> +
>>> + if (!unix_fd) {
>>> + fprintf(stderr, "Unix file descriptor is invalid.\n");
>>> + return EXIT_FAILURE;
>>> + }
>>> +
>>> + ebpf_rss_init(&ctx);
>>> + if (!ebpf_rss_load(&ctx)) {
>>> + fprintf(stderr, "Can't load ebpf.\n");
>>> + return EXIT_FAILURE;
>>> + }
>>> + fds[0] = ctx.program_fd;
>>> + fds[1] = ctx.map_configuration;
>>> +
>>> + ret = send_fds(unix_fd, fds, EBPF_RSS_MAX_FDS);
>>> + if (ret < 0) {
>>> + fprintf(stderr, "Issue while sending fds: %s.\n", strerror(errno));
>>> + }
>>> +
>>> + ebpf_rss_unload(&ctx);
>>> +
>>> + return ret < 0 ? EXIT_FAILURE : EXIT_SUCCESS;
>>> +}
>>> +
>>> diff --git a/meson.build b/meson.build
>>> index 257e51d91b..913aa1fee5 100644
>>> --- a/meson.build
>>> +++ b/meson.build
>>> @@ -1033,19 +1033,22 @@ if not get_option('fuse_lseek').disabled()
>>> endif
>>>
>>> # libbpf
>>> -libbpf = dependency('libbpf', required: get_option('bpf'), method: 'pkg-config')
>>> -if libbpf.found() and not cc.links('''
>>> - #include <bpf/libbpf.h>
>>> - int main(void)
>>> - {
>>> - bpf_object__destroy_skeleton(NULL);
>>> - return 0;
>>> - }''', dependencies: libbpf)
>>> - libbpf = not_found
>>> - if get_option('bpf').enabled()
>>> - error('libbpf skeleton test failed')
>>> - else
>>> - warning('libbpf skeleton test failed, disabling')
>>> +libbpf = not_found
>>> +if targetos == 'linux'
>>> + libbpf = dependency('libbpf', required: get_option('bpf'), method: 'pkg-config')
>>> + if libbpf.found() and not cc.links('''
>>> + #include <bpf/libbpf.h>
>>> + int main(void)
>>> + {
>>> + bpf_object__destroy_skeleton(NULL);
>>
>> Do we need to test whether the bpf can do mmap() here?
>>
>> Thanks
>>
>>
>>> + return 0;
>>> + }''', dependencies: libbpf)
>>> + libbpf = not_found
>>> + if get_option('bpf').enabled()
>>> + error('libbpf skeleton test failed')
>>> + else
>>> + warning('libbpf skeleton test failed, disabling')
>>> + endif
>>> endif
>>> endif
>>>
>>> @@ -2423,6 +2426,14 @@ if have_tools
>>> dependencies: [authz, crypto, io, qom, qemuutil,
>>> libcap_ng, mpathpersist],
>>> install: true)
>>> +
>>> + if libbpf.found()
>>> + executable('qemu-ebpf-rss-helper', files(
>>> + 'ebpf/qemu-ebpf-rss-helper.c', 'ebpf/ebpf_rss.c'),
>>> + dependencies: [qemuutil, libbpf, glib],
>>> + install: true,
>>> + install_dir: get_option('libexecdir'))
>>> + endif
>>> endif
>>>
>>> if 'CONFIG_IVSHMEM' in config_host
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 4/5] ebpf_rss_helper: Added helper for eBPF RSS.
2021-09-01 6:42 ` Jason Wang
@ 2021-09-06 15:50 ` Andrew Melnichenko
2021-09-07 3:22 ` Jason Wang
2021-09-07 10:40 ` Yuri Benditovich
1 sibling, 1 reply; 60+ messages in thread
From: Andrew Melnichenko @ 2021-09-06 15:50 UTC (permalink / raw)
To: Jason Wang
Cc: Daniel P. Berrangé,
Michael S . Tsirkin, qemu-devel, Markus Armbruster,
Yuri Benditovich, Yan Vugenfirer, Eric Blake
[-- Attachment #1: Type: text/plain, Size: 9568 bytes --]
Hi,
> I think it's for back-compatibility.
>
> E.g current codes works without mmap(), and user will surprise that it
> wont' work after upgrading their qemu.
>
Well, the current code would require additional capabilities with
"kernel.unprivileged_bpf_disabled=1", which may be possible on RedHat
systems.
Technically we may have mmap test which will show that mmap for
BPF_MAP_TYPE_ARRAY works, but on the target system, we will know it only in
runtime.
If I'm not mistaken, mmap for BPF_MAP_TYPE_ARRAY was added before kernel
5.4 and our bpf program requires kernel 5.8+.
So, there are no reasons to add bpf() update map as a fallback for mmap().
On Wed, Sep 1, 2021 at 9:42 AM Jason Wang <jasowang@redhat.com> wrote:
>
> 在 2021/8/31 上午1:07, Yuri Benditovich 写道:
> > On Fri, Aug 20, 2021 at 6:41 AM Jason Wang <jasowang@redhat.com> wrote:
> >>
> >> 在 2021/7/13 下午11:37, Andrew Melnychenko 写道:
> >>> Helper program. Loads eBPF RSS program and maps and passes them
> through unix socket.
> >>> Libvirt may launch this helper and pass eBPF fds to qemu virtio-net.
> >>
> >> I wonder if this can be done as helper for TAP/bridge.
> >>
> >> E.g it's the qemu to launch those helper with set-uid.
> >>
> >> Then libvirt won't even need to care about that?
> >>
> > There are pros and cons for such a solution with set-uid.
> > From my point of view one of the cons is that set-uid is efficient
> > only at install time so the coexistence of different qemu builds (and
> > different helpers for each one) is kind of problematic.
> > With the current solution this does not present any problem: the
> > developer can have several different builds, each one automatically
> > has its own helper and there is no conflict between these builds and
> > between these builds and installed qemu package. Changing the
> > 'emulator' in the libvirt profile automatically brings the proper
> > helper to work.
>
>
> I'm not sure I get you here. We can still have default/sample helper to
> make sure it works for different builds.
>
> If we can avoid the involvement of libvirt, that would be better.
>
> Thanks
>
>
> >
> >>> Also, libbpf dependency now exclusively for Linux.
> >>> Libbpf is used for eBPF RSS steering, which is supported only by Linux
> TAP.
> >>> There is no reason yet to build eBPF loader and helper for non Linux
> systems,
> >>> even if libbpf is present.
> >>>
> >>> Signed-off-by: Andrew Melnychenko <andrew@daynix.com>
> >>> ---
> >>> ebpf/qemu-ebpf-rss-helper.c | 130
> ++++++++++++++++++++++++++++++++++++
> >>> meson.build | 37 ++++++----
> >>> 2 files changed, 154 insertions(+), 13 deletions(-)
> >>> create mode 100644 ebpf/qemu-ebpf-rss-helper.c
> >>>
> >>> diff --git a/ebpf/qemu-ebpf-rss-helper.c b/ebpf/qemu-ebpf-rss-helper.c
> >>> new file mode 100644
> >>> index 0000000000..fe68758f57
> >>> --- /dev/null
> >>> +++ b/ebpf/qemu-ebpf-rss-helper.c
> >>> @@ -0,0 +1,130 @@
> >>> +/*
> >>> + * eBPF RSS Helper
> >>> + *
> >>> + * Developed by Daynix Computing LTD (http://www.daynix.com)
> >>> + *
> >>> + * Authors:
> >>> + * Andrew Melnychenko <andrew@daynix.com>
> >>> + *
> >>> + * This work is licensed under the terms of the GNU GPL, version 2.
> See
> >>> + * the COPYING file in the top-level directory.
> >>> + *
> >>> + * Description: This is helper program for libvirtd.
> >>> + * It loads eBPF RSS program and passes fds through unix
> socket.
> >>> + * Built by meson, target - 'qemu-ebpf-rss-helper'.
> >>> + */
> >>> +
> >>> +#include <stdio.h>
> >>> +#include <stdint.h>
> >>> +#include <stdlib.h>
> >>> +#include <stdbool.h>
> >>> +#include <getopt.h>
> >>> +#include <memory.h>
> >>> +#include <errno.h>
> >>> +#include <sys/socket.h>
> >>> +
> >>> +#include "ebpf_rss.h"
> >>> +
> >>> +#include "qemu-helper-stamp.h"
> >>> +
> >>> +void QEMU_HELPER_STAMP(void) {}
> >>> +
> >>> +static int send_fds(int socket, int *fds, int n)
> >>> +{
> >>> + struct msghdr msg = {};
> >>> + struct cmsghdr *cmsg = NULL;
> >>> + char buf[CMSG_SPACE(n * sizeof(int))];
> >>> + char dummy_buffer = 0;
> >>> + struct iovec io = { .iov_base = &dummy_buffer,
> >>> + .iov_len = sizeof(dummy_buffer) };
> >>> +
> >>> + memset(buf, 0, sizeof(buf));
> >>> +
> >>> + msg.msg_iov = &io;
> >>> + msg.msg_iovlen = 1;
> >>> + msg.msg_control = buf;
> >>> + msg.msg_controllen = sizeof(buf);
> >>> +
> >>> + cmsg = CMSG_FIRSTHDR(&msg);
> >>> + cmsg->cmsg_level = SOL_SOCKET;
> >>> + cmsg->cmsg_type = SCM_RIGHTS;
> >>> + cmsg->cmsg_len = CMSG_LEN(n * sizeof(int));
> >>> +
> >>> + memcpy(CMSG_DATA(cmsg), fds, n * sizeof(int));
> >>> +
> >>> + return sendmsg(socket, &msg, 0);
> >>> +}
> >>> +
> >>> +static void print_help_and_exit(const char *prog, int exitcode)
> >>> +{
> >>> + fprintf(stderr, "%s - load eBPF RSS program for qemu and pass
> eBPF fds"
> >>> + " through unix socket.\n", prog);
> >>> + fprintf(stderr, "\t--fd <num>, -f <num> - unix socket file
> descriptor"
> >>> + " used to pass eBPF fds.\n");
> >>> + fprintf(stderr, "\t--help, -h - this help.\n");
> >>> + exit(exitcode);
> >>> +}
> >>> +
> >>> +int main(int argc, char **argv)
> >>> +{
> >>> + char *fd_string = NULL;
> >>> + int unix_fd = 0;
> >>> + struct EBPFRSSContext ctx = {};
> >>> + int fds[EBPF_RSS_MAX_FDS] = {};
> >>> + int ret = -1;
> >>> +
> >>> + for (;;) {
> >>> + int c;
> >>> + static struct option long_options[] = {
> >>> + {"help", no_argument, 0, 'h'},
> >>> + {"fd", required_argument, 0, 'f'},
> >>> + {0, 0, 0, 0}
> >>> + };
> >>> + c = getopt_long(argc, argv, "hf:",
> >>> + long_options, NULL);
> >>> +
> >>> + if (c == -1) {
> >>> + break;
> >>> + }
> >>> +
> >>> + switch (c) {
> >>> + case 'f':
> >>> + fd_string = optarg;
> >>> + break;
> >>> + case 'h':
> >>> + default:
> >>> + print_help_and_exit(argv[0],
> >>> + c == 'h' ? EXIT_SUCCESS : EXIT_FAILURE);
> >>> + }
> >>> + }
> >>> +
> >>> + if (!fd_string) {
> >>> + fprintf(stderr, "Unix file descriptor not present.\n");
> >>> + print_help_and_exit(argv[0], EXIT_FAILURE);
> >>> + }
> >>> +
> >>> + unix_fd = atoi(fd_string);
> >>> +
> >>> + if (!unix_fd) {
> >>> + fprintf(stderr, "Unix file descriptor is invalid.\n");
> >>> + return EXIT_FAILURE;
> >>> + }
> >>> +
> >>> + ebpf_rss_init(&ctx);
> >>> + if (!ebpf_rss_load(&ctx)) {
> >>> + fprintf(stderr, "Can't load ebpf.\n");
> >>> + return EXIT_FAILURE;
> >>> + }
> >>> + fds[0] = ctx.program_fd;
> >>> + fds[1] = ctx.map_configuration;
> >>> +
> >>> + ret = send_fds(unix_fd, fds, EBPF_RSS_MAX_FDS);
> >>> + if (ret < 0) {
> >>> + fprintf(stderr, "Issue while sending fds: %s.\n",
> strerror(errno));
> >>> + }
> >>> +
> >>> + ebpf_rss_unload(&ctx);
> >>> +
> >>> + return ret < 0 ? EXIT_FAILURE : EXIT_SUCCESS;
> >>> +}
> >>> +
> >>> diff --git a/meson.build b/meson.build
> >>> index 257e51d91b..913aa1fee5 100644
> >>> --- a/meson.build
> >>> +++ b/meson.build
> >>> @@ -1033,19 +1033,22 @@ if not get_option('fuse_lseek').disabled()
> >>> endif
> >>>
> >>> # libbpf
> >>> -libbpf = dependency('libbpf', required: get_option('bpf'), method:
> 'pkg-config')
> >>> -if libbpf.found() and not cc.links('''
> >>> - #include <bpf/libbpf.h>
> >>> - int main(void)
> >>> - {
> >>> - bpf_object__destroy_skeleton(NULL);
> >>> - return 0;
> >>> - }''', dependencies: libbpf)
> >>> - libbpf = not_found
> >>> - if get_option('bpf').enabled()
> >>> - error('libbpf skeleton test failed')
> >>> - else
> >>> - warning('libbpf skeleton test failed, disabling')
> >>> +libbpf = not_found
> >>> +if targetos == 'linux'
> >>> + libbpf = dependency('libbpf', required: get_option('bpf'), method:
> 'pkg-config')
> >>> + if libbpf.found() and not cc.links('''
> >>> + #include <bpf/libbpf.h>
> >>> + int main(void)
> >>> + {
> >>> + bpf_object__destroy_skeleton(NULL);
> >>
> >> Do we need to test whether the bpf can do mmap() here?
> >>
> >> Thanks
> >>
> >>
> >>> + return 0;
> >>> + }''', dependencies: libbpf)
> >>> + libbpf = not_found
> >>> + if get_option('bpf').enabled()
> >>> + error('libbpf skeleton test failed')
> >>> + else
> >>> + warning('libbpf skeleton test failed, disabling')
> >>> + endif
> >>> endif
> >>> endif
> >>>
> >>> @@ -2423,6 +2426,14 @@ if have_tools
> >>> dependencies: [authz, crypto, io, qom, qemuutil,
> >>> libcap_ng, mpathpersist],
> >>> install: true)
> >>> +
> >>> + if libbpf.found()
> >>> + executable('qemu-ebpf-rss-helper', files(
> >>> + 'ebpf/qemu-ebpf-rss-helper.c', 'ebpf/ebpf_rss.c'),
> >>> + dependencies: [qemuutil, libbpf, glib],
> >>> + install: true,
> >>> + install_dir: get_option('libexecdir'))
> >>> + endif
> >>> endif
> >>>
> >>> if 'CONFIG_IVSHMEM' in config_host
>
>
[-- Attachment #2: Type: text/html, Size: 13705 bytes --]
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 4/5] ebpf_rss_helper: Added helper for eBPF RSS.
2021-09-06 15:50 ` Andrew Melnichenko
@ 2021-09-07 3:22 ` Jason Wang
0 siblings, 0 replies; 60+ messages in thread
From: Jason Wang @ 2021-09-07 3:22 UTC (permalink / raw)
To: Andrew Melnichenko
Cc: Daniel P. Berrangé,
Michael S . Tsirkin, qemu-devel, Markus Armbruster,
Yuri Benditovich, Yan Vugenfirer, Eric Blake
On Mon, Sep 6, 2021 at 11:50 PM Andrew Melnichenko <andrew@daynix.com> wrote:
>
> Hi,
>>
>> I think it's for back-compatibility.
>>
>> E.g current codes works without mmap(), and user will surprise that it
>> wont' work after upgrading their qemu.
>
> Well, the current code would require additional capabilities with "kernel.unprivileged_bpf_disabled=1", which may be possible on RedHat systems.
> Technically we may have mmap test which will show that mmap for BPF_MAP_TYPE_ARRAY works, but on the target system, we will know it only in runtime.
> If I'm not mistaken, mmap for BPF_MAP_TYPE_ARRAY was added before kernel 5.4 and our bpf program requires kernel 5.8+.
Ok, if this is the case, please explain this in the commit log.
Btw, any reason that 5.8 is required for our bpf program?
Thanks
> So, there are no reasons to add bpf() update map as a fallback for mmap().
>
> On Wed, Sep 1, 2021 at 9:42 AM Jason Wang <jasowang@redhat.com> wrote:
>>
>>
>> 在 2021/8/31 上午1:07, Yuri Benditovich 写道:
>> > On Fri, Aug 20, 2021 at 6:41 AM Jason Wang <jasowang@redhat.com> wrote:
>> >>
>> >> 在 2021/7/13 下午11:37, Andrew Melnychenko 写道:
>> >>> Helper program. Loads eBPF RSS program and maps and passes them through unix socket.
>> >>> Libvirt may launch this helper and pass eBPF fds to qemu virtio-net.
>> >>
>> >> I wonder if this can be done as helper for TAP/bridge.
>> >>
>> >> E.g it's the qemu to launch those helper with set-uid.
>> >>
>> >> Then libvirt won't even need to care about that?
>> >>
>> > There are pros and cons for such a solution with set-uid.
>> > From my point of view one of the cons is that set-uid is efficient
>> > only at install time so the coexistence of different qemu builds (and
>> > different helpers for each one) is kind of problematic.
>> > With the current solution this does not present any problem: the
>> > developer can have several different builds, each one automatically
>> > has its own helper and there is no conflict between these builds and
>> > between these builds and installed qemu package. Changing the
>> > 'emulator' in the libvirt profile automatically brings the proper
>> > helper to work.
>>
>>
>> I'm not sure I get you here. We can still have default/sample helper to
>> make sure it works for different builds.
>>
>> If we can avoid the involvement of libvirt, that would be better.
>>
>> Thanks
>>
>>
>> >
>> >>> Also, libbpf dependency now exclusively for Linux.
>> >>> Libbpf is used for eBPF RSS steering, which is supported only by Linux TAP.
>> >>> There is no reason yet to build eBPF loader and helper for non Linux systems,
>> >>> even if libbpf is present.
>> >>>
>> >>> Signed-off-by: Andrew Melnychenko <andrew@daynix.com>
>> >>> ---
>> >>> ebpf/qemu-ebpf-rss-helper.c | 130 ++++++++++++++++++++++++++++++++++++
>> >>> meson.build | 37 ++++++----
>> >>> 2 files changed, 154 insertions(+), 13 deletions(-)
>> >>> create mode 100644 ebpf/qemu-ebpf-rss-helper.c
>> >>>
>> >>> diff --git a/ebpf/qemu-ebpf-rss-helper.c b/ebpf/qemu-ebpf-rss-helper.c
>> >>> new file mode 100644
>> >>> index 0000000000..fe68758f57
>> >>> --- /dev/null
>> >>> +++ b/ebpf/qemu-ebpf-rss-helper.c
>> >>> @@ -0,0 +1,130 @@
>> >>> +/*
>> >>> + * eBPF RSS Helper
>> >>> + *
>> >>> + * Developed by Daynix Computing LTD (http://www.daynix.com)
>> >>> + *
>> >>> + * Authors:
>> >>> + * Andrew Melnychenko <andrew@daynix.com>
>> >>> + *
>> >>> + * This work is licensed under the terms of the GNU GPL, version 2. See
>> >>> + * the COPYING file in the top-level directory.
>> >>> + *
>> >>> + * Description: This is helper program for libvirtd.
>> >>> + * It loads eBPF RSS program and passes fds through unix socket.
>> >>> + * Built by meson, target - 'qemu-ebpf-rss-helper'.
>> >>> + */
>> >>> +
>> >>> +#include <stdio.h>
>> >>> +#include <stdint.h>
>> >>> +#include <stdlib.h>
>> >>> +#include <stdbool.h>
>> >>> +#include <getopt.h>
>> >>> +#include <memory.h>
>> >>> +#include <errno.h>
>> >>> +#include <sys/socket.h>
>> >>> +
>> >>> +#include "ebpf_rss.h"
>> >>> +
>> >>> +#include "qemu-helper-stamp.h"
>> >>> +
>> >>> +void QEMU_HELPER_STAMP(void) {}
>> >>> +
>> >>> +static int send_fds(int socket, int *fds, int n)
>> >>> +{
>> >>> + struct msghdr msg = {};
>> >>> + struct cmsghdr *cmsg = NULL;
>> >>> + char buf[CMSG_SPACE(n * sizeof(int))];
>> >>> + char dummy_buffer = 0;
>> >>> + struct iovec io = { .iov_base = &dummy_buffer,
>> >>> + .iov_len = sizeof(dummy_buffer) };
>> >>> +
>> >>> + memset(buf, 0, sizeof(buf));
>> >>> +
>> >>> + msg.msg_iov = &io;
>> >>> + msg.msg_iovlen = 1;
>> >>> + msg.msg_control = buf;
>> >>> + msg.msg_controllen = sizeof(buf);
>> >>> +
>> >>> + cmsg = CMSG_FIRSTHDR(&msg);
>> >>> + cmsg->cmsg_level = SOL_SOCKET;
>> >>> + cmsg->cmsg_type = SCM_RIGHTS;
>> >>> + cmsg->cmsg_len = CMSG_LEN(n * sizeof(int));
>> >>> +
>> >>> + memcpy(CMSG_DATA(cmsg), fds, n * sizeof(int));
>> >>> +
>> >>> + return sendmsg(socket, &msg, 0);
>> >>> +}
>> >>> +
>> >>> +static void print_help_and_exit(const char *prog, int exitcode)
>> >>> +{
>> >>> + fprintf(stderr, "%s - load eBPF RSS program for qemu and pass eBPF fds"
>> >>> + " through unix socket.\n", prog);
>> >>> + fprintf(stderr, "\t--fd <num>, -f <num> - unix socket file descriptor"
>> >>> + " used to pass eBPF fds.\n");
>> >>> + fprintf(stderr, "\t--help, -h - this help.\n");
>> >>> + exit(exitcode);
>> >>> +}
>> >>> +
>> >>> +int main(int argc, char **argv)
>> >>> +{
>> >>> + char *fd_string = NULL;
>> >>> + int unix_fd = 0;
>> >>> + struct EBPFRSSContext ctx = {};
>> >>> + int fds[EBPF_RSS_MAX_FDS] = {};
>> >>> + int ret = -1;
>> >>> +
>> >>> + for (;;) {
>> >>> + int c;
>> >>> + static struct option long_options[] = {
>> >>> + {"help", no_argument, 0, 'h'},
>> >>> + {"fd", required_argument, 0, 'f'},
>> >>> + {0, 0, 0, 0}
>> >>> + };
>> >>> + c = getopt_long(argc, argv, "hf:",
>> >>> + long_options, NULL);
>> >>> +
>> >>> + if (c == -1) {
>> >>> + break;
>> >>> + }
>> >>> +
>> >>> + switch (c) {
>> >>> + case 'f':
>> >>> + fd_string = optarg;
>> >>> + break;
>> >>> + case 'h':
>> >>> + default:
>> >>> + print_help_and_exit(argv[0],
>> >>> + c == 'h' ? EXIT_SUCCESS : EXIT_FAILURE);
>> >>> + }
>> >>> + }
>> >>> +
>> >>> + if (!fd_string) {
>> >>> + fprintf(stderr, "Unix file descriptor not present.\n");
>> >>> + print_help_and_exit(argv[0], EXIT_FAILURE);
>> >>> + }
>> >>> +
>> >>> + unix_fd = atoi(fd_string);
>> >>> +
>> >>> + if (!unix_fd) {
>> >>> + fprintf(stderr, "Unix file descriptor is invalid.\n");
>> >>> + return EXIT_FAILURE;
>> >>> + }
>> >>> +
>> >>> + ebpf_rss_init(&ctx);
>> >>> + if (!ebpf_rss_load(&ctx)) {
>> >>> + fprintf(stderr, "Can't load ebpf.\n");
>> >>> + return EXIT_FAILURE;
>> >>> + }
>> >>> + fds[0] = ctx.program_fd;
>> >>> + fds[1] = ctx.map_configuration;
>> >>> +
>> >>> + ret = send_fds(unix_fd, fds, EBPF_RSS_MAX_FDS);
>> >>> + if (ret < 0) {
>> >>> + fprintf(stderr, "Issue while sending fds: %s.\n", strerror(errno));
>> >>> + }
>> >>> +
>> >>> + ebpf_rss_unload(&ctx);
>> >>> +
>> >>> + return ret < 0 ? EXIT_FAILURE : EXIT_SUCCESS;
>> >>> +}
>> >>> +
>> >>> diff --git a/meson.build b/meson.build
>> >>> index 257e51d91b..913aa1fee5 100644
>> >>> --- a/meson.build
>> >>> +++ b/meson.build
>> >>> @@ -1033,19 +1033,22 @@ if not get_option('fuse_lseek').disabled()
>> >>> endif
>> >>>
>> >>> # libbpf
>> >>> -libbpf = dependency('libbpf', required: get_option('bpf'), method: 'pkg-config')
>> >>> -if libbpf.found() and not cc.links('''
>> >>> - #include <bpf/libbpf.h>
>> >>> - int main(void)
>> >>> - {
>> >>> - bpf_object__destroy_skeleton(NULL);
>> >>> - return 0;
>> >>> - }''', dependencies: libbpf)
>> >>> - libbpf = not_found
>> >>> - if get_option('bpf').enabled()
>> >>> - error('libbpf skeleton test failed')
>> >>> - else
>> >>> - warning('libbpf skeleton test failed, disabling')
>> >>> +libbpf = not_found
>> >>> +if targetos == 'linux'
>> >>> + libbpf = dependency('libbpf', required: get_option('bpf'), method: 'pkg-config')
>> >>> + if libbpf.found() and not cc.links('''
>> >>> + #include <bpf/libbpf.h>
>> >>> + int main(void)
>> >>> + {
>> >>> + bpf_object__destroy_skeleton(NULL);
>> >>
>> >> Do we need to test whether the bpf can do mmap() here?
>> >>
>> >> Thanks
>> >>
>> >>
>> >>> + return 0;
>> >>> + }''', dependencies: libbpf)
>> >>> + libbpf = not_found
>> >>> + if get_option('bpf').enabled()
>> >>> + error('libbpf skeleton test failed')
>> >>> + else
>> >>> + warning('libbpf skeleton test failed, disabling')
>> >>> + endif
>> >>> endif
>> >>> endif
>> >>>
>> >>> @@ -2423,6 +2426,14 @@ if have_tools
>> >>> dependencies: [authz, crypto, io, qom, qemuutil,
>> >>> libcap_ng, mpathpersist],
>> >>> install: true)
>> >>> +
>> >>> + if libbpf.found()
>> >>> + executable('qemu-ebpf-rss-helper', files(
>> >>> + 'ebpf/qemu-ebpf-rss-helper.c', 'ebpf/ebpf_rss.c'),
>> >>> + dependencies: [qemuutil, libbpf, glib],
>> >>> + install: true,
>> >>> + install_dir: get_option('libexecdir'))
>> >>> + endif
>> >>> endif
>> >>>
>> >>> if 'CONFIG_IVSHMEM' in config_host
>>
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 4/5] ebpf_rss_helper: Added helper for eBPF RSS.
2021-09-01 6:42 ` Jason Wang
2021-09-06 15:50 ` Andrew Melnichenko
@ 2021-09-07 10:40 ` Yuri Benditovich
2021-09-08 3:45 ` Jason Wang
1 sibling, 1 reply; 60+ messages in thread
From: Yuri Benditovich @ 2021-09-07 10:40 UTC (permalink / raw)
To: Jason Wang
Cc: Andrew Melnychenko, Daniel P. Berrangé,
Michael S . Tsirkin, qemu-devel, Markus Armbruster,
Yan Vugenfirer, Eric Blake
On Wed, Sep 1, 2021 at 9:42 AM Jason Wang <jasowang@redhat.com> wrote:
>
>
> 在 2021/8/31 上午1:07, Yuri Benditovich 写道:
> > On Fri, Aug 20, 2021 at 6:41 AM Jason Wang <jasowang@redhat.com> wrote:
> >>
> >> 在 2021/7/13 下午11:37, Andrew Melnychenko 写道:
> >>> Helper program. Loads eBPF RSS program and maps and passes them through unix socket.
> >>> Libvirt may launch this helper and pass eBPF fds to qemu virtio-net.
> >>
> >> I wonder if this can be done as helper for TAP/bridge.
> >>
> >> E.g it's the qemu to launch those helper with set-uid.
> >>
> >> Then libvirt won't even need to care about that?
> >>
> > There are pros and cons for such a solution with set-uid.
> > From my point of view one of the cons is that set-uid is efficient
> > only at install time so the coexistence of different qemu builds (and
> > different helpers for each one) is kind of problematic.
> > With the current solution this does not present any problem: the
> > developer can have several different builds, each one automatically
> > has its own helper and there is no conflict between these builds and
> > between these builds and installed qemu package. Changing the
> > 'emulator' in the libvirt profile automatically brings the proper
> > helper to work.
>
>
> I'm not sure I get you here. We can still have default/sample helper to
> make sure it works for different builds.
>
> If we can avoid the involvement of libvirt, that would be better.
Hi Jason,
Indeed I did not get the idea, can you please explain it in more
details (as detailed as possible to avoid future misunderstanding),
especially how exactly we can use the set-uid and what is the 'default' helper.
We also would prefer to do everything from qemu but we do not see how
we can do that.
Our main points (what should be addressed):
- qemu should be able to load ebpf and use the maps when it runs from
libvirt (without special caps) and standalone (with caps)
- it is possible that there are different qemu builds on the machine,
one of them might be installed, their ebpf's might be different and
the interface between qemu and ebpf (exact content of maps and number
of maps)
- qemu configures the RSS dynamically according to the commands
provided by the guest
Thanks in advance
Yuri
>
> Thanks
>
>
> >
> >>> Also, libbpf dependency now exclusively for Linux.
> >>> Libbpf is used for eBPF RSS steering, which is supported only by Linux TAP.
> >>> There is no reason yet to build eBPF loader and helper for non Linux systems,
> >>> even if libbpf is present.
> >>>
> >>> Signed-off-by: Andrew Melnychenko <andrew@daynix.com>
> >>> ---
> >>> ebpf/qemu-ebpf-rss-helper.c | 130 ++++++++++++++++++++++++++++++++++++
> >>> meson.build | 37 ++++++----
> >>> 2 files changed, 154 insertions(+), 13 deletions(-)
> >>> create mode 100644 ebpf/qemu-ebpf-rss-helper.c
> >>>
> >>> diff --git a/ebpf/qemu-ebpf-rss-helper.c b/ebpf/qemu-ebpf-rss-helper.c
> >>> new file mode 100644
> >>> index 0000000000..fe68758f57
> >>> --- /dev/null
> >>> +++ b/ebpf/qemu-ebpf-rss-helper.c
> >>> @@ -0,0 +1,130 @@
> >>> +/*
> >>> + * eBPF RSS Helper
> >>> + *
> >>> + * Developed by Daynix Computing LTD (http://www.daynix.com)
> >>> + *
> >>> + * Authors:
> >>> + * Andrew Melnychenko <andrew@daynix.com>
> >>> + *
> >>> + * This work is licensed under the terms of the GNU GPL, version 2. See
> >>> + * the COPYING file in the top-level directory.
> >>> + *
> >>> + * Description: This is helper program for libvirtd.
> >>> + * It loads eBPF RSS program and passes fds through unix socket.
> >>> + * Built by meson, target - 'qemu-ebpf-rss-helper'.
> >>> + */
> >>> +
> >>> +#include <stdio.h>
> >>> +#include <stdint.h>
> >>> +#include <stdlib.h>
> >>> +#include <stdbool.h>
> >>> +#include <getopt.h>
> >>> +#include <memory.h>
> >>> +#include <errno.h>
> >>> +#include <sys/socket.h>
> >>> +
> >>> +#include "ebpf_rss.h"
> >>> +
> >>> +#include "qemu-helper-stamp.h"
> >>> +
> >>> +void QEMU_HELPER_STAMP(void) {}
> >>> +
> >>> +static int send_fds(int socket, int *fds, int n)
> >>> +{
> >>> + struct msghdr msg = {};
> >>> + struct cmsghdr *cmsg = NULL;
> >>> + char buf[CMSG_SPACE(n * sizeof(int))];
> >>> + char dummy_buffer = 0;
> >>> + struct iovec io = { .iov_base = &dummy_buffer,
> >>> + .iov_len = sizeof(dummy_buffer) };
> >>> +
> >>> + memset(buf, 0, sizeof(buf));
> >>> +
> >>> + msg.msg_iov = &io;
> >>> + msg.msg_iovlen = 1;
> >>> + msg.msg_control = buf;
> >>> + msg.msg_controllen = sizeof(buf);
> >>> +
> >>> + cmsg = CMSG_FIRSTHDR(&msg);
> >>> + cmsg->cmsg_level = SOL_SOCKET;
> >>> + cmsg->cmsg_type = SCM_RIGHTS;
> >>> + cmsg->cmsg_len = CMSG_LEN(n * sizeof(int));
> >>> +
> >>> + memcpy(CMSG_DATA(cmsg), fds, n * sizeof(int));
> >>> +
> >>> + return sendmsg(socket, &msg, 0);
> >>> +}
> >>> +
> >>> +static void print_help_and_exit(const char *prog, int exitcode)
> >>> +{
> >>> + fprintf(stderr, "%s - load eBPF RSS program for qemu and pass eBPF fds"
> >>> + " through unix socket.\n", prog);
> >>> + fprintf(stderr, "\t--fd <num>, -f <num> - unix socket file descriptor"
> >>> + " used to pass eBPF fds.\n");
> >>> + fprintf(stderr, "\t--help, -h - this help.\n");
> >>> + exit(exitcode);
> >>> +}
> >>> +
> >>> +int main(int argc, char **argv)
> >>> +{
> >>> + char *fd_string = NULL;
> >>> + int unix_fd = 0;
> >>> + struct EBPFRSSContext ctx = {};
> >>> + int fds[EBPF_RSS_MAX_FDS] = {};
> >>> + int ret = -1;
> >>> +
> >>> + for (;;) {
> >>> + int c;
> >>> + static struct option long_options[] = {
> >>> + {"help", no_argument, 0, 'h'},
> >>> + {"fd", required_argument, 0, 'f'},
> >>> + {0, 0, 0, 0}
> >>> + };
> >>> + c = getopt_long(argc, argv, "hf:",
> >>> + long_options, NULL);
> >>> +
> >>> + if (c == -1) {
> >>> + break;
> >>> + }
> >>> +
> >>> + switch (c) {
> >>> + case 'f':
> >>> + fd_string = optarg;
> >>> + break;
> >>> + case 'h':
> >>> + default:
> >>> + print_help_and_exit(argv[0],
> >>> + c == 'h' ? EXIT_SUCCESS : EXIT_FAILURE);
> >>> + }
> >>> + }
> >>> +
> >>> + if (!fd_string) {
> >>> + fprintf(stderr, "Unix file descriptor not present.\n");
> >>> + print_help_and_exit(argv[0], EXIT_FAILURE);
> >>> + }
> >>> +
> >>> + unix_fd = atoi(fd_string);
> >>> +
> >>> + if (!unix_fd) {
> >>> + fprintf(stderr, "Unix file descriptor is invalid.\n");
> >>> + return EXIT_FAILURE;
> >>> + }
> >>> +
> >>> + ebpf_rss_init(&ctx);
> >>> + if (!ebpf_rss_load(&ctx)) {
> >>> + fprintf(stderr, "Can't load ebpf.\n");
> >>> + return EXIT_FAILURE;
> >>> + }
> >>> + fds[0] = ctx.program_fd;
> >>> + fds[1] = ctx.map_configuration;
> >>> +
> >>> + ret = send_fds(unix_fd, fds, EBPF_RSS_MAX_FDS);
> >>> + if (ret < 0) {
> >>> + fprintf(stderr, "Issue while sending fds: %s.\n", strerror(errno));
> >>> + }
> >>> +
> >>> + ebpf_rss_unload(&ctx);
> >>> +
> >>> + return ret < 0 ? EXIT_FAILURE : EXIT_SUCCESS;
> >>> +}
> >>> +
> >>> diff --git a/meson.build b/meson.build
> >>> index 257e51d91b..913aa1fee5 100644
> >>> --- a/meson.build
> >>> +++ b/meson.build
> >>> @@ -1033,19 +1033,22 @@ if not get_option('fuse_lseek').disabled()
> >>> endif
> >>>
> >>> # libbpf
> >>> -libbpf = dependency('libbpf', required: get_option('bpf'), method: 'pkg-config')
> >>> -if libbpf.found() and not cc.links('''
> >>> - #include <bpf/libbpf.h>
> >>> - int main(void)
> >>> - {
> >>> - bpf_object__destroy_skeleton(NULL);
> >>> - return 0;
> >>> - }''', dependencies: libbpf)
> >>> - libbpf = not_found
> >>> - if get_option('bpf').enabled()
> >>> - error('libbpf skeleton test failed')
> >>> - else
> >>> - warning('libbpf skeleton test failed, disabling')
> >>> +libbpf = not_found
> >>> +if targetos == 'linux'
> >>> + libbpf = dependency('libbpf', required: get_option('bpf'), method: 'pkg-config')
> >>> + if libbpf.found() and not cc.links('''
> >>> + #include <bpf/libbpf.h>
> >>> + int main(void)
> >>> + {
> >>> + bpf_object__destroy_skeleton(NULL);
> >>
> >> Do we need to test whether the bpf can do mmap() here?
> >>
> >> Thanks
> >>
> >>
> >>> + return 0;
> >>> + }''', dependencies: libbpf)
> >>> + libbpf = not_found
> >>> + if get_option('bpf').enabled()
> >>> + error('libbpf skeleton test failed')
> >>> + else
> >>> + warning('libbpf skeleton test failed, disabling')
> >>> + endif
> >>> endif
> >>> endif
> >>>
> >>> @@ -2423,6 +2426,14 @@ if have_tools
> >>> dependencies: [authz, crypto, io, qom, qemuutil,
> >>> libcap_ng, mpathpersist],
> >>> install: true)
> >>> +
> >>> + if libbpf.found()
> >>> + executable('qemu-ebpf-rss-helper', files(
> >>> + 'ebpf/qemu-ebpf-rss-helper.c', 'ebpf/ebpf_rss.c'),
> >>> + dependencies: [qemuutil, libbpf, glib],
> >>> + install: true,
> >>> + install_dir: get_option('libexecdir'))
> >>> + endif
> >>> endif
> >>>
> >>> if 'CONFIG_IVSHMEM' in config_host
>
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 4/5] ebpf_rss_helper: Added helper for eBPF RSS.
2021-09-07 10:40 ` Yuri Benditovich
@ 2021-09-08 3:45 ` Jason Wang
2021-09-09 0:00 ` Yuri Benditovich
0 siblings, 1 reply; 60+ messages in thread
From: Jason Wang @ 2021-09-08 3:45 UTC (permalink / raw)
To: Yuri Benditovich
Cc: Andrew Melnychenko, Daniel P. Berrangé,
Michael S . Tsirkin, qemu-devel, Markus Armbruster,
Yan Vugenfirer, Eric Blake
On Tue, Sep 7, 2021 at 6:40 PM Yuri Benditovich
<yuri.benditovich@daynix.com> wrote:
>
> On Wed, Sep 1, 2021 at 9:42 AM Jason Wang <jasowang@redhat.com> wrote:
> >
> >
> > 在 2021/8/31 上午1:07, Yuri Benditovich 写道:
> > > On Fri, Aug 20, 2021 at 6:41 AM Jason Wang <jasowang@redhat.com> wrote:
> > >>
> > >> 在 2021/7/13 下午11:37, Andrew Melnychenko 写道:
> > >>> Helper program. Loads eBPF RSS program and maps and passes them through unix socket.
> > >>> Libvirt may launch this helper and pass eBPF fds to qemu virtio-net.
> > >>
> > >> I wonder if this can be done as helper for TAP/bridge.
> > >>
> > >> E.g it's the qemu to launch those helper with set-uid.
> > >>
> > >> Then libvirt won't even need to care about that?
> > >>
> > > There are pros and cons for such a solution with set-uid.
> > > From my point of view one of the cons is that set-uid is efficient
> > > only at install time so the coexistence of different qemu builds (and
> > > different helpers for each one) is kind of problematic.
> > > With the current solution this does not present any problem: the
> > > developer can have several different builds, each one automatically
> > > has its own helper and there is no conflict between these builds and
> > > between these builds and installed qemu package. Changing the
> > > 'emulator' in the libvirt profile automatically brings the proper
> > > helper to work.
> >
> >
> > I'm not sure I get you here. We can still have default/sample helper to
> > make sure it works for different builds.
> >
> > If we can avoid the involvement of libvirt, that would be better.
>
> Hi Jason,
>
> Indeed I did not get the idea, can you please explain it in more
> details (as detailed as possible to avoid future misunderstanding),
> especially how exactly we can use the set-uid and what is the 'default' helper.
> We also would prefer to do everything from qemu but we do not see how
> we can do that.
Something like:
1) -netdev tap,rss_helper=/path/to/name
2) having a sample/default helper implemented in Qemu
3) we can introduce something special path like "default", then if
-netdev tap,rss_helper="default" is specified, qemu will use the
sample helper
So we have:
1) set set-uid for the helper
2) libvirt may just choose to launch the default helper
>
> Our main points (what should be addressed):
> - qemu should be able to load ebpf and use the maps when it runs from
> libvirt (without special caps) and standalone (with caps)
This is solved by leaving the privileged operations to the helper with set-uid.
> - it is possible that there are different qemu builds on the machine,
> one of them might be installed, their ebpf's might be different and
> the interface between qemu and ebpf (exact content of maps and number
> of maps)
We can use different helpers in this way.
> - qemu configures the RSS dynamically according to the commands
> provided by the guest
Consider we decided to use mmap() based maps, this is not an issue.
Or am I missing something?
Thanks
>
> Thanks in advance
> Yuri
>
> >
> > Thanks
> >
> >
> > >
> > >>> Also, libbpf dependency now exclusively for Linux.
> > >>> Libbpf is used for eBPF RSS steering, which is supported only by Linux TAP.
> > >>> There is no reason yet to build eBPF loader and helper for non Linux systems,
> > >>> even if libbpf is present.
> > >>>
> > >>> Signed-off-by: Andrew Melnychenko <andrew@daynix.com>
> > >>> ---
> > >>> ebpf/qemu-ebpf-rss-helper.c | 130 ++++++++++++++++++++++++++++++++++++
> > >>> meson.build | 37 ++++++----
> > >>> 2 files changed, 154 insertions(+), 13 deletions(-)
> > >>> create mode 100644 ebpf/qemu-ebpf-rss-helper.c
> > >>>
> > >>> diff --git a/ebpf/qemu-ebpf-rss-helper.c b/ebpf/qemu-ebpf-rss-helper.c
> > >>> new file mode 100644
> > >>> index 0000000000..fe68758f57
> > >>> --- /dev/null
> > >>> +++ b/ebpf/qemu-ebpf-rss-helper.c
> > >>> @@ -0,0 +1,130 @@
> > >>> +/*
> > >>> + * eBPF RSS Helper
> > >>> + *
> > >>> + * Developed by Daynix Computing LTD (http://www.daynix.com)
> > >>> + *
> > >>> + * Authors:
> > >>> + * Andrew Melnychenko <andrew@daynix.com>
> > >>> + *
> > >>> + * This work is licensed under the terms of the GNU GPL, version 2. See
> > >>> + * the COPYING file in the top-level directory.
> > >>> + *
> > >>> + * Description: This is helper program for libvirtd.
> > >>> + * It loads eBPF RSS program and passes fds through unix socket.
> > >>> + * Built by meson, target - 'qemu-ebpf-rss-helper'.
> > >>> + */
> > >>> +
> > >>> +#include <stdio.h>
> > >>> +#include <stdint.h>
> > >>> +#include <stdlib.h>
> > >>> +#include <stdbool.h>
> > >>> +#include <getopt.h>
> > >>> +#include <memory.h>
> > >>> +#include <errno.h>
> > >>> +#include <sys/socket.h>
> > >>> +
> > >>> +#include "ebpf_rss.h"
> > >>> +
> > >>> +#include "qemu-helper-stamp.h"
> > >>> +
> > >>> +void QEMU_HELPER_STAMP(void) {}
> > >>> +
> > >>> +static int send_fds(int socket, int *fds, int n)
> > >>> +{
> > >>> + struct msghdr msg = {};
> > >>> + struct cmsghdr *cmsg = NULL;
> > >>> + char buf[CMSG_SPACE(n * sizeof(int))];
> > >>> + char dummy_buffer = 0;
> > >>> + struct iovec io = { .iov_base = &dummy_buffer,
> > >>> + .iov_len = sizeof(dummy_buffer) };
> > >>> +
> > >>> + memset(buf, 0, sizeof(buf));
> > >>> +
> > >>> + msg.msg_iov = &io;
> > >>> + msg.msg_iovlen = 1;
> > >>> + msg.msg_control = buf;
> > >>> + msg.msg_controllen = sizeof(buf);
> > >>> +
> > >>> + cmsg = CMSG_FIRSTHDR(&msg);
> > >>> + cmsg->cmsg_level = SOL_SOCKET;
> > >>> + cmsg->cmsg_type = SCM_RIGHTS;
> > >>> + cmsg->cmsg_len = CMSG_LEN(n * sizeof(int));
> > >>> +
> > >>> + memcpy(CMSG_DATA(cmsg), fds, n * sizeof(int));
> > >>> +
> > >>> + return sendmsg(socket, &msg, 0);
> > >>> +}
> > >>> +
> > >>> +static void print_help_and_exit(const char *prog, int exitcode)
> > >>> +{
> > >>> + fprintf(stderr, "%s - load eBPF RSS program for qemu and pass eBPF fds"
> > >>> + " through unix socket.\n", prog);
> > >>> + fprintf(stderr, "\t--fd <num>, -f <num> - unix socket file descriptor"
> > >>> + " used to pass eBPF fds.\n");
> > >>> + fprintf(stderr, "\t--help, -h - this help.\n");
> > >>> + exit(exitcode);
> > >>> +}
> > >>> +
> > >>> +int main(int argc, char **argv)
> > >>> +{
> > >>> + char *fd_string = NULL;
> > >>> + int unix_fd = 0;
> > >>> + struct EBPFRSSContext ctx = {};
> > >>> + int fds[EBPF_RSS_MAX_FDS] = {};
> > >>> + int ret = -1;
> > >>> +
> > >>> + for (;;) {
> > >>> + int c;
> > >>> + static struct option long_options[] = {
> > >>> + {"help", no_argument, 0, 'h'},
> > >>> + {"fd", required_argument, 0, 'f'},
> > >>> + {0, 0, 0, 0}
> > >>> + };
> > >>> + c = getopt_long(argc, argv, "hf:",
> > >>> + long_options, NULL);
> > >>> +
> > >>> + if (c == -1) {
> > >>> + break;
> > >>> + }
> > >>> +
> > >>> + switch (c) {
> > >>> + case 'f':
> > >>> + fd_string = optarg;
> > >>> + break;
> > >>> + case 'h':
> > >>> + default:
> > >>> + print_help_and_exit(argv[0],
> > >>> + c == 'h' ? EXIT_SUCCESS : EXIT_FAILURE);
> > >>> + }
> > >>> + }
> > >>> +
> > >>> + if (!fd_string) {
> > >>> + fprintf(stderr, "Unix file descriptor not present.\n");
> > >>> + print_help_and_exit(argv[0], EXIT_FAILURE);
> > >>> + }
> > >>> +
> > >>> + unix_fd = atoi(fd_string);
> > >>> +
> > >>> + if (!unix_fd) {
> > >>> + fprintf(stderr, "Unix file descriptor is invalid.\n");
> > >>> + return EXIT_FAILURE;
> > >>> + }
> > >>> +
> > >>> + ebpf_rss_init(&ctx);
> > >>> + if (!ebpf_rss_load(&ctx)) {
> > >>> + fprintf(stderr, "Can't load ebpf.\n");
> > >>> + return EXIT_FAILURE;
> > >>> + }
> > >>> + fds[0] = ctx.program_fd;
> > >>> + fds[1] = ctx.map_configuration;
> > >>> +
> > >>> + ret = send_fds(unix_fd, fds, EBPF_RSS_MAX_FDS);
> > >>> + if (ret < 0) {
> > >>> + fprintf(stderr, "Issue while sending fds: %s.\n", strerror(errno));
> > >>> + }
> > >>> +
> > >>> + ebpf_rss_unload(&ctx);
> > >>> +
> > >>> + return ret < 0 ? EXIT_FAILURE : EXIT_SUCCESS;
> > >>> +}
> > >>> +
> > >>> diff --git a/meson.build b/meson.build
> > >>> index 257e51d91b..913aa1fee5 100644
> > >>> --- a/meson.build
> > >>> +++ b/meson.build
> > >>> @@ -1033,19 +1033,22 @@ if not get_option('fuse_lseek').disabled()
> > >>> endif
> > >>>
> > >>> # libbpf
> > >>> -libbpf = dependency('libbpf', required: get_option('bpf'), method: 'pkg-config')
> > >>> -if libbpf.found() and not cc.links('''
> > >>> - #include <bpf/libbpf.h>
> > >>> - int main(void)
> > >>> - {
> > >>> - bpf_object__destroy_skeleton(NULL);
> > >>> - return 0;
> > >>> - }''', dependencies: libbpf)
> > >>> - libbpf = not_found
> > >>> - if get_option('bpf').enabled()
> > >>> - error('libbpf skeleton test failed')
> > >>> - else
> > >>> - warning('libbpf skeleton test failed, disabling')
> > >>> +libbpf = not_found
> > >>> +if targetos == 'linux'
> > >>> + libbpf = dependency('libbpf', required: get_option('bpf'), method: 'pkg-config')
> > >>> + if libbpf.found() and not cc.links('''
> > >>> + #include <bpf/libbpf.h>
> > >>> + int main(void)
> > >>> + {
> > >>> + bpf_object__destroy_skeleton(NULL);
> > >>
> > >> Do we need to test whether the bpf can do mmap() here?
> > >>
> > >> Thanks
> > >>
> > >>
> > >>> + return 0;
> > >>> + }''', dependencies: libbpf)
> > >>> + libbpf = not_found
> > >>> + if get_option('bpf').enabled()
> > >>> + error('libbpf skeleton test failed')
> > >>> + else
> > >>> + warning('libbpf skeleton test failed, disabling')
> > >>> + endif
> > >>> endif
> > >>> endif
> > >>>
> > >>> @@ -2423,6 +2426,14 @@ if have_tools
> > >>> dependencies: [authz, crypto, io, qom, qemuutil,
> > >>> libcap_ng, mpathpersist],
> > >>> install: true)
> > >>> +
> > >>> + if libbpf.found()
> > >>> + executable('qemu-ebpf-rss-helper', files(
> > >>> + 'ebpf/qemu-ebpf-rss-helper.c', 'ebpf/ebpf_rss.c'),
> > >>> + dependencies: [qemuutil, libbpf, glib],
> > >>> + install: true,
> > >>> + install_dir: get_option('libexecdir'))
> > >>> + endif
> > >>> endif
> > >>>
> > >>> if 'CONFIG_IVSHMEM' in config_host
> >
>
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 4/5] ebpf_rss_helper: Added helper for eBPF RSS.
2021-09-08 3:45 ` Jason Wang
@ 2021-09-09 0:00 ` Yuri Benditovich
2021-09-09 1:16 ` Jason Wang
0 siblings, 1 reply; 60+ messages in thread
From: Yuri Benditovich @ 2021-09-09 0:00 UTC (permalink / raw)
To: Jason Wang
Cc: Andrew Melnychenko, Daniel P. Berrangé,
Michael S . Tsirkin, qemu-devel, Markus Armbruster,
Yan Vugenfirer, Eric Blake
On Wed, Sep 8, 2021 at 6:45 AM Jason Wang <jasowang@redhat.com> wrote:
>
> On Tue, Sep 7, 2021 at 6:40 PM Yuri Benditovich
> <yuri.benditovich@daynix.com> wrote:
> >
> > On Wed, Sep 1, 2021 at 9:42 AM Jason Wang <jasowang@redhat.com> wrote:
> > >
> > >
> > > 在 2021/8/31 上午1:07, Yuri Benditovich 写道:
> > > > On Fri, Aug 20, 2021 at 6:41 AM Jason Wang <jasowang@redhat.com> wrote:
> > > >>
> > > >> 在 2021/7/13 下午11:37, Andrew Melnychenko 写道:
> > > >>> Helper program. Loads eBPF RSS program and maps and passes them through unix socket.
> > > >>> Libvirt may launch this helper and pass eBPF fds to qemu virtio-net.
> > > >>
> > > >> I wonder if this can be done as helper for TAP/bridge.
> > > >>
> > > >> E.g it's the qemu to launch those helper with set-uid.
> > > >>
> > > >> Then libvirt won't even need to care about that?
> > > >>
> > > > There are pros and cons for such a solution with set-uid.
> > > > From my point of view one of the cons is that set-uid is efficient
> > > > only at install time so the coexistence of different qemu builds (and
> > > > different helpers for each one) is kind of problematic.
> > > > With the current solution this does not present any problem: the
> > > > developer can have several different builds, each one automatically
> > > > has its own helper and there is no conflict between these builds and
> > > > between these builds and installed qemu package. Changing the
> > > > 'emulator' in the libvirt profile automatically brings the proper
> > > > helper to work.
> > >
> > >
> > > I'm not sure I get you here. We can still have default/sample helper to
> > > make sure it works for different builds.
> > >
> > > If we can avoid the involvement of libvirt, that would be better.
> >
> > Hi Jason,
> >
> > Indeed I did not get the idea, can you please explain it in more
> > details (as detailed as possible to avoid future misunderstanding),
> > especially how exactly we can use the set-uid and what is the 'default' helper.
> > We also would prefer to do everything from qemu but we do not see how
> > we can do that.
>
>
Some more questions to understand the idea better:
> Something like:
>
> 1) -netdev tap,rss_helper=/path/to/name
So, on each editing of 'emulator' in the xml the helper path should
be set manually or be default?
> 2) having a sample/default helper implemented in Qemu
Does it mean the default helper is the code in the qemu (without
running additional executable, like it does today) or this is qemu
itself with dedicated command line?
As far as I remember Daniel had strong objections of ever running qemu
with capabilities
> 3) we can introduce something special path like "default", then if
> -netdev tap,rss_helper="default" is specified, qemu will use the
> sample helper
Probably this is not so important but the rss helper and rss in
general has no relation to netdev, much more they are related to
virtio-net
>
> So we have:
> 1) set set-uid for the helper
Who and when does set-uid to the helper binary? Only installer or
libvirt can do that, correct?
> 2) libvirt may just choose to launch the default helper
All this discussion is to avoid launching the helper from libvirt, correct?
>
> >
> > Our main points (what should be addressed):
> > - qemu should be able to load ebpf and use the maps when it runs from
> > libvirt (without special caps) and standalone (with caps)
>
> This is solved by leaving the privileged operations to the helper with set-uid.
>
> > - it is possible that there are different qemu builds on the machine,
> > one of them might be installed, their ebpf's might be different and
> > the interface between qemu and ebpf (exact content of maps and number
> > of maps)
>
> We can use different helpers in this way.
>
> > - qemu configures the RSS dynamically according to the commands
> > provided by the guest
>
> Consider we decided to use mmap() based maps, this is not an issue.
>
> Or am I missing something?
>
> Thanks
>
> >
> > Thanks in advance
> > Yuri
> >
> > >
> > > Thanks
> > >
> > >
> > > >
> > > >>> Also, libbpf dependency now exclusively for Linux.
> > > >>> Libbpf is used for eBPF RSS steering, which is supported only by Linux TAP.
> > > >>> There is no reason yet to build eBPF loader and helper for non Linux systems,
> > > >>> even if libbpf is present.
> > > >>>
> > > >>> Signed-off-by: Andrew Melnychenko <andrew@daynix.com>
> > > >>> ---
> > > >>> ebpf/qemu-ebpf-rss-helper.c | 130 ++++++++++++++++++++++++++++++++++++
> > > >>> meson.build | 37 ++++++----
> > > >>> 2 files changed, 154 insertions(+), 13 deletions(-)
> > > >>> create mode 100644 ebpf/qemu-ebpf-rss-helper.c
> > > >>>
> > > >>> diff --git a/ebpf/qemu-ebpf-rss-helper.c b/ebpf/qemu-ebpf-rss-helper.c
> > > >>> new file mode 100644
> > > >>> index 0000000000..fe68758f57
> > > >>> --- /dev/null
> > > >>> +++ b/ebpf/qemu-ebpf-rss-helper.c
> > > >>> @@ -0,0 +1,130 @@
> > > >>> +/*
> > > >>> + * eBPF RSS Helper
> > > >>> + *
> > > >>> + * Developed by Daynix Computing LTD (http://www.daynix.com)
> > > >>> + *
> > > >>> + * Authors:
> > > >>> + * Andrew Melnychenko <andrew@daynix.com>
> > > >>> + *
> > > >>> + * This work is licensed under the terms of the GNU GPL, version 2. See
> > > >>> + * the COPYING file in the top-level directory.
> > > >>> + *
> > > >>> + * Description: This is helper program for libvirtd.
> > > >>> + * It loads eBPF RSS program and passes fds through unix socket.
> > > >>> + * Built by meson, target - 'qemu-ebpf-rss-helper'.
> > > >>> + */
> > > >>> +
> > > >>> +#include <stdio.h>
> > > >>> +#include <stdint.h>
> > > >>> +#include <stdlib.h>
> > > >>> +#include <stdbool.h>
> > > >>> +#include <getopt.h>
> > > >>> +#include <memory.h>
> > > >>> +#include <errno.h>
> > > >>> +#include <sys/socket.h>
> > > >>> +
> > > >>> +#include "ebpf_rss.h"
> > > >>> +
> > > >>> +#include "qemu-helper-stamp.h"
> > > >>> +
> > > >>> +void QEMU_HELPER_STAMP(void) {}
> > > >>> +
> > > >>> +static int send_fds(int socket, int *fds, int n)
> > > >>> +{
> > > >>> + struct msghdr msg = {};
> > > >>> + struct cmsghdr *cmsg = NULL;
> > > >>> + char buf[CMSG_SPACE(n * sizeof(int))];
> > > >>> + char dummy_buffer = 0;
> > > >>> + struct iovec io = { .iov_base = &dummy_buffer,
> > > >>> + .iov_len = sizeof(dummy_buffer) };
> > > >>> +
> > > >>> + memset(buf, 0, sizeof(buf));
> > > >>> +
> > > >>> + msg.msg_iov = &io;
> > > >>> + msg.msg_iovlen = 1;
> > > >>> + msg.msg_control = buf;
> > > >>> + msg.msg_controllen = sizeof(buf);
> > > >>> +
> > > >>> + cmsg = CMSG_FIRSTHDR(&msg);
> > > >>> + cmsg->cmsg_level = SOL_SOCKET;
> > > >>> + cmsg->cmsg_type = SCM_RIGHTS;
> > > >>> + cmsg->cmsg_len = CMSG_LEN(n * sizeof(int));
> > > >>> +
> > > >>> + memcpy(CMSG_DATA(cmsg), fds, n * sizeof(int));
> > > >>> +
> > > >>> + return sendmsg(socket, &msg, 0);
> > > >>> +}
> > > >>> +
> > > >>> +static void print_help_and_exit(const char *prog, int exitcode)
> > > >>> +{
> > > >>> + fprintf(stderr, "%s - load eBPF RSS program for qemu and pass eBPF fds"
> > > >>> + " through unix socket.\n", prog);
> > > >>> + fprintf(stderr, "\t--fd <num>, -f <num> - unix socket file descriptor"
> > > >>> + " used to pass eBPF fds.\n");
> > > >>> + fprintf(stderr, "\t--help, -h - this help.\n");
> > > >>> + exit(exitcode);
> > > >>> +}
> > > >>> +
> > > >>> +int main(int argc, char **argv)
> > > >>> +{
> > > >>> + char *fd_string = NULL;
> > > >>> + int unix_fd = 0;
> > > >>> + struct EBPFRSSContext ctx = {};
> > > >>> + int fds[EBPF_RSS_MAX_FDS] = {};
> > > >>> + int ret = -1;
> > > >>> +
> > > >>> + for (;;) {
> > > >>> + int c;
> > > >>> + static struct option long_options[] = {
> > > >>> + {"help", no_argument, 0, 'h'},
> > > >>> + {"fd", required_argument, 0, 'f'},
> > > >>> + {0, 0, 0, 0}
> > > >>> + };
> > > >>> + c = getopt_long(argc, argv, "hf:",
> > > >>> + long_options, NULL);
> > > >>> +
> > > >>> + if (c == -1) {
> > > >>> + break;
> > > >>> + }
> > > >>> +
> > > >>> + switch (c) {
> > > >>> + case 'f':
> > > >>> + fd_string = optarg;
> > > >>> + break;
> > > >>> + case 'h':
> > > >>> + default:
> > > >>> + print_help_and_exit(argv[0],
> > > >>> + c == 'h' ? EXIT_SUCCESS : EXIT_FAILURE);
> > > >>> + }
> > > >>> + }
> > > >>> +
> > > >>> + if (!fd_string) {
> > > >>> + fprintf(stderr, "Unix file descriptor not present.\n");
> > > >>> + print_help_and_exit(argv[0], EXIT_FAILURE);
> > > >>> + }
> > > >>> +
> > > >>> + unix_fd = atoi(fd_string);
> > > >>> +
> > > >>> + if (!unix_fd) {
> > > >>> + fprintf(stderr, "Unix file descriptor is invalid.\n");
> > > >>> + return EXIT_FAILURE;
> > > >>> + }
> > > >>> +
> > > >>> + ebpf_rss_init(&ctx);
> > > >>> + if (!ebpf_rss_load(&ctx)) {
> > > >>> + fprintf(stderr, "Can't load ebpf.\n");
> > > >>> + return EXIT_FAILURE;
> > > >>> + }
> > > >>> + fds[0] = ctx.program_fd;
> > > >>> + fds[1] = ctx.map_configuration;
> > > >>> +
> > > >>> + ret = send_fds(unix_fd, fds, EBPF_RSS_MAX_FDS);
> > > >>> + if (ret < 0) {
> > > >>> + fprintf(stderr, "Issue while sending fds: %s.\n", strerror(errno));
> > > >>> + }
> > > >>> +
> > > >>> + ebpf_rss_unload(&ctx);
> > > >>> +
> > > >>> + return ret < 0 ? EXIT_FAILURE : EXIT_SUCCESS;
> > > >>> +}
> > > >>> +
> > > >>> diff --git a/meson.build b/meson.build
> > > >>> index 257e51d91b..913aa1fee5 100644
> > > >>> --- a/meson.build
> > > >>> +++ b/meson.build
> > > >>> @@ -1033,19 +1033,22 @@ if not get_option('fuse_lseek').disabled()
> > > >>> endif
> > > >>>
> > > >>> # libbpf
> > > >>> -libbpf = dependency('libbpf', required: get_option('bpf'), method: 'pkg-config')
> > > >>> -if libbpf.found() and not cc.links('''
> > > >>> - #include <bpf/libbpf.h>
> > > >>> - int main(void)
> > > >>> - {
> > > >>> - bpf_object__destroy_skeleton(NULL);
> > > >>> - return 0;
> > > >>> - }''', dependencies: libbpf)
> > > >>> - libbpf = not_found
> > > >>> - if get_option('bpf').enabled()
> > > >>> - error('libbpf skeleton test failed')
> > > >>> - else
> > > >>> - warning('libbpf skeleton test failed, disabling')
> > > >>> +libbpf = not_found
> > > >>> +if targetos == 'linux'
> > > >>> + libbpf = dependency('libbpf', required: get_option('bpf'), method: 'pkg-config')
> > > >>> + if libbpf.found() and not cc.links('''
> > > >>> + #include <bpf/libbpf.h>
> > > >>> + int main(void)
> > > >>> + {
> > > >>> + bpf_object__destroy_skeleton(NULL);
> > > >>
> > > >> Do we need to test whether the bpf can do mmap() here?
> > > >>
> > > >> Thanks
> > > >>
> > > >>
> > > >>> + return 0;
> > > >>> + }''', dependencies: libbpf)
> > > >>> + libbpf = not_found
> > > >>> + if get_option('bpf').enabled()
> > > >>> + error('libbpf skeleton test failed')
> > > >>> + else
> > > >>> + warning('libbpf skeleton test failed, disabling')
> > > >>> + endif
> > > >>> endif
> > > >>> endif
> > > >>>
> > > >>> @@ -2423,6 +2426,14 @@ if have_tools
> > > >>> dependencies: [authz, crypto, io, qom, qemuutil,
> > > >>> libcap_ng, mpathpersist],
> > > >>> install: true)
> > > >>> +
> > > >>> + if libbpf.found()
> > > >>> + executable('qemu-ebpf-rss-helper', files(
> > > >>> + 'ebpf/qemu-ebpf-rss-helper.c', 'ebpf/ebpf_rss.c'),
> > > >>> + dependencies: [qemuutil, libbpf, glib],
> > > >>> + install: true,
> > > >>> + install_dir: get_option('libexecdir'))
> > > >>> + endif
> > > >>> endif
> > > >>>
> > > >>> if 'CONFIG_IVSHMEM' in config_host
> > >
> >
>
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 4/5] ebpf_rss_helper: Added helper for eBPF RSS.
2021-09-09 0:00 ` Yuri Benditovich
@ 2021-09-09 1:16 ` Jason Wang
2021-09-09 23:43 ` Yuri Benditovich
0 siblings, 1 reply; 60+ messages in thread
From: Jason Wang @ 2021-09-09 1:16 UTC (permalink / raw)
To: Yuri Benditovich
Cc: Andrew Melnychenko, Daniel P. Berrangé,
Michael S . Tsirkin, qemu-devel, Markus Armbruster,
Yan Vugenfirer, Eric Blake
On Thu, Sep 9, 2021 at 8:00 AM Yuri Benditovich
<yuri.benditovich@daynix.com> wrote:
>
> On Wed, Sep 8, 2021 at 6:45 AM Jason Wang <jasowang@redhat.com> wrote:
> >
> > On Tue, Sep 7, 2021 at 6:40 PM Yuri Benditovich
> > <yuri.benditovich@daynix.com> wrote:
> > >
> > > On Wed, Sep 1, 2021 at 9:42 AM Jason Wang <jasowang@redhat.com> wrote:
> > > >
> > > >
> > > > 在 2021/8/31 上午1:07, Yuri Benditovich 写道:
> > > > > On Fri, Aug 20, 2021 at 6:41 AM Jason Wang <jasowang@redhat.com> wrote:
> > > > >>
> > > > >> 在 2021/7/13 下午11:37, Andrew Melnychenko 写道:
> > > > >>> Helper program. Loads eBPF RSS program and maps and passes them through unix socket.
> > > > >>> Libvirt may launch this helper and pass eBPF fds to qemu virtio-net.
> > > > >>
> > > > >> I wonder if this can be done as helper for TAP/bridge.
> > > > >>
> > > > >> E.g it's the qemu to launch those helper with set-uid.
> > > > >>
> > > > >> Then libvirt won't even need to care about that?
> > > > >>
> > > > > There are pros and cons for such a solution with set-uid.
> > > > > From my point of view one of the cons is that set-uid is efficient
> > > > > only at install time so the coexistence of different qemu builds (and
> > > > > different helpers for each one) is kind of problematic.
> > > > > With the current solution this does not present any problem: the
> > > > > developer can have several different builds, each one automatically
> > > > > has its own helper and there is no conflict between these builds and
> > > > > between these builds and installed qemu package. Changing the
> > > > > 'emulator' in the libvirt profile automatically brings the proper
> > > > > helper to work.
> > > >
> > > >
> > > > I'm not sure I get you here. We can still have default/sample helper to
> > > > make sure it works for different builds.
> > > >
> > > > If we can avoid the involvement of libvirt, that would be better.
> > >
> > > Hi Jason,
> > >
> > > Indeed I did not get the idea, can you please explain it in more
> > > details (as detailed as possible to avoid future misunderstanding),
> > > especially how exactly we can use the set-uid and what is the 'default' helper.
> > > We also would prefer to do everything from qemu but we do not see how
> > > we can do that.
> >
> >
> Some more questions to understand the idea better:
> > Something like:
> >
> > 1) -netdev tap,rss_helper=/path/to/name
>
> So, on each editing of 'emulator' in the xml the helper path should
> be set manually or be default?
It could done manually, or we can have a default path.
>
> > 2) having a sample/default helper implemented in Qemu
>
> Does it mean the default helper is the code in the qemu (without
> running additional executable, like it does today)
Yes.
or this is qemu
> itself with dedicated command line?
> As far as I remember Daniel had strong objections of ever running qemu
> with capabilities
Qemu won't run with capabilities but the helper.
>
> > 3) we can introduce something special path like "default", then if
> > -netdev tap,rss_helper="default" is specified, qemu will use the
> > sample helper
>
> Probably this is not so important but the rss helper and rss in
> general has no relation to netdev, much more they are related to
> virtio-net
So I think the reason for this is that we currently only support
eBPF/RSS for tap.
>
> >
> > So we have:
> > 1) set set-uid for the helper
> Who and when does set-uid to the helper binary? Only installer or
> libvirt can do that, correct?
Yes, it could be done the installer, or other system provision tools.
>
> > 2) libvirt may just choose to launch the default helper
> All this discussion is to avoid launching the helper from libvirt, correct?
Sorry, it's a typo. I meant, libvirt launch qemu, and then qemu will
launch the helper.
Thanks
>
> >
> > >
> > > Our main points (what should be addressed):
> > > - qemu should be able to load ebpf and use the maps when it runs from
> > > libvirt (without special caps) and standalone (with caps)
> >
> > This is solved by leaving the privileged operations to the helper with set-uid.
> >
> > > - it is possible that there are different qemu builds on the machine,
> > > one of them might be installed, their ebpf's might be different and
> > > the interface between qemu and ebpf (exact content of maps and number
> > > of maps)
> >
> > We can use different helpers in this way.
> >
> > > - qemu configures the RSS dynamically according to the commands
> > > provided by the guest
> >
> > Consider we decided to use mmap() based maps, this is not an issue.
> >
> > Or am I missing something?
> >
> > Thanks
> >
> > >
> > > Thanks in advance
> > > Yuri
> > >
> > > >
> > > > Thanks
> > > >
> > > >
> > > > >
> > > > >>> Also, libbpf dependency now exclusively for Linux.
> > > > >>> Libbpf is used for eBPF RSS steering, which is supported only by Linux TAP.
> > > > >>> There is no reason yet to build eBPF loader and helper for non Linux systems,
> > > > >>> even if libbpf is present.
> > > > >>>
> > > > >>> Signed-off-by: Andrew Melnychenko <andrew@daynix.com>
> > > > >>> ---
> > > > >>> ebpf/qemu-ebpf-rss-helper.c | 130 ++++++++++++++++++++++++++++++++++++
> > > > >>> meson.build | 37 ++++++----
> > > > >>> 2 files changed, 154 insertions(+), 13 deletions(-)
> > > > >>> create mode 100644 ebpf/qemu-ebpf-rss-helper.c
> > > > >>>
> > > > >>> diff --git a/ebpf/qemu-ebpf-rss-helper.c b/ebpf/qemu-ebpf-rss-helper.c
> > > > >>> new file mode 100644
> > > > >>> index 0000000000..fe68758f57
> > > > >>> --- /dev/null
> > > > >>> +++ b/ebpf/qemu-ebpf-rss-helper.c
> > > > >>> @@ -0,0 +1,130 @@
> > > > >>> +/*
> > > > >>> + * eBPF RSS Helper
> > > > >>> + *
> > > > >>> + * Developed by Daynix Computing LTD (http://www.daynix.com)
> > > > >>> + *
> > > > >>> + * Authors:
> > > > >>> + * Andrew Melnychenko <andrew@daynix.com>
> > > > >>> + *
> > > > >>> + * This work is licensed under the terms of the GNU GPL, version 2. See
> > > > >>> + * the COPYING file in the top-level directory.
> > > > >>> + *
> > > > >>> + * Description: This is helper program for libvirtd.
> > > > >>> + * It loads eBPF RSS program and passes fds through unix socket.
> > > > >>> + * Built by meson, target - 'qemu-ebpf-rss-helper'.
> > > > >>> + */
> > > > >>> +
> > > > >>> +#include <stdio.h>
> > > > >>> +#include <stdint.h>
> > > > >>> +#include <stdlib.h>
> > > > >>> +#include <stdbool.h>
> > > > >>> +#include <getopt.h>
> > > > >>> +#include <memory.h>
> > > > >>> +#include <errno.h>
> > > > >>> +#include <sys/socket.h>
> > > > >>> +
> > > > >>> +#include "ebpf_rss.h"
> > > > >>> +
> > > > >>> +#include "qemu-helper-stamp.h"
> > > > >>> +
> > > > >>> +void QEMU_HELPER_STAMP(void) {}
> > > > >>> +
> > > > >>> +static int send_fds(int socket, int *fds, int n)
> > > > >>> +{
> > > > >>> + struct msghdr msg = {};
> > > > >>> + struct cmsghdr *cmsg = NULL;
> > > > >>> + char buf[CMSG_SPACE(n * sizeof(int))];
> > > > >>> + char dummy_buffer = 0;
> > > > >>> + struct iovec io = { .iov_base = &dummy_buffer,
> > > > >>> + .iov_len = sizeof(dummy_buffer) };
> > > > >>> +
> > > > >>> + memset(buf, 0, sizeof(buf));
> > > > >>> +
> > > > >>> + msg.msg_iov = &io;
> > > > >>> + msg.msg_iovlen = 1;
> > > > >>> + msg.msg_control = buf;
> > > > >>> + msg.msg_controllen = sizeof(buf);
> > > > >>> +
> > > > >>> + cmsg = CMSG_FIRSTHDR(&msg);
> > > > >>> + cmsg->cmsg_level = SOL_SOCKET;
> > > > >>> + cmsg->cmsg_type = SCM_RIGHTS;
> > > > >>> + cmsg->cmsg_len = CMSG_LEN(n * sizeof(int));
> > > > >>> +
> > > > >>> + memcpy(CMSG_DATA(cmsg), fds, n * sizeof(int));
> > > > >>> +
> > > > >>> + return sendmsg(socket, &msg, 0);
> > > > >>> +}
> > > > >>> +
> > > > >>> +static void print_help_and_exit(const char *prog, int exitcode)
> > > > >>> +{
> > > > >>> + fprintf(stderr, "%s - load eBPF RSS program for qemu and pass eBPF fds"
> > > > >>> + " through unix socket.\n", prog);
> > > > >>> + fprintf(stderr, "\t--fd <num>, -f <num> - unix socket file descriptor"
> > > > >>> + " used to pass eBPF fds.\n");
> > > > >>> + fprintf(stderr, "\t--help, -h - this help.\n");
> > > > >>> + exit(exitcode);
> > > > >>> +}
> > > > >>> +
> > > > >>> +int main(int argc, char **argv)
> > > > >>> +{
> > > > >>> + char *fd_string = NULL;
> > > > >>> + int unix_fd = 0;
> > > > >>> + struct EBPFRSSContext ctx = {};
> > > > >>> + int fds[EBPF_RSS_MAX_FDS] = {};
> > > > >>> + int ret = -1;
> > > > >>> +
> > > > >>> + for (;;) {
> > > > >>> + int c;
> > > > >>> + static struct option long_options[] = {
> > > > >>> + {"help", no_argument, 0, 'h'},
> > > > >>> + {"fd", required_argument, 0, 'f'},
> > > > >>> + {0, 0, 0, 0}
> > > > >>> + };
> > > > >>> + c = getopt_long(argc, argv, "hf:",
> > > > >>> + long_options, NULL);
> > > > >>> +
> > > > >>> + if (c == -1) {
> > > > >>> + break;
> > > > >>> + }
> > > > >>> +
> > > > >>> + switch (c) {
> > > > >>> + case 'f':
> > > > >>> + fd_string = optarg;
> > > > >>> + break;
> > > > >>> + case 'h':
> > > > >>> + default:
> > > > >>> + print_help_and_exit(argv[0],
> > > > >>> + c == 'h' ? EXIT_SUCCESS : EXIT_FAILURE);
> > > > >>> + }
> > > > >>> + }
> > > > >>> +
> > > > >>> + if (!fd_string) {
> > > > >>> + fprintf(stderr, "Unix file descriptor not present.\n");
> > > > >>> + print_help_and_exit(argv[0], EXIT_FAILURE);
> > > > >>> + }
> > > > >>> +
> > > > >>> + unix_fd = atoi(fd_string);
> > > > >>> +
> > > > >>> + if (!unix_fd) {
> > > > >>> + fprintf(stderr, "Unix file descriptor is invalid.\n");
> > > > >>> + return EXIT_FAILURE;
> > > > >>> + }
> > > > >>> +
> > > > >>> + ebpf_rss_init(&ctx);
> > > > >>> + if (!ebpf_rss_load(&ctx)) {
> > > > >>> + fprintf(stderr, "Can't load ebpf.\n");
> > > > >>> + return EXIT_FAILURE;
> > > > >>> + }
> > > > >>> + fds[0] = ctx.program_fd;
> > > > >>> + fds[1] = ctx.map_configuration;
> > > > >>> +
> > > > >>> + ret = send_fds(unix_fd, fds, EBPF_RSS_MAX_FDS);
> > > > >>> + if (ret < 0) {
> > > > >>> + fprintf(stderr, "Issue while sending fds: %s.\n", strerror(errno));
> > > > >>> + }
> > > > >>> +
> > > > >>> + ebpf_rss_unload(&ctx);
> > > > >>> +
> > > > >>> + return ret < 0 ? EXIT_FAILURE : EXIT_SUCCESS;
> > > > >>> +}
> > > > >>> +
> > > > >>> diff --git a/meson.build b/meson.build
> > > > >>> index 257e51d91b..913aa1fee5 100644
> > > > >>> --- a/meson.build
> > > > >>> +++ b/meson.build
> > > > >>> @@ -1033,19 +1033,22 @@ if not get_option('fuse_lseek').disabled()
> > > > >>> endif
> > > > >>>
> > > > >>> # libbpf
> > > > >>> -libbpf = dependency('libbpf', required: get_option('bpf'), method: 'pkg-config')
> > > > >>> -if libbpf.found() and not cc.links('''
> > > > >>> - #include <bpf/libbpf.h>
> > > > >>> - int main(void)
> > > > >>> - {
> > > > >>> - bpf_object__destroy_skeleton(NULL);
> > > > >>> - return 0;
> > > > >>> - }''', dependencies: libbpf)
> > > > >>> - libbpf = not_found
> > > > >>> - if get_option('bpf').enabled()
> > > > >>> - error('libbpf skeleton test failed')
> > > > >>> - else
> > > > >>> - warning('libbpf skeleton test failed, disabling')
> > > > >>> +libbpf = not_found
> > > > >>> +if targetos == 'linux'
> > > > >>> + libbpf = dependency('libbpf', required: get_option('bpf'), method: 'pkg-config')
> > > > >>> + if libbpf.found() and not cc.links('''
> > > > >>> + #include <bpf/libbpf.h>
> > > > >>> + int main(void)
> > > > >>> + {
> > > > >>> + bpf_object__destroy_skeleton(NULL);
> > > > >>
> > > > >> Do we need to test whether the bpf can do mmap() here?
> > > > >>
> > > > >> Thanks
> > > > >>
> > > > >>
> > > > >>> + return 0;
> > > > >>> + }''', dependencies: libbpf)
> > > > >>> + libbpf = not_found
> > > > >>> + if get_option('bpf').enabled()
> > > > >>> + error('libbpf skeleton test failed')
> > > > >>> + else
> > > > >>> + warning('libbpf skeleton test failed, disabling')
> > > > >>> + endif
> > > > >>> endif
> > > > >>> endif
> > > > >>>
> > > > >>> @@ -2423,6 +2426,14 @@ if have_tools
> > > > >>> dependencies: [authz, crypto, io, qom, qemuutil,
> > > > >>> libcap_ng, mpathpersist],
> > > > >>> install: true)
> > > > >>> +
> > > > >>> + if libbpf.found()
> > > > >>> + executable('qemu-ebpf-rss-helper', files(
> > > > >>> + 'ebpf/qemu-ebpf-rss-helper.c', 'ebpf/ebpf_rss.c'),
> > > > >>> + dependencies: [qemuutil, libbpf, glib],
> > > > >>> + install: true,
> > > > >>> + install_dir: get_option('libexecdir'))
> > > > >>> + endif
> > > > >>> endif
> > > > >>>
> > > > >>> if 'CONFIG_IVSHMEM' in config_host
> > > >
> > >
> >
>
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 4/5] ebpf_rss_helper: Added helper for eBPF RSS.
2021-09-09 1:16 ` Jason Wang
@ 2021-09-09 23:43 ` Yuri Benditovich
2021-09-10 1:37 ` Jason Wang
0 siblings, 1 reply; 60+ messages in thread
From: Yuri Benditovich @ 2021-09-09 23:43 UTC (permalink / raw)
To: Jason Wang
Cc: Andrew Melnychenko, Daniel P. Berrangé,
Michael S . Tsirkin, qemu-devel, Markus Armbruster,
Yan Vugenfirer, Eric Blake
On Thu, Sep 9, 2021 at 4:16 AM Jason Wang <jasowang@redhat.com> wrote:
>
> On Thu, Sep 9, 2021 at 8:00 AM Yuri Benditovich
> <yuri.benditovich@daynix.com> wrote:
> >
> > On Wed, Sep 8, 2021 at 6:45 AM Jason Wang <jasowang@redhat.com> wrote:
> > >
> > > On Tue, Sep 7, 2021 at 6:40 PM Yuri Benditovich
> > > <yuri.benditovich@daynix.com> wrote:
> > > >
> > > > On Wed, Sep 1, 2021 at 9:42 AM Jason Wang <jasowang@redhat.com> wrote:
> > > > >
> > > > >
> > > > > 在 2021/8/31 上午1:07, Yuri Benditovich 写道:
> > > > > > On Fri, Aug 20, 2021 at 6:41 AM Jason Wang <jasowang@redhat.com> wrote:
> > > > > >>
> > > > > >> 在 2021/7/13 下午11:37, Andrew Melnychenko 写道:
> > > > > >>> Helper program. Loads eBPF RSS program and maps and passes them through unix socket.
> > > > > >>> Libvirt may launch this helper and pass eBPF fds to qemu virtio-net.
> > > > > >>
> > > > > >> I wonder if this can be done as helper for TAP/bridge.
> > > > > >>
> > > > > >> E.g it's the qemu to launch those helper with set-uid.
> > > > > >>
> > > > > >> Then libvirt won't even need to care about that?
> > > > > >>
> > > > > > There are pros and cons for such a solution with set-uid.
> > > > > > From my point of view one of the cons is that set-uid is efficient
> > > > > > only at install time so the coexistence of different qemu builds (and
> > > > > > different helpers for each one) is kind of problematic.
> > > > > > With the current solution this does not present any problem: the
> > > > > > developer can have several different builds, each one automatically
> > > > > > has its own helper and there is no conflict between these builds and
> > > > > > between these builds and installed qemu package. Changing the
> > > > > > 'emulator' in the libvirt profile automatically brings the proper
> > > > > > helper to work.
> > > > >
> > > > >
> > > > > I'm not sure I get you here. We can still have default/sample helper to
> > > > > make sure it works for different builds.
> > > > >
> > > > > If we can avoid the involvement of libvirt, that would be better.
> > > >
> > > > Hi Jason,
> > > >
> > > > Indeed I did not get the idea, can you please explain it in more
> > > > details (as detailed as possible to avoid future misunderstanding),
> > > > especially how exactly we can use the set-uid and what is the 'default' helper.
> > > > We also would prefer to do everything from qemu but we do not see how
> > > > we can do that.
> > >
> > >
> > Some more questions to understand the idea better:
> > > Something like:
> > >
> > > 1) -netdev tap,rss_helper=/path/to/name
> >
> > So, on each editing of 'emulator' in the xml the helper path should
> > be set manually or be default?
>
> It could done manually, or we can have a default path.
>
> >
> > > 2) having a sample/default helper implemented in Qemu
> >
> > Does it mean the default helper is the code in the qemu (without
> > running additional executable, like it does today)
>
> Yes.
If the "default helper" is just a keyword and it is like what we have
today (i.e. part of qemu) it can't work under libvirt and should never
be used by libvirt.
>
> or this is qemu
> > itself with dedicated command line?
> > As far as I remember Daniel had strong objections of ever running qemu
> > with capabilities
>
> Qemu won't run with capabilities but the helper.
So under libvirt the helper is always separate executable and not
"default helper"
>
> >
> > > 3) we can introduce something special path like "default", then if
> > > -netdev tap,rss_helper="default" is specified, qemu will use the
> > > sample helper
> >
> > Probably this is not so important but the rss helper and rss in
> > general has no relation to netdev, much more they are related to
> > virtio-net
>
> So I think the reason for this is that we currently only support
> eBPF/RSS for tap.
This is just because only tap supports respective ioctls.
>
> >
> > >
> > > So we have:
> > > 1) set set-uid for the helper
> > Who and when does set-uid to the helper binary? Only installer or
> > libvirt can do that, correct?
>
> Yes, it could be done the installer, or other system provision tools.
So this changes the rule of the game: currently libvirt runs helpers
that require privileges and qemu runs helpers that do not require any
privileges (like TPM).
If we follow your suggestion - only for RSS we will create the helper
that must be used with set-uid.
Who are stakeholders we need to have a consensus with?
>
> >
> > > 2) libvirt may just choose to launch the default helper
> > All this discussion is to avoid launching the helper from libvirt, correct?
>
> Sorry, it's a typo. I meant, libvirt launch qemu, and then qemu will
> launch the helper.
>
> Thanks
>
> >
> > >
> > > >
> > > > Our main points (what should be addressed):
> > > > - qemu should be able to load ebpf and use the maps when it runs from
> > > > libvirt (without special caps) and standalone (with caps)
> > >
> > > This is solved by leaving the privileged operations to the helper with set-uid.
> > >
> > > > - it is possible that there are different qemu builds on the machine,
> > > > one of them might be installed, their ebpf's might be different and
> > > > the interface between qemu and ebpf (exact content of maps and number
> > > > of maps)
> > >
> > > We can use different helpers in this way.
> > >
> > > > - qemu configures the RSS dynamically according to the commands
> > > > provided by the guest
> > >
> > > Consider we decided to use mmap() based maps, this is not an issue.
> > >
> > > Or am I missing something?
> > >
> > > Thanks
> > >
> > > >
> > > > Thanks in advance
> > > > Yuri
> > > >
> > > > >
> > > > > Thanks
> > > > >
> > > > >
> > > > > >
> > > > > >>> Also, libbpf dependency now exclusively for Linux.
> > > > > >>> Libbpf is used for eBPF RSS steering, which is supported only by Linux TAP.
> > > > > >>> There is no reason yet to build eBPF loader and helper for non Linux systems,
> > > > > >>> even if libbpf is present.
> > > > > >>>
> > > > > >>> Signed-off-by: Andrew Melnychenko <andrew@daynix.com>
> > > > > >>> ---
> > > > > >>> ebpf/qemu-ebpf-rss-helper.c | 130 ++++++++++++++++++++++++++++++++++++
> > > > > >>> meson.build | 37 ++++++----
> > > > > >>> 2 files changed, 154 insertions(+), 13 deletions(-)
> > > > > >>> create mode 100644 ebpf/qemu-ebpf-rss-helper.c
> > > > > >>>
> > > > > >>> diff --git a/ebpf/qemu-ebpf-rss-helper.c b/ebpf/qemu-ebpf-rss-helper.c
> > > > > >>> new file mode 100644
> > > > > >>> index 0000000000..fe68758f57
> > > > > >>> --- /dev/null
> > > > > >>> +++ b/ebpf/qemu-ebpf-rss-helper.c
> > > > > >>> @@ -0,0 +1,130 @@
> > > > > >>> +/*
> > > > > >>> + * eBPF RSS Helper
> > > > > >>> + *
> > > > > >>> + * Developed by Daynix Computing LTD (http://www.daynix.com)
> > > > > >>> + *
> > > > > >>> + * Authors:
> > > > > >>> + * Andrew Melnychenko <andrew@daynix.com>
> > > > > >>> + *
> > > > > >>> + * This work is licensed under the terms of the GNU GPL, version 2. See
> > > > > >>> + * the COPYING file in the top-level directory.
> > > > > >>> + *
> > > > > >>> + * Description: This is helper program for libvirtd.
> > > > > >>> + * It loads eBPF RSS program and passes fds through unix socket.
> > > > > >>> + * Built by meson, target - 'qemu-ebpf-rss-helper'.
> > > > > >>> + */
> > > > > >>> +
> > > > > >>> +#include <stdio.h>
> > > > > >>> +#include <stdint.h>
> > > > > >>> +#include <stdlib.h>
> > > > > >>> +#include <stdbool.h>
> > > > > >>> +#include <getopt.h>
> > > > > >>> +#include <memory.h>
> > > > > >>> +#include <errno.h>
> > > > > >>> +#include <sys/socket.h>
> > > > > >>> +
> > > > > >>> +#include "ebpf_rss.h"
> > > > > >>> +
> > > > > >>> +#include "qemu-helper-stamp.h"
> > > > > >>> +
> > > > > >>> +void QEMU_HELPER_STAMP(void) {}
> > > > > >>> +
> > > > > >>> +static int send_fds(int socket, int *fds, int n)
> > > > > >>> +{
> > > > > >>> + struct msghdr msg = {};
> > > > > >>> + struct cmsghdr *cmsg = NULL;
> > > > > >>> + char buf[CMSG_SPACE(n * sizeof(int))];
> > > > > >>> + char dummy_buffer = 0;
> > > > > >>> + struct iovec io = { .iov_base = &dummy_buffer,
> > > > > >>> + .iov_len = sizeof(dummy_buffer) };
> > > > > >>> +
> > > > > >>> + memset(buf, 0, sizeof(buf));
> > > > > >>> +
> > > > > >>> + msg.msg_iov = &io;
> > > > > >>> + msg.msg_iovlen = 1;
> > > > > >>> + msg.msg_control = buf;
> > > > > >>> + msg.msg_controllen = sizeof(buf);
> > > > > >>> +
> > > > > >>> + cmsg = CMSG_FIRSTHDR(&msg);
> > > > > >>> + cmsg->cmsg_level = SOL_SOCKET;
> > > > > >>> + cmsg->cmsg_type = SCM_RIGHTS;
> > > > > >>> + cmsg->cmsg_len = CMSG_LEN(n * sizeof(int));
> > > > > >>> +
> > > > > >>> + memcpy(CMSG_DATA(cmsg), fds, n * sizeof(int));
> > > > > >>> +
> > > > > >>> + return sendmsg(socket, &msg, 0);
> > > > > >>> +}
> > > > > >>> +
> > > > > >>> +static void print_help_and_exit(const char *prog, int exitcode)
> > > > > >>> +{
> > > > > >>> + fprintf(stderr, "%s - load eBPF RSS program for qemu and pass eBPF fds"
> > > > > >>> + " through unix socket.\n", prog);
> > > > > >>> + fprintf(stderr, "\t--fd <num>, -f <num> - unix socket file descriptor"
> > > > > >>> + " used to pass eBPF fds.\n");
> > > > > >>> + fprintf(stderr, "\t--help, -h - this help.\n");
> > > > > >>> + exit(exitcode);
> > > > > >>> +}
> > > > > >>> +
> > > > > >>> +int main(int argc, char **argv)
> > > > > >>> +{
> > > > > >>> + char *fd_string = NULL;
> > > > > >>> + int unix_fd = 0;
> > > > > >>> + struct EBPFRSSContext ctx = {};
> > > > > >>> + int fds[EBPF_RSS_MAX_FDS] = {};
> > > > > >>> + int ret = -1;
> > > > > >>> +
> > > > > >>> + for (;;) {
> > > > > >>> + int c;
> > > > > >>> + static struct option long_options[] = {
> > > > > >>> + {"help", no_argument, 0, 'h'},
> > > > > >>> + {"fd", required_argument, 0, 'f'},
> > > > > >>> + {0, 0, 0, 0}
> > > > > >>> + };
> > > > > >>> + c = getopt_long(argc, argv, "hf:",
> > > > > >>> + long_options, NULL);
> > > > > >>> +
> > > > > >>> + if (c == -1) {
> > > > > >>> + break;
> > > > > >>> + }
> > > > > >>> +
> > > > > >>> + switch (c) {
> > > > > >>> + case 'f':
> > > > > >>> + fd_string = optarg;
> > > > > >>> + break;
> > > > > >>> + case 'h':
> > > > > >>> + default:
> > > > > >>> + print_help_and_exit(argv[0],
> > > > > >>> + c == 'h' ? EXIT_SUCCESS : EXIT_FAILURE);
> > > > > >>> + }
> > > > > >>> + }
> > > > > >>> +
> > > > > >>> + if (!fd_string) {
> > > > > >>> + fprintf(stderr, "Unix file descriptor not present.\n");
> > > > > >>> + print_help_and_exit(argv[0], EXIT_FAILURE);
> > > > > >>> + }
> > > > > >>> +
> > > > > >>> + unix_fd = atoi(fd_string);
> > > > > >>> +
> > > > > >>> + if (!unix_fd) {
> > > > > >>> + fprintf(stderr, "Unix file descriptor is invalid.\n");
> > > > > >>> + return EXIT_FAILURE;
> > > > > >>> + }
> > > > > >>> +
> > > > > >>> + ebpf_rss_init(&ctx);
> > > > > >>> + if (!ebpf_rss_load(&ctx)) {
> > > > > >>> + fprintf(stderr, "Can't load ebpf.\n");
> > > > > >>> + return EXIT_FAILURE;
> > > > > >>> + }
> > > > > >>> + fds[0] = ctx.program_fd;
> > > > > >>> + fds[1] = ctx.map_configuration;
> > > > > >>> +
> > > > > >>> + ret = send_fds(unix_fd, fds, EBPF_RSS_MAX_FDS);
> > > > > >>> + if (ret < 0) {
> > > > > >>> + fprintf(stderr, "Issue while sending fds: %s.\n", strerror(errno));
> > > > > >>> + }
> > > > > >>> +
> > > > > >>> + ebpf_rss_unload(&ctx);
> > > > > >>> +
> > > > > >>> + return ret < 0 ? EXIT_FAILURE : EXIT_SUCCESS;
> > > > > >>> +}
> > > > > >>> +
> > > > > >>> diff --git a/meson.build b/meson.build
> > > > > >>> index 257e51d91b..913aa1fee5 100644
> > > > > >>> --- a/meson.build
> > > > > >>> +++ b/meson.build
> > > > > >>> @@ -1033,19 +1033,22 @@ if not get_option('fuse_lseek').disabled()
> > > > > >>> endif
> > > > > >>>
> > > > > >>> # libbpf
> > > > > >>> -libbpf = dependency('libbpf', required: get_option('bpf'), method: 'pkg-config')
> > > > > >>> -if libbpf.found() and not cc.links('''
> > > > > >>> - #include <bpf/libbpf.h>
> > > > > >>> - int main(void)
> > > > > >>> - {
> > > > > >>> - bpf_object__destroy_skeleton(NULL);
> > > > > >>> - return 0;
> > > > > >>> - }''', dependencies: libbpf)
> > > > > >>> - libbpf = not_found
> > > > > >>> - if get_option('bpf').enabled()
> > > > > >>> - error('libbpf skeleton test failed')
> > > > > >>> - else
> > > > > >>> - warning('libbpf skeleton test failed, disabling')
> > > > > >>> +libbpf = not_found
> > > > > >>> +if targetos == 'linux'
> > > > > >>> + libbpf = dependency('libbpf', required: get_option('bpf'), method: 'pkg-config')
> > > > > >>> + if libbpf.found() and not cc.links('''
> > > > > >>> + #include <bpf/libbpf.h>
> > > > > >>> + int main(void)
> > > > > >>> + {
> > > > > >>> + bpf_object__destroy_skeleton(NULL);
> > > > > >>
> > > > > >> Do we need to test whether the bpf can do mmap() here?
> > > > > >>
> > > > > >> Thanks
> > > > > >>
> > > > > >>
> > > > > >>> + return 0;
> > > > > >>> + }''', dependencies: libbpf)
> > > > > >>> + libbpf = not_found
> > > > > >>> + if get_option('bpf').enabled()
> > > > > >>> + error('libbpf skeleton test failed')
> > > > > >>> + else
> > > > > >>> + warning('libbpf skeleton test failed, disabling')
> > > > > >>> + endif
> > > > > >>> endif
> > > > > >>> endif
> > > > > >>>
> > > > > >>> @@ -2423,6 +2426,14 @@ if have_tools
> > > > > >>> dependencies: [authz, crypto, io, qom, qemuutil,
> > > > > >>> libcap_ng, mpathpersist],
> > > > > >>> install: true)
> > > > > >>> +
> > > > > >>> + if libbpf.found()
> > > > > >>> + executable('qemu-ebpf-rss-helper', files(
> > > > > >>> + 'ebpf/qemu-ebpf-rss-helper.c', 'ebpf/ebpf_rss.c'),
> > > > > >>> + dependencies: [qemuutil, libbpf, glib],
> > > > > >>> + install: true,
> > > > > >>> + install_dir: get_option('libexecdir'))
> > > > > >>> + endif
> > > > > >>> endif
> > > > > >>>
> > > > > >>> if 'CONFIG_IVSHMEM' in config_host
> > > > >
> > > >
> > >
> >
>
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 4/5] ebpf_rss_helper: Added helper for eBPF RSS.
2021-09-09 23:43 ` Yuri Benditovich
@ 2021-09-10 1:37 ` Jason Wang
0 siblings, 0 replies; 60+ messages in thread
From: Jason Wang @ 2021-09-10 1:37 UTC (permalink / raw)
To: Yuri Benditovich
Cc: Andrew Melnychenko, Daniel P. Berrangé,
Michael S . Tsirkin, qemu-devel, Markus Armbruster,
Yan Vugenfirer, Eric Blake
On Fri, Sep 10, 2021 at 7:44 AM Yuri Benditovich
<yuri.benditovich@daynix.com> wrote:
>
> On Thu, Sep 9, 2021 at 4:16 AM Jason Wang <jasowang@redhat.com> wrote:
> >
> > On Thu, Sep 9, 2021 at 8:00 AM Yuri Benditovich
> > <yuri.benditovich@daynix.com> wrote:
> > >
> > > On Wed, Sep 8, 2021 at 6:45 AM Jason Wang <jasowang@redhat.com> wrote:
> > > >
> > > > On Tue, Sep 7, 2021 at 6:40 PM Yuri Benditovich
> > > > <yuri.benditovich@daynix.com> wrote:
> > > > >
> > > > > On Wed, Sep 1, 2021 at 9:42 AM Jason Wang <jasowang@redhat.com> wrote:
> > > > > >
> > > > > >
> > > > > > 在 2021/8/31 上午1:07, Yuri Benditovich 写道:
> > > > > > > On Fri, Aug 20, 2021 at 6:41 AM Jason Wang <jasowang@redhat.com> wrote:
> > > > > > >>
> > > > > > >> 在 2021/7/13 下午11:37, Andrew Melnychenko 写道:
> > > > > > >>> Helper program. Loads eBPF RSS program and maps and passes them through unix socket.
> > > > > > >>> Libvirt may launch this helper and pass eBPF fds to qemu virtio-net.
> > > > > > >>
> > > > > > >> I wonder if this can be done as helper for TAP/bridge.
> > > > > > >>
> > > > > > >> E.g it's the qemu to launch those helper with set-uid.
> > > > > > >>
> > > > > > >> Then libvirt won't even need to care about that?
> > > > > > >>
> > > > > > > There are pros and cons for such a solution with set-uid.
> > > > > > > From my point of view one of the cons is that set-uid is efficient
> > > > > > > only at install time so the coexistence of different qemu builds (and
> > > > > > > different helpers for each one) is kind of problematic.
> > > > > > > With the current solution this does not present any problem: the
> > > > > > > developer can have several different builds, each one automatically
> > > > > > > has its own helper and there is no conflict between these builds and
> > > > > > > between these builds and installed qemu package. Changing the
> > > > > > > 'emulator' in the libvirt profile automatically brings the proper
> > > > > > > helper to work.
> > > > > >
> > > > > >
> > > > > > I'm not sure I get you here. We can still have default/sample helper to
> > > > > > make sure it works for different builds.
> > > > > >
> > > > > > If we can avoid the involvement of libvirt, that would be better.
> > > > >
> > > > > Hi Jason,
> > > > >
> > > > > Indeed I did not get the idea, can you please explain it in more
> > > > > details (as detailed as possible to avoid future misunderstanding),
> > > > > especially how exactly we can use the set-uid and what is the 'default' helper.
> > > > > We also would prefer to do everything from qemu but we do not see how
> > > > > we can do that.
> > > >
> > > >
> > > Some more questions to understand the idea better:
> > > > Something like:
> > > >
> > > > 1) -netdev tap,rss_helper=/path/to/name
> > >
> > > So, on each editing of 'emulator' in the xml the helper path should
> > > be set manually or be default?
> >
> > It could done manually, or we can have a default path.
> >
> > >
> > > > 2) having a sample/default helper implemented in Qemu
> > >
> > > Does it mean the default helper is the code in the qemu (without
> > > running additional executable, like it does today)
> >
> > Yes.
> If the "default helper" is just a keyword and it is like what we have
> today (i.e. part of qemu)
Yes, it's something like "-netdev tap,script=no"
> it can't work under libvirt and should never
> be used by libvirt.
Any reason for this?
> >
> > or this is qemu
> > > itself with dedicated command line?
> > > As far as I remember Daniel had strong objections of ever running qemu
> > > with capabilities
> >
> > Qemu won't run with capabilities but the helper.
> So under libvirt the helper is always separate executable and not
> "default helper"
That's fine, we can ship the helper as an independent package I think.
>
> >
> > >
> > > > 3) we can introduce something special path like "default", then if
> > > > -netdev tap,rss_helper="default" is specified, qemu will use the
> > > > sample helper
> > >
> > > Probably this is not so important but the rss helper and rss in
> > > general has no relation to netdev, much more they are related to
> > > virtio-net
> >
> > So I think the reason for this is that we currently only support
> > eBPF/RSS for tap.
>
> This is just because only tap supports respective ioctls.
>
> >
> > >
> > > >
> > > > So we have:
> > > > 1) set set-uid for the helper
> > > Who and when does set-uid to the helper binary? Only installer or
> > > libvirt can do that, correct?
> >
> > Yes, it could be done the installer, or other system provision tools.
>
> So this changes the rule of the game: currently libvirt runs helpers
> that require privileges and qemu runs helpers that do not require any
> privileges (like TPM).
I don't understand here, if the helper doesn't need any privilege, why
not do that in qemu itself?
> If we follow your suggestion - only for RSS we will create the helper
> that must be used with set-uid.
> Who are stakeholders we need to have a consensus with?
I think we need inputs from libvirt guys. Daniel, any idea on this?
Thanks
>
> >
> > >
> > > > 2) libvirt may just choose to launch the default helper
> > > All this discussion is to avoid launching the helper from libvirt, correct?
> >
> > Sorry, it's a typo. I meant, libvirt launch qemu, and then qemu will
> > launch the helper.
> >
> > Thanks
> >
> > >
> > > >
> > > > >
> > > > > Our main points (what should be addressed):
> > > > > - qemu should be able to load ebpf and use the maps when it runs from
> > > > > libvirt (without special caps) and standalone (with caps)
> > > >
> > > > This is solved by leaving the privileged operations to the helper with set-uid.
> > > >
> > > > > - it is possible that there are different qemu builds on the machine,
> > > > > one of them might be installed, their ebpf's might be different and
> > > > > the interface between qemu and ebpf (exact content of maps and number
> > > > > of maps)
> > > >
> > > > We can use different helpers in this way.
> > > >
> > > > > - qemu configures the RSS dynamically according to the commands
> > > > > provided by the guest
> > > >
> > > > Consider we decided to use mmap() based maps, this is not an issue.
> > > >
> > > > Or am I missing something?
> > > >
> > > > Thanks
> > > >
> > > > >
> > > > > Thanks in advance
> > > > > Yuri
> > > > >
> > > > > >
> > > > > > Thanks
> > > > > >
> > > > > >
> > > > > > >
> > > > > > >>> Also, libbpf dependency now exclusively for Linux.
> > > > > > >>> Libbpf is used for eBPF RSS steering, which is supported only by Linux TAP.
> > > > > > >>> There is no reason yet to build eBPF loader and helper for non Linux systems,
> > > > > > >>> even if libbpf is present.
> > > > > > >>>
> > > > > > >>> Signed-off-by: Andrew Melnychenko <andrew@daynix.com>
> > > > > > >>> ---
> > > > > > >>> ebpf/qemu-ebpf-rss-helper.c | 130 ++++++++++++++++++++++++++++++++++++
> > > > > > >>> meson.build | 37 ++++++----
> > > > > > >>> 2 files changed, 154 insertions(+), 13 deletions(-)
> > > > > > >>> create mode 100644 ebpf/qemu-ebpf-rss-helper.c
> > > > > > >>>
> > > > > > >>> diff --git a/ebpf/qemu-ebpf-rss-helper.c b/ebpf/qemu-ebpf-rss-helper.c
> > > > > > >>> new file mode 100644
> > > > > > >>> index 0000000000..fe68758f57
> > > > > > >>> --- /dev/null
> > > > > > >>> +++ b/ebpf/qemu-ebpf-rss-helper.c
> > > > > > >>> @@ -0,0 +1,130 @@
> > > > > > >>> +/*
> > > > > > >>> + * eBPF RSS Helper
> > > > > > >>> + *
> > > > > > >>> + * Developed by Daynix Computing LTD (http://www.daynix.com)
> > > > > > >>> + *
> > > > > > >>> + * Authors:
> > > > > > >>> + * Andrew Melnychenko <andrew@daynix.com>
> > > > > > >>> + *
> > > > > > >>> + * This work is licensed under the terms of the GNU GPL, version 2. See
> > > > > > >>> + * the COPYING file in the top-level directory.
> > > > > > >>> + *
> > > > > > >>> + * Description: This is helper program for libvirtd.
> > > > > > >>> + * It loads eBPF RSS program and passes fds through unix socket.
> > > > > > >>> + * Built by meson, target - 'qemu-ebpf-rss-helper'.
> > > > > > >>> + */
> > > > > > >>> +
> > > > > > >>> +#include <stdio.h>
> > > > > > >>> +#include <stdint.h>
> > > > > > >>> +#include <stdlib.h>
> > > > > > >>> +#include <stdbool.h>
> > > > > > >>> +#include <getopt.h>
> > > > > > >>> +#include <memory.h>
> > > > > > >>> +#include <errno.h>
> > > > > > >>> +#include <sys/socket.h>
> > > > > > >>> +
> > > > > > >>> +#include "ebpf_rss.h"
> > > > > > >>> +
> > > > > > >>> +#include "qemu-helper-stamp.h"
> > > > > > >>> +
> > > > > > >>> +void QEMU_HELPER_STAMP(void) {}
> > > > > > >>> +
> > > > > > >>> +static int send_fds(int socket, int *fds, int n)
> > > > > > >>> +{
> > > > > > >>> + struct msghdr msg = {};
> > > > > > >>> + struct cmsghdr *cmsg = NULL;
> > > > > > >>> + char buf[CMSG_SPACE(n * sizeof(int))];
> > > > > > >>> + char dummy_buffer = 0;
> > > > > > >>> + struct iovec io = { .iov_base = &dummy_buffer,
> > > > > > >>> + .iov_len = sizeof(dummy_buffer) };
> > > > > > >>> +
> > > > > > >>> + memset(buf, 0, sizeof(buf));
> > > > > > >>> +
> > > > > > >>> + msg.msg_iov = &io;
> > > > > > >>> + msg.msg_iovlen = 1;
> > > > > > >>> + msg.msg_control = buf;
> > > > > > >>> + msg.msg_controllen = sizeof(buf);
> > > > > > >>> +
> > > > > > >>> + cmsg = CMSG_FIRSTHDR(&msg);
> > > > > > >>> + cmsg->cmsg_level = SOL_SOCKET;
> > > > > > >>> + cmsg->cmsg_type = SCM_RIGHTS;
> > > > > > >>> + cmsg->cmsg_len = CMSG_LEN(n * sizeof(int));
> > > > > > >>> +
> > > > > > >>> + memcpy(CMSG_DATA(cmsg), fds, n * sizeof(int));
> > > > > > >>> +
> > > > > > >>> + return sendmsg(socket, &msg, 0);
> > > > > > >>> +}
> > > > > > >>> +
> > > > > > >>> +static void print_help_and_exit(const char *prog, int exitcode)
> > > > > > >>> +{
> > > > > > >>> + fprintf(stderr, "%s - load eBPF RSS program for qemu and pass eBPF fds"
> > > > > > >>> + " through unix socket.\n", prog);
> > > > > > >>> + fprintf(stderr, "\t--fd <num>, -f <num> - unix socket file descriptor"
> > > > > > >>> + " used to pass eBPF fds.\n");
> > > > > > >>> + fprintf(stderr, "\t--help, -h - this help.\n");
> > > > > > >>> + exit(exitcode);
> > > > > > >>> +}
> > > > > > >>> +
> > > > > > >>> +int main(int argc, char **argv)
> > > > > > >>> +{
> > > > > > >>> + char *fd_string = NULL;
> > > > > > >>> + int unix_fd = 0;
> > > > > > >>> + struct EBPFRSSContext ctx = {};
> > > > > > >>> + int fds[EBPF_RSS_MAX_FDS] = {};
> > > > > > >>> + int ret = -1;
> > > > > > >>> +
> > > > > > >>> + for (;;) {
> > > > > > >>> + int c;
> > > > > > >>> + static struct option long_options[] = {
> > > > > > >>> + {"help", no_argument, 0, 'h'},
> > > > > > >>> + {"fd", required_argument, 0, 'f'},
> > > > > > >>> + {0, 0, 0, 0}
> > > > > > >>> + };
> > > > > > >>> + c = getopt_long(argc, argv, "hf:",
> > > > > > >>> + long_options, NULL);
> > > > > > >>> +
> > > > > > >>> + if (c == -1) {
> > > > > > >>> + break;
> > > > > > >>> + }
> > > > > > >>> +
> > > > > > >>> + switch (c) {
> > > > > > >>> + case 'f':
> > > > > > >>> + fd_string = optarg;
> > > > > > >>> + break;
> > > > > > >>> + case 'h':
> > > > > > >>> + default:
> > > > > > >>> + print_help_and_exit(argv[0],
> > > > > > >>> + c == 'h' ? EXIT_SUCCESS : EXIT_FAILURE);
> > > > > > >>> + }
> > > > > > >>> + }
> > > > > > >>> +
> > > > > > >>> + if (!fd_string) {
> > > > > > >>> + fprintf(stderr, "Unix file descriptor not present.\n");
> > > > > > >>> + print_help_and_exit(argv[0], EXIT_FAILURE);
> > > > > > >>> + }
> > > > > > >>> +
> > > > > > >>> + unix_fd = atoi(fd_string);
> > > > > > >>> +
> > > > > > >>> + if (!unix_fd) {
> > > > > > >>> + fprintf(stderr, "Unix file descriptor is invalid.\n");
> > > > > > >>> + return EXIT_FAILURE;
> > > > > > >>> + }
> > > > > > >>> +
> > > > > > >>> + ebpf_rss_init(&ctx);
> > > > > > >>> + if (!ebpf_rss_load(&ctx)) {
> > > > > > >>> + fprintf(stderr, "Can't load ebpf.\n");
> > > > > > >>> + return EXIT_FAILURE;
> > > > > > >>> + }
> > > > > > >>> + fds[0] = ctx.program_fd;
> > > > > > >>> + fds[1] = ctx.map_configuration;
> > > > > > >>> +
> > > > > > >>> + ret = send_fds(unix_fd, fds, EBPF_RSS_MAX_FDS);
> > > > > > >>> + if (ret < 0) {
> > > > > > >>> + fprintf(stderr, "Issue while sending fds: %s.\n", strerror(errno));
> > > > > > >>> + }
> > > > > > >>> +
> > > > > > >>> + ebpf_rss_unload(&ctx);
> > > > > > >>> +
> > > > > > >>> + return ret < 0 ? EXIT_FAILURE : EXIT_SUCCESS;
> > > > > > >>> +}
> > > > > > >>> +
> > > > > > >>> diff --git a/meson.build b/meson.build
> > > > > > >>> index 257e51d91b..913aa1fee5 100644
> > > > > > >>> --- a/meson.build
> > > > > > >>> +++ b/meson.build
> > > > > > >>> @@ -1033,19 +1033,22 @@ if not get_option('fuse_lseek').disabled()
> > > > > > >>> endif
> > > > > > >>>
> > > > > > >>> # libbpf
> > > > > > >>> -libbpf = dependency('libbpf', required: get_option('bpf'), method: 'pkg-config')
> > > > > > >>> -if libbpf.found() and not cc.links('''
> > > > > > >>> - #include <bpf/libbpf.h>
> > > > > > >>> - int main(void)
> > > > > > >>> - {
> > > > > > >>> - bpf_object__destroy_skeleton(NULL);
> > > > > > >>> - return 0;
> > > > > > >>> - }''', dependencies: libbpf)
> > > > > > >>> - libbpf = not_found
> > > > > > >>> - if get_option('bpf').enabled()
> > > > > > >>> - error('libbpf skeleton test failed')
> > > > > > >>> - else
> > > > > > >>> - warning('libbpf skeleton test failed, disabling')
> > > > > > >>> +libbpf = not_found
> > > > > > >>> +if targetos == 'linux'
> > > > > > >>> + libbpf = dependency('libbpf', required: get_option('bpf'), method: 'pkg-config')
> > > > > > >>> + if libbpf.found() and not cc.links('''
> > > > > > >>> + #include <bpf/libbpf.h>
> > > > > > >>> + int main(void)
> > > > > > >>> + {
> > > > > > >>> + bpf_object__destroy_skeleton(NULL);
> > > > > > >>
> > > > > > >> Do we need to test whether the bpf can do mmap() here?
> > > > > > >>
> > > > > > >> Thanks
> > > > > > >>
> > > > > > >>
> > > > > > >>> + return 0;
> > > > > > >>> + }''', dependencies: libbpf)
> > > > > > >>> + libbpf = not_found
> > > > > > >>> + if get_option('bpf').enabled()
> > > > > > >>> + error('libbpf skeleton test failed')
> > > > > > >>> + else
> > > > > > >>> + warning('libbpf skeleton test failed, disabling')
> > > > > > >>> + endif
> > > > > > >>> endif
> > > > > > >>> endif
> > > > > > >>>
> > > > > > >>> @@ -2423,6 +2426,14 @@ if have_tools
> > > > > > >>> dependencies: [authz, crypto, io, qom, qemuutil,
> > > > > > >>> libcap_ng, mpathpersist],
> > > > > > >>> install: true)
> > > > > > >>> +
> > > > > > >>> + if libbpf.found()
> > > > > > >>> + executable('qemu-ebpf-rss-helper', files(
> > > > > > >>> + 'ebpf/qemu-ebpf-rss-helper.c', 'ebpf/ebpf_rss.c'),
> > > > > > >>> + dependencies: [qemuutil, libbpf, glib],
> > > > > > >>> + install: true,
> > > > > > >>> + install_dir: get_option('libexecdir'))
> > > > > > >>> + endif
> > > > > > >>> endif
> > > > > > >>>
> > > > > > >>> if 'CONFIG_IVSHMEM' in config_host
> > > > > >
> > > > >
> > > >
> > >
> >
>
^ permalink raw reply [flat|nested] 60+ messages in thread
* [PATCH 5/5] qmp: Added qemu-ebpf-rss-path command.
2021-07-13 15:37 [PATCH 0/5] ebpf: Added ebpf helper for libvirtd Andrew Melnychenko
` (3 preceding siblings ...)
2021-07-13 15:37 ` [PATCH 4/5] ebpf_rss_helper: Added helper for eBPF RSS Andrew Melnychenko
@ 2021-07-13 15:37 ` Andrew Melnychenko
2021-08-07 12:54 ` Markus Armbruster
2021-07-22 8:37 ` [PATCH 0/5] ebpf: Added ebpf helper for libvirtd Andrew Melnichenko
2021-08-20 3:43 ` Jason Wang
6 siblings, 1 reply; 60+ messages in thread
From: Andrew Melnychenko @ 2021-07-13 15:37 UTC (permalink / raw)
To: mst, yuri.benditovich, jasowang, armbru, eblake, berrange; +Cc: yan, qemu-devel
New qmp command to query ebpf helper.
It's crucial that qemu and helper are in sync and in touch.
Technically helper should pass eBPF fds that qemu may accept.
And different qemu's builds may have different eBPF programs and helpers.
Qemu returns helper that should "fit" to virtio-net.
Signed-off-by: Andrew Melnychenko <andrew@daynix.com>
---
monitor/qmp-cmds.c | 32 ++++++++++++++++++++++++++++++++
qapi/misc.json | 33 +++++++++++++++++++++++++++++++++
2 files changed, 65 insertions(+)
diff --git a/monitor/qmp-cmds.c b/monitor/qmp-cmds.c
index f7d64a6457..c042ab5466 100644
--- a/monitor/qmp-cmds.c
+++ b/monitor/qmp-cmds.c
@@ -40,6 +40,7 @@
#include "qapi/qmp/qerror.h"
#include "hw/mem/memory-device.h"
#include "hw/acpi/acpi_dev_interface.h"
+#include "qemu-helper-stamp-utils.h"
NameInfo *qmp_query_name(Error **errp)
{
@@ -351,3 +352,34 @@ void qmp_display_reload(DisplayReloadOptions *arg, Error **errp)
abort();
}
}
+
+HelperPathList *qmp_query_helper_paths(Error **errp)
+{
+ HelperPathList *ret = NULL;
+ struct {
+ const char *helper;
+ bool check_stamp;
+ } helpers_list[] = {
+#ifdef CONFIG_EBPF
+ { "qemu-ebpf-rss-helper", true },
+#endif
+ { "qemu-pr-helper", false },
+ { "qemu-bridge-helper", false },
+ { NULL, false },
+ }, *helper_iter;
+ helper_iter = helpers_list;
+
+ for (; helper_iter->helper != NULL; ++helper_iter) {
+ char *path = qemu_find_helper(helper_iter->helper,
+ helper_iter->check_stamp);
+ if (path) {
+ HelperPath *helper = g_new0(HelperPath, 1);
+ helper->name = g_strdup(helper_iter->helper);
+ helper->path = path;
+
+ QAPI_LIST_PREPEND(ret, helper);
+ }
+ }
+
+ return ret;
+}
diff --git a/qapi/misc.json b/qapi/misc.json
index 156f98203e..9aaf8fbcca 100644
--- a/qapi/misc.json
+++ b/qapi/misc.json
@@ -519,3 +519,36 @@
'data': { '*option': 'str' },
'returns': ['CommandLineOptionInfo'],
'allow-preconfig': true }
+
+##
+# @HelperPath:
+#
+# Name of the helper and binary location.
+##
+{ 'struct': 'HelperPath',
+ 'data': {'name': 'str', 'path': 'str'} }
+
+##
+# @query-helper-paths:
+#
+# Query helper paths. Initially, this command was added for
+# qemu-ebpf-rss-helper. The qemu would check "the stamp" and
+# returns proper helper.
+#
+# Returns: list of object that contains name and path for helper.
+#
+# Since: 6.1
+#
+# Example:
+#
+# -> { "execute": "query-helper-paths" }
+# <- { "return": [
+# {
+# "name": "qemu-ebpf-rss-helper",
+# "path": "/usr/local/libexec/qemu-ebpf-rss-helper"
+# }
+# ]
+# }
+#
+##
+{ 'command': 'query-helper-paths', 'returns': ['HelperPath'] }
--
2.31.1
^ permalink raw reply related [flat|nested] 60+ messages in thread
* Re: [PATCH 5/5] qmp: Added qemu-ebpf-rss-path command.
2021-07-13 15:37 ` [PATCH 5/5] qmp: Added qemu-ebpf-rss-path command Andrew Melnychenko
@ 2021-08-07 12:54 ` Markus Armbruster
2021-08-10 11:58 ` Andrew Melnichenko
0 siblings, 1 reply; 60+ messages in thread
From: Markus Armbruster @ 2021-08-07 12:54 UTC (permalink / raw)
To: Andrew Melnychenko
Cc: berrange, mst, jasowang, qemu-devel, yuri.benditovich, yan, eblake
Andrew Melnychenko <andrew@daynix.com> writes:
> New qmp command to query ebpf helper.
> It's crucial that qemu and helper are in sync and in touch.
> Technically helper should pass eBPF fds that qemu may accept.
> And different qemu's builds may have different eBPF programs and helpers.
> Qemu returns helper that should "fit" to virtio-net.
>
> Signed-off-by: Andrew Melnychenko <andrew@daynix.com>
[...]
> diff --git a/qapi/misc.json b/qapi/misc.json
> index 156f98203e..9aaf8fbcca 100644
> --- a/qapi/misc.json
> +++ b/qapi/misc.json
> @@ -519,3 +519,36 @@
> 'data': { '*option': 'str' },
> 'returns': ['CommandLineOptionInfo'],
> 'allow-preconfig': true }
> +
> +##
> +# @HelperPath:
> +#
> +# Name of the helper and binary location.
> +##
> +{ 'struct': 'HelperPath',
> + 'data': {'name': 'str', 'path': 'str'} }
> +
> +##
> +# @query-helper-paths:
> +#
> +# Query helper paths. Initially, this command was added for
> +# qemu-ebpf-rss-helper. The qemu would check "the stamp" and
> +# returns proper helper.
> +#
> +# Returns: list of object that contains name and path for helper.
> +#
> +# Since: 6.1
> +#
> +# Example:
> +#
> +# -> { "execute": "query-helper-paths" }
> +# <- { "return": [
> +# {
> +# "name": "qemu-ebpf-rss-helper",
> +# "path": "/usr/local/libexec/qemu-ebpf-rss-helper"
> +# }
> +# ]
> +# }
> +#
> +##
> +{ 'command': 'query-helper-paths', 'returns': ['HelperPath'] }
Hmm.
I understand the desire to help management applications to use the right
helper. But I'm not sure this command is actually useful. The helper
may or may not be installed at the path compiled into QEMU.
What happens when you use the wrong helper?
Even if we conclude this is the right approach for this helper, we still
need to review the other helpers to see which of them we should have
query-helper-paths cover.
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 5/5] qmp: Added qemu-ebpf-rss-path command.
2021-08-07 12:54 ` Markus Armbruster
@ 2021-08-10 11:58 ` Andrew Melnichenko
2021-08-24 6:41 ` Markus Armbruster
0 siblings, 1 reply; 60+ messages in thread
From: Andrew Melnichenko @ 2021-08-10 11:58 UTC (permalink / raw)
To: Markus Armbruster
Cc: Daniel P. Berrangé,
Michael S. Tsirkin, Jason Wang, qemu-devel, Yuri Benditovich,
Yan Vugenfirer, Eric Blake
[-- Attachment #1: Type: text/plain, Size: 2464 bytes --]
Hi,
> The helper may or may not be installed at the path compiled into QEMU.
>
Yes, so the helper will not be called - QEMU will try to initiate eBPF RSS
or use "in-qemu" RSS.
What happens when you use the wrong helper?
>
UB - in most cases, eBPF program will work with wrong configurations.
That's why the stamp was added.
query-helper-paths checks the stamp only for RSS helper.
On Sat, Aug 7, 2021 at 3:54 PM Markus Armbruster <armbru@redhat.com> wrote:
> Andrew Melnychenko <andrew@daynix.com> writes:
>
> > New qmp command to query ebpf helper.
> > It's crucial that qemu and helper are in sync and in touch.
> > Technically helper should pass eBPF fds that qemu may accept.
> > And different qemu's builds may have different eBPF programs and helpers.
> > Qemu returns helper that should "fit" to virtio-net.
> >
> > Signed-off-by: Andrew Melnychenko <andrew@daynix.com>
>
> [...]
>
> > diff --git a/qapi/misc.json b/qapi/misc.json
> > index 156f98203e..9aaf8fbcca 100644
> > --- a/qapi/misc.json
> > +++ b/qapi/misc.json
> > @@ -519,3 +519,36 @@
> > 'data': { '*option': 'str' },
> > 'returns': ['CommandLineOptionInfo'],
> > 'allow-preconfig': true }
> > +
> > +##
> > +# @HelperPath:
> > +#
> > +# Name of the helper and binary location.
> > +##
> > +{ 'struct': 'HelperPath',
> > + 'data': {'name': 'str', 'path': 'str'} }
> > +
> > +##
> > +# @query-helper-paths:
> > +#
> > +# Query helper paths. Initially, this command was added for
> > +# qemu-ebpf-rss-helper. The qemu would check "the stamp" and
> > +# returns proper helper.
> > +#
> > +# Returns: list of object that contains name and path for helper.
> > +#
> > +# Since: 6.1
> > +#
> > +# Example:
> > +#
> > +# -> { "execute": "query-helper-paths" }
> > +# <- { "return": [
> > +# {
> > +# "name": "qemu-ebpf-rss-helper",
> > +# "path": "/usr/local/libexec/qemu-ebpf-rss-helper"
> > +# }
> > +# ]
> > +# }
> > +#
> > +##
> > +{ 'command': 'query-helper-paths', 'returns': ['HelperPath'] }
>
>
> Hmm.
>
> I understand the desire to help management applications to use the right
> helper. But I'm not sure this command is actually useful. The helper
> may or may not be installed at the path compiled into QEMU.
>
> What happens when you use the wrong helper?
>
> Even if we conclude this is the right approach for this helper, we still
> need to review the other helpers to see which of them we should have
> query-helper-paths cover.
>
>
[-- Attachment #2: Type: text/html, Size: 3871 bytes --]
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 5/5] qmp: Added qemu-ebpf-rss-path command.
2021-08-10 11:58 ` Andrew Melnichenko
@ 2021-08-24 6:41 ` Markus Armbruster
2021-08-25 18:45 ` Andrew Melnichenko
2021-08-29 20:13 ` Yuri Benditovich
0 siblings, 2 replies; 60+ messages in thread
From: Markus Armbruster @ 2021-08-24 6:41 UTC (permalink / raw)
To: Andrew Melnichenko
Cc: Daniel P. Berrangé,
Michael S. Tsirkin, Jason Wang, qemu-devel, Yuri Benditovich,
Yan Vugenfirer, Eric Blake
Andrew Melnichenko <andrew@daynix.com> writes:
> Hi,
>
>> The helper may or may not be installed at the path compiled into QEMU.
>>
> Yes, so the helper will not be called - QEMU will try to initiate eBPF RSS
> or use "in-qemu" RSS.
My point is: the proposed command's mission is to help the management
application run the right helper. However, its advice is *unreliable*.
It may point to the wrong helper, or to nothing at all. The right
helper may still exist elsewhere.
I suspect you're trying to address the problem at the wrong level.
Similar versioning issues exist with other helpers. We've been doing
fine without QEMU providing unreliable advice on where they might sit in
the file system. What makes this one different?
>> What happens when you use the wrong helper?
>>
> UB - in most cases, eBPF program will work with wrong configurations.
> That's why the stamp was added.
>
> query-helper-paths checks the stamp only for RSS helper.
I have no idea what you're talking about :)
My best guess is that you're trying to tell me that attempting to work
with the wrong helper will fail cleanly due to some stamp check. That
would be nice.
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 5/5] qmp: Added qemu-ebpf-rss-path command.
2021-08-24 6:41 ` Markus Armbruster
@ 2021-08-25 18:45 ` Andrew Melnichenko
2021-08-26 4:53 ` Markus Armbruster
2021-08-29 20:13 ` Yuri Benditovich
1 sibling, 1 reply; 60+ messages in thread
From: Andrew Melnichenko @ 2021-08-25 18:45 UTC (permalink / raw)
To: Markus Armbruster
Cc: Daniel P. Berrangé,
Michael S. Tsirkin, Jason Wang, qemu-devel, Yuri Benditovich,
Yan Vugenfirer, Eric Blake
[-- Attachment #1: Type: text/plain, Size: 1522 bytes --]
Hi,
Yes, the stamp check was added.
So the qemu emulator should return a suitable RSS BPF helper or nothing.
Each qemu emulator may have a different helper that suits it.
So, the idea is to ask for the helper from qemu.
On Tue, Aug 24, 2021 at 9:41 AM Markus Armbruster <armbru@redhat.com> wrote:
> Andrew Melnichenko <andrew@daynix.com> writes:
>
> > Hi,
> >
> >> The helper may or may not be installed at the path compiled into QEMU.
> >>
> > Yes, so the helper will not be called - QEMU will try to initiate eBPF
> RSS
> > or use "in-qemu" RSS.
>
> My point is: the proposed command's mission is to help the management
> application run the right helper. However, its advice is *unreliable*.
> It may point to the wrong helper, or to nothing at all. The right
> helper may still exist elsewhere.
>
> I suspect you're trying to address the problem at the wrong level.
>
> Similar versioning issues exist with other helpers. We've been doing
> fine without QEMU providing unreliable advice on where they might sit in
> the file system. What makes this one different?
>
> >> What happens when you use the wrong helper?
> >>
> > UB - in most cases, eBPF program will work with wrong configurations.
> > That's why the stamp was added.
> >
> > query-helper-paths checks the stamp only for RSS helper.
>
> I have no idea what you're talking about :)
>
> My best guess is that you're trying to tell me that attempting to work
> with the wrong helper will fail cleanly due to some stamp check. That
> would be nice.
>
>
[-- Attachment #2: Type: text/html, Size: 2105 bytes --]
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 5/5] qmp: Added qemu-ebpf-rss-path command.
2021-08-25 18:45 ` Andrew Melnichenko
@ 2021-08-26 4:53 ` Markus Armbruster
0 siblings, 0 replies; 60+ messages in thread
From: Markus Armbruster @ 2021-08-26 4:53 UTC (permalink / raw)
To: Andrew Melnichenko
Cc: Daniel P. Berrangé,
Michael S. Tsirkin, Jason Wang, qemu-devel, Yuri Benditovich,
Yan Vugenfirer, Eric Blake
Andrew Melnichenko <andrew@daynix.com> writes:
> Hi,
> Yes, the stamp check was added.
> So the qemu emulator should return a suitable RSS BPF helper or nothing.
> Each qemu emulator may have a different helper that suits it.
> So, the idea is to ask for the helper from qemu.
Please answer the question below.
>
> On Tue, Aug 24, 2021 at 9:41 AM Markus Armbruster <armbru@redhat.com> wrote:
>
>> Andrew Melnichenko <andrew@daynix.com> writes:
>>
>> > Hi,
>> >
>> >> The helper may or may not be installed at the path compiled into QEMU.
>> >>
>> > Yes, so the helper will not be called - QEMU will try to initiate eBPF
>> RSS
>> > or use "in-qemu" RSS.
>>
>> My point is: the proposed command's mission is to help the management
>> application run the right helper. However, its advice is *unreliable*.
>> It may point to the wrong helper, or to nothing at all. The right
>> helper may still exist elsewhere.
>>
>> I suspect you're trying to address the problem at the wrong level.
>>
>> Similar versioning issues exist with other helpers. We've been doing
>> fine without QEMU providing unreliable advice on where they might sit in
>> the file system. What makes this one different?
This one.
>> >> What happens when you use the wrong helper?
>> >>
>> > UB - in most cases, eBPF program will work with wrong configurations.
>> > That's why the stamp was added.
>> >
>> > query-helper-paths checks the stamp only for RSS helper.
>>
>> I have no idea what you're talking about :)
>>
>> My best guess is that you're trying to tell me that attempting to work
>> with the wrong helper will fail cleanly due to some stamp check. That
>> would be nice.
>>
>>
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 5/5] qmp: Added qemu-ebpf-rss-path command.
2021-08-24 6:41 ` Markus Armbruster
2021-08-25 18:45 ` Andrew Melnichenko
@ 2021-08-29 20:13 ` Yuri Benditovich
2021-08-30 6:10 ` Markus Armbruster
1 sibling, 1 reply; 60+ messages in thread
From: Yuri Benditovich @ 2021-08-29 20:13 UTC (permalink / raw)
To: Markus Armbruster
Cc: Andrew Melnichenko, Daniel P. Berrangé,
Michael S. Tsirkin, Jason Wang, qemu-devel, Yan Vugenfirer,
Eric Blake
On Tue, Aug 24, 2021 at 9:41 AM Markus Armbruster <armbru@redhat.com> wrote:
>
> Andrew Melnichenko <andrew@daynix.com> writes:
>
> > Hi,
> >
> >> The helper may or may not be installed at the path compiled into QEMU.
> >>
> > Yes, so the helper will not be called - QEMU will try to initiate eBPF RSS
> > or use "in-qemu" RSS.
>
> My point is: the proposed command's mission is to help the management
> application run the right helper. However, its advice is *unreliable*.
> It may point to the wrong helper, or to nothing at all. The right
> helper may still exist elsewhere.
Hi Markus,
Indeed the intention of this command is to return the proper helper.
Especially in the case of RSS helper this is *reliable* advice and it
points to the helper that was built together with QEMU, i.e. with the
same headers.
This was discussed earlier, for example in
https://lists.nongnu.org/archive/html/qemu-devel/2021-06/msg02248.html
>
> I suspect you're trying to address the problem at the wrong level.
What is the proper solution for the problem from your point of view?
>
> Similar versioning issues exist with other helpers. We've been doing
> fine without QEMU providing unreliable advice on where they might sit in
> the file system. What makes this one different?
This one is required to be *fully synchronized* with the existing build of QEMU.
Other helpers are probably less restrictive and do not have common
structures definitions with the QEMU, otherwise they would face the
same problem.
>
> >> What happens when you use the wrong helper?
Our intention is that libvirt should never use the wrong RSS helper.
But it does not have any ability to check which helper is compatible
with the QEMU.
QEMU can easily recognize the correct one.
> >>
> > UB - in most cases, eBPF program will work with wrong configurations.
> > That's why the stamp was added.
> >
> > query-helper-paths checks the stamp only for RSS helper.
>
> I have no idea what you're talking about :)
>
> My best guess is that you're trying to tell me that attempting to work
> with the wrong helper will fail cleanly due to some stamp check. That
> would be nice.
>
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 5/5] qmp: Added qemu-ebpf-rss-path command.
2021-08-29 20:13 ` Yuri Benditovich
@ 2021-08-30 6:10 ` Markus Armbruster
2021-08-30 7:51 ` Yuri Benditovich
0 siblings, 1 reply; 60+ messages in thread
From: Markus Armbruster @ 2021-08-30 6:10 UTC (permalink / raw)
To: Yuri Benditovich
Cc: Andrew Melnichenko, Daniel P. Berrangé,
Michael S. Tsirkin, Jason Wang, qemu-devel, Yan Vugenfirer,
Eric Blake
Yuri Benditovich <yuri.benditovich@daynix.com> writes:
> On Tue, Aug 24, 2021 at 9:41 AM Markus Armbruster <armbru@redhat.com> wrote:
>>
>> Andrew Melnichenko <andrew@daynix.com> writes:
>>
>> > Hi,
>> >
>> >> The helper may or may not be installed at the path compiled into QEMU.
>> >>
>> > Yes, so the helper will not be called - QEMU will try to initiate eBPF RSS
>> > or use "in-qemu" RSS.
>>
>> My point is: the proposed command's mission is to help the management
>> application run the right helper. However, its advice is *unreliable*.
>> It may point to the wrong helper, or to nothing at all. The right
>> helper may still exist elsewhere.
>
> Hi Markus,
> Indeed the intention of this command is to return the proper helper.
> Especially in the case of RSS helper this is *reliable* advice and it
> points to the helper that was built together with QEMU, i.e. with the
> same headers.
> This was discussed earlier, for example in
> https://lists.nongnu.org/archive/html/qemu-devel/2021-06/msg02248.html
>
>>
>> I suspect you're trying to address the problem at the wrong level.
>
> What is the proper solution for the problem from your point of view?
I'll explain in more detail, but first I'd like you to answer my
question below.
>> Similar versioning issues exist with other helpers. We've been doing
>> fine without QEMU providing unreliable advice on where they might sit in
>> the file system. What makes this one different?
>
> This one is required to be *fully synchronized* with the existing build of QEMU.
> Other helpers are probably less restrictive and do not have common
> structures definitions with the QEMU, otherwise they would face the
> same problem.
>
>>
>> >> What happens when you use the wrong helper?
>
> Our intention is that libvirt should never use the wrong RSS helper.
> But it does not have any ability to check which helper is compatible
> with the QEMU.
> QEMU can easily recognize the correct one.
You did not actually answer my question :)
So let's try again: if libvirt does use the wrong RSS helper, how does
the system behave?
>> >>
>> > UB - in most cases, eBPF program will work with wrong configurations.
>> > That's why the stamp was added.
>> >
>> > query-helper-paths checks the stamp only for RSS helper.
>>
>> I have no idea what you're talking about :)
>>
>> My best guess is that you're trying to tell me that attempting to work
>> with the wrong helper will fail cleanly due to some stamp check. That
>> would be nice.
>>
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 5/5] qmp: Added qemu-ebpf-rss-path command.
2021-08-30 6:10 ` Markus Armbruster
@ 2021-08-30 7:51 ` Yuri Benditovich
2021-08-30 8:13 ` Markus Armbruster
0 siblings, 1 reply; 60+ messages in thread
From: Yuri Benditovich @ 2021-08-30 7:51 UTC (permalink / raw)
To: Markus Armbruster
Cc: Andrew Melnichenko, Daniel P. Berrangé,
Michael S. Tsirkin, Jason Wang, qemu-devel, Yan Vugenfirer,
Eric Blake
On Mon, Aug 30, 2021 at 9:10 AM Markus Armbruster <armbru@redhat.com> wrote:
>
> Yuri Benditovich <yuri.benditovich@daynix.com> writes:
>
> > On Tue, Aug 24, 2021 at 9:41 AM Markus Armbruster <armbru@redhat.com> wrote:
> >>
> >> Andrew Melnichenko <andrew@daynix.com> writes:
> >>
> >> > Hi,
> >> >
> >> >> The helper may or may not be installed at the path compiled into QEMU.
> >> >>
> >> > Yes, so the helper will not be called - QEMU will try to initiate eBPF RSS
> >> > or use "in-qemu" RSS.
> >>
> >> My point is: the proposed command's mission is to help the management
> >> application run the right helper. However, its advice is *unreliable*.
> >> It may point to the wrong helper, or to nothing at all. The right
> >> helper may still exist elsewhere.
> >
> > Hi Markus,
> > Indeed the intention of this command is to return the proper helper.
> > Especially in the case of RSS helper this is *reliable* advice and it
> > points to the helper that was built together with QEMU, i.e. with the
> > same headers.
> > This was discussed earlier, for example in
> > https://lists.nongnu.org/archive/html/qemu-devel/2021-06/msg02248.html
> >
> >>
> >> I suspect you're trying to address the problem at the wrong level.
> >
> > What is the proper solution for the problem from your point of view?
>
> I'll explain in more detail, but first I'd like you to answer my
> question below.
>
> >> Similar versioning issues exist with other helpers. We've been doing
> >> fine without QEMU providing unreliable advice on where they might sit in
> >> the file system. What makes this one different?
> >
> > This one is required to be *fully synchronized* with the existing build of QEMU.
> > Other helpers are probably less restrictive and do not have common
> > structures definitions with the QEMU, otherwise they would face the
> > same problem.
> >
> >>
> >> >> What happens when you use the wrong helper?
> >
> > Our intention is that libvirt should never use the wrong RSS helper.
> > But it does not have any ability to check which helper is compatible
> > with the QEMU.
> > QEMU can easily recognize the correct one.
>
> You did not actually answer my question :)
>
> So let's try again: if libvirt does use the wrong RSS helper, how does
> the system behave?
The receive-side scaling may work incorrectly, i.e. finally may move
incoming packets to a virtqueue different than expected one.
>
> >> >>
> >> > UB - in most cases, eBPF program will work with wrong configurations.
> >> > That's why the stamp was added.
> >> >
> >> > query-helper-paths checks the stamp only for RSS helper.
> >>
> >> I have no idea what you're talking about :)
> >>
> >> My best guess is that you're trying to tell me that attempting to work
> >> with the wrong helper will fail cleanly due to some stamp check. That
> >> would be nice.
> >>
>
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 5/5] qmp: Added qemu-ebpf-rss-path command.
2021-08-30 7:51 ` Yuri Benditovich
@ 2021-08-30 8:13 ` Markus Armbruster
2021-08-30 16:56 ` Yuri Benditovich
0 siblings, 1 reply; 60+ messages in thread
From: Markus Armbruster @ 2021-08-30 8:13 UTC (permalink / raw)
To: Yuri Benditovich
Cc: Andrew Melnichenko, Daniel P. Berrangé,
Michael S. Tsirkin, Jason Wang, qemu-devel, Yan Vugenfirer,
Eric Blake
Yuri Benditovich <yuri.benditovich@daynix.com> writes:
> On Mon, Aug 30, 2021 at 9:10 AM Markus Armbruster <armbru@redhat.com> wrote:
>>
>> Yuri Benditovich <yuri.benditovich@daynix.com> writes:
>>
>> > On Tue, Aug 24, 2021 at 9:41 AM Markus Armbruster <armbru@redhat.com> wrote:
>> >>
>> >> Andrew Melnichenko <andrew@daynix.com> writes:
>> >>
>> >> > Hi,
>> >> >
>> >> >> The helper may or may not be installed at the path compiled into QEMU.
>> >> >>
>> >> > Yes, so the helper will not be called - QEMU will try to initiate eBPF RSS
>> >> > or use "in-qemu" RSS.
>> >>
>> >> My point is: the proposed command's mission is to help the management
>> >> application run the right helper. However, its advice is *unreliable*.
>> >> It may point to the wrong helper, or to nothing at all. The right
>> >> helper may still exist elsewhere.
>> >
>> > Hi Markus,
>> > Indeed the intention of this command is to return the proper helper.
>> > Especially in the case of RSS helper this is *reliable* advice and it
>> > points to the helper that was built together with QEMU, i.e. with the
>> > same headers.
>> > This was discussed earlier, for example in
>> > https://lists.nongnu.org/archive/html/qemu-devel/2021-06/msg02248.html
>> >
>> >>
>> >> I suspect you're trying to address the problem at the wrong level.
>> >
>> > What is the proper solution for the problem from your point of view?
>>
>> I'll explain in more detail, but first I'd like you to answer my
>> question below.
>>
>> >> Similar versioning issues exist with other helpers. We've been doing
>> >> fine without QEMU providing unreliable advice on where they might sit in
>> >> the file system. What makes this one different?
>> >
>> > This one is required to be *fully synchronized* with the existing build of QEMU.
>> > Other helpers are probably less restrictive and do not have common
>> > structures definitions with the QEMU, otherwise they would face the
>> > same problem.
>> >
>> >>
>> >> >> What happens when you use the wrong helper?
>> >
>> > Our intention is that libvirt should never use the wrong RSS helper.
>> > But it does not have any ability to check which helper is compatible
>> > with the QEMU.
>> > QEMU can easily recognize the correct one.
>>
>> You did not actually answer my question :)
>>
>> So let's try again: if libvirt does use the wrong RSS helper, how does
>> the system behave?
>
> The receive-side scaling may work incorrectly, i.e. finally may move
> incoming packets to a virtqueue different than expected one.
Then I'm confused about the purpose of "the stamp" mentioned below. Can
you enlighten me?
>
>>
>> >> >>
>> >> > UB - in most cases, eBPF program will work with wrong configurations.
>> >> > That's why the stamp was added.
>> >> >
>> >> > query-helper-paths checks the stamp only for RSS helper.
>> >>
>> >> I have no idea what you're talking about :)
>> >>
>> >> My best guess is that you're trying to tell me that attempting to work
>> >> with the wrong helper will fail cleanly due to some stamp check. That
>> >> would be nice.
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 5/5] qmp: Added qemu-ebpf-rss-path command.
2021-08-30 8:13 ` Markus Armbruster
@ 2021-08-30 16:56 ` Yuri Benditovich
2021-08-31 15:00 ` Markus Armbruster
2021-09-02 16:06 ` Markus Armbruster
0 siblings, 2 replies; 60+ messages in thread
From: Yuri Benditovich @ 2021-08-30 16:56 UTC (permalink / raw)
To: Markus Armbruster
Cc: Andrew Melnichenko, Daniel P. Berrangé,
Michael S. Tsirkin, Jason Wang, qemu-devel, Yan Vugenfirer,
Eric Blake
On Mon, Aug 30, 2021 at 11:14 AM Markus Armbruster <armbru@redhat.com> wrote:
>
> Yuri Benditovich <yuri.benditovich@daynix.com> writes:
>
> > On Mon, Aug 30, 2021 at 9:10 AM Markus Armbruster <armbru@redhat.com> wrote:
> >>
> >> Yuri Benditovich <yuri.benditovich@daynix.com> writes:
> >>
> >> > On Tue, Aug 24, 2021 at 9:41 AM Markus Armbruster <armbru@redhat.com> wrote:
> >> >>
> >> >> Andrew Melnichenko <andrew@daynix.com> writes:
> >> >>
> >> >> > Hi,
> >> >> >
> >> >> >> The helper may or may not be installed at the path compiled into QEMU.
> >> >> >>
> >> >> > Yes, so the helper will not be called - QEMU will try to initiate eBPF RSS
> >> >> > or use "in-qemu" RSS.
> >> >>
> >> >> My point is: the proposed command's mission is to help the management
> >> >> application run the right helper. However, its advice is *unreliable*.
> >> >> It may point to the wrong helper, or to nothing at all. The right
> >> >> helper may still exist elsewhere.
> >> >
> >> > Hi Markus,
> >> > Indeed the intention of this command is to return the proper helper.
> >> > Especially in the case of RSS helper this is *reliable* advice and it
> >> > points to the helper that was built together with QEMU, i.e. with the
> >> > same headers.
> >> > This was discussed earlier, for example in
> >> > https://lists.nongnu.org/archive/html/qemu-devel/2021-06/msg02248.html
> >> >
> >> >>
> >> >> I suspect you're trying to address the problem at the wrong level.
> >> >
> >> > What is the proper solution for the problem from your point of view?
> >>
> >> I'll explain in more detail, but first I'd like you to answer my
> >> question below.
> >>
> >> >> Similar versioning issues exist with other helpers. We've been doing
> >> >> fine without QEMU providing unreliable advice on where they might sit in
> >> >> the file system. What makes this one different?
> >> >
> >> > This one is required to be *fully synchronized* with the existing build of QEMU.
> >> > Other helpers are probably less restrictive and do not have common
> >> > structures definitions with the QEMU, otherwise they would face the
> >> > same problem.
> >> >
> >> >>
> >> >> >> What happens when you use the wrong helper?
> >> >
> >> > Our intention is that libvirt should never use the wrong RSS helper.
> >> > But it does not have any ability to check which helper is compatible
> >> > with the QEMU.
> >> > QEMU can easily recognize the correct one.
> >>
> >> You did not actually answer my question :)
> >>
> >> So let's try again: if libvirt does use the wrong RSS helper, how does
> >> the system behave?
> >
> > The receive-side scaling may work incorrectly, i.e. finally may move
> > incoming packets to a virtqueue different than expected one.
>
> Then I'm confused about the purpose of "the stamp" mentioned below. Can
> you enlighten me?
The stamp is a string (common for qemu executable and RSS helper
executable during build) that qemu can later retrieve from the helper
in run-time and ensure this helper is fully compatible with this build
of qemu (in terms of eBPF operation). The helper is built with the
same C headers (related to ebpf operation) as the qemu, the qemu is
able to receive file descriptors created by the helper (of ebpf
program and ebpf data structure's maps) from libvirt and deal with
them as if it has created them.
>
> >
> >>
> >> >> >>
> >> >> > UB - in most cases, eBPF program will work with wrong configurations.
> >> >> > That's why the stamp was added.
> >> >> >
> >> >> > query-helper-paths checks the stamp only for RSS helper.
> >> >>
> >> >> I have no idea what you're talking about :)
> >> >>
> >> >> My best guess is that you're trying to tell me that attempting to work
> >> >> with the wrong helper will fail cleanly due to some stamp check. That
> >> >> would be nice.
>
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 5/5] qmp: Added qemu-ebpf-rss-path command.
2021-08-30 16:56 ` Yuri Benditovich
@ 2021-08-31 15:00 ` Markus Armbruster
2021-08-31 19:37 ` Andrew Melnichenko
2021-09-02 16:06 ` Markus Armbruster
1 sibling, 1 reply; 60+ messages in thread
From: Markus Armbruster @ 2021-08-31 15:00 UTC (permalink / raw)
To: Yuri Benditovich
Cc: Andrew Melnichenko, Daniel P. Berrangé,
Michael S. Tsirkin, Jason Wang, qemu-devel, Yan Vugenfirer,
Eric Blake
Yuri Benditovich <yuri.benditovich@daynix.com> writes:
> On Mon, Aug 30, 2021 at 11:14 AM Markus Armbruster <armbru@redhat.com> wrote:
>>
>> Yuri Benditovich <yuri.benditovich@daynix.com> writes:
>>
>> > On Mon, Aug 30, 2021 at 9:10 AM Markus Armbruster <armbru@redhat.com> wrote:
[...]
>> >> So let's try again: if libvirt does use the wrong RSS helper, how does
>> >> the system behave?
>> >
>> > The receive-side scaling may work incorrectly, i.e. finally may move
>> > incoming packets to a virtqueue different than expected one.
>>
>> Then I'm confused about the purpose of "the stamp" mentioned below. Can
>> you enlighten me?
>
> The stamp is a string (common for qemu executable and RSS helper
> executable during build) that qemu can later retrieve from the helper
> in run-time and ensure this helper is fully compatible with this build
> of qemu (in terms of eBPF operation). The helper is built with the
> same C headers (related to ebpf operation) as the qemu, the qemu is
> able to receive file descriptors created by the helper (of ebpf
> program and ebpf data structure's maps) from libvirt and deal with
> them as if it has created them.
Thanks.
I tried to apply your series to check a few of my assumptions, but I'm
getting conflicts. Got something I could git-pull?
[...]
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 5/5] qmp: Added qemu-ebpf-rss-path command.
2021-08-31 15:00 ` Markus Armbruster
@ 2021-08-31 19:37 ` Andrew Melnichenko
2021-09-01 7:16 ` Markus Armbruster
0 siblings, 1 reply; 60+ messages in thread
From: Andrew Melnichenko @ 2021-08-31 19:37 UTC (permalink / raw)
To: Markus Armbruster
Cc: Daniel P. Berrangé,
Michael S. Tsirkin, Jason Wang, qemu-devel, Yuri Benditovich,
Yan Vugenfirer, Eric Blake
[-- Attachment #1: Type: text/plain, Size: 1662 bytes --]
Hi,
> Got something I could git-pull?
>
I can share some links for tests:
https://github.com/daynix/qemu/tree/HelperEBPFv3 - qemu with helper
https://github.com/daynix/libvirt/tree/RSS_RFC_v1 - libvirt with RSS
On Tue, Aug 31, 2021 at 6:00 PM Markus Armbruster <armbru@redhat.com> wrote:
> Yuri Benditovich <yuri.benditovich@daynix.com> writes:
>
> > On Mon, Aug 30, 2021 at 11:14 AM Markus Armbruster <armbru@redhat.com>
> wrote:
> >>
> >> Yuri Benditovich <yuri.benditovich@daynix.com> writes:
> >>
> >> > On Mon, Aug 30, 2021 at 9:10 AM Markus Armbruster <armbru@redhat.com>
> wrote:
>
> [...]
>
> >> >> So let's try again: if libvirt does use the wrong RSS helper, how
> does
> >> >> the system behave?
> >> >
> >> > The receive-side scaling may work incorrectly, i.e. finally may move
> >> > incoming packets to a virtqueue different than expected one.
> >>
> >> Then I'm confused about the purpose of "the stamp" mentioned below. Can
> >> you enlighten me?
> >
> > The stamp is a string (common for qemu executable and RSS helper
> > executable during build) that qemu can later retrieve from the helper
> > in run-time and ensure this helper is fully compatible with this build
> > of qemu (in terms of eBPF operation). The helper is built with the
> > same C headers (related to ebpf operation) as the qemu, the qemu is
> > able to receive file descriptors created by the helper (of ebpf
> > program and ebpf data structure's maps) from libvirt and deal with
> > them as if it has created them.
>
> Thanks.
>
> I tried to apply your series to check a few of my assumptions, but I'm
> getting conflicts. Got something I could git-pull?
>
> [...]
>
>
[-- Attachment #2: Type: text/html, Size: 2769 bytes --]
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 5/5] qmp: Added qemu-ebpf-rss-path command.
2021-08-31 19:37 ` Andrew Melnichenko
@ 2021-09-01 7:16 ` Markus Armbruster
[not found] ` <CABcq3pGzs=RqLCuu70KyWt7W6T=qEhihK6v=iHJyfuGqiN_Q+A@mail.gmail.com>
0 siblings, 1 reply; 60+ messages in thread
From: Markus Armbruster @ 2021-09-01 7:16 UTC (permalink / raw)
To: Andrew Melnichenko
Cc: Daniel P. Berrangé,
Michael S. Tsirkin, Jason Wang, qemu-devel, Yuri Benditovich,
Yan Vugenfirer, Eric Blake
Andrew Melnichenko <andrew@daynix.com> writes:
> Hi,
>
>> Got something I could git-pull?
>>
> I can share some links for tests:
> https://github.com/daynix/qemu/tree/HelperEBPFv3 - qemu with helper
> https://github.com/daynix/libvirt/tree/RSS_RFC_v1 - libvirt with RSS
Thanks!
I get a few warnings. I'm copying the ones from Clang:
../hw/net/virtio-net.c:1274:63: warning: array index 2 is past the end of the array (which contains 2 elements) [-Warray-bounds]
ret = ebpf_rss_load_fds(&n->ebpf_rss, fds[0], fds[1], fds[2], fds[3]);
^ ~
../hw/net/virtio-net.c:1257:5: note: array 'fds' declared here
int fds[EBPF_RSS_MAX_FDS];
^
../hw/net/virtio-net.c:1274:71: warning: array index 3 is past the end of the array (which contains 2 elements) [-Warray-bounds]
ret = ebpf_rss_load_fds(&n->ebpf_rss, fds[0], fds[1], fds[2], fds[3]);
^ ~
../hw/net/virtio-net.c:1257:5: note: array 'fds' declared here
int fds[EBPF_RSS_MAX_FDS];
^
../ebpf/qemu-ebpf-rss-helper.c:30:6: warning: no previous prototype for function 'qemuHelperStamp_a59ff8818a819adc85c28504b7961bdf89f0d9c0' [-Wmissing-prototypes]
void QEMU_HELPER_STAMP(void) {}
^
./qemu-helper-stamp.h:1:27: note: expanded from macro 'QEMU_HELPER_STAMP'
#define QEMU_HELPER_STAMP qemuHelperStamp_a59ff8818a819adc85c28504b7961bdf89f0d9c0
^
../ebpf/qemu-ebpf-rss-helper.c:30:1: note: declare 'static' if the function is not intended to be used outside of this translation unit
void QEMU_HELPER_STAMP(void) {}
^
static
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 5/5] qmp: Added qemu-ebpf-rss-path command.
2021-08-30 16:56 ` Yuri Benditovich
2021-08-31 15:00 ` Markus Armbruster
@ 2021-09-02 16:06 ` Markus Armbruster
1 sibling, 0 replies; 60+ messages in thread
From: Markus Armbruster @ 2021-09-02 16:06 UTC (permalink / raw)
To: Yuri Benditovich
Cc: Andrew Melnichenko, Daniel P. Berrangé,
Michael S. Tsirkin, Jason Wang, qemu-devel, Yan Vugenfirer,
Eric Blake
Yuri Benditovich <yuri.benditovich@daynix.com> writes:
> On Mon, Aug 30, 2021 at 11:14 AM Markus Armbruster <armbru@redhat.com> wrote:
>>
>> Yuri Benditovich <yuri.benditovich@daynix.com> writes:
>>
>> > On Mon, Aug 30, 2021 at 9:10 AM Markus Armbruster <armbru@redhat.com> wrote:
>> >>
>> >> Yuri Benditovich <yuri.benditovich@daynix.com> writes:
>> >>
>> >> > On Tue, Aug 24, 2021 at 9:41 AM Markus Armbruster <armbru@redhat.com> wrote:
>> >> >>
>> >> >> Andrew Melnichenko <andrew@daynix.com> writes:
>> >> >>
>> >> >> > Hi,
>> >> >> >
>> >> >> >> The helper may or may not be installed at the path compiled into QEMU.
>> >> >> >>
>> >> >> > Yes, so the helper will not be called - QEMU will try to initiate eBPF RSS
>> >> >> > or use "in-qemu" RSS.
>> >> >>
>> >> >> My point is: the proposed command's mission is to help the management
>> >> >> application run the right helper. However, its advice is *unreliable*.
>> >> >> It may point to the wrong helper, or to nothing at all. The right
>> >> >> helper may still exist elsewhere.
>> >> >
>> >> > Hi Markus,
>> >> > Indeed the intention of this command is to return the proper helper.
>> >> > Especially in the case of RSS helper this is *reliable* advice and it
>> >> > points to the helper that was built together with QEMU, i.e. with the
>> >> > same headers.
In my testing, the patch fails at providing reliable advice.
I did a "git fetch https://github.com/daynix/qemu HelperEBPFv3", rebased
the result to master, build the thing, ran qemu-system-x86_64 right from
the build tree, tried query-helper-paths and got
{"return": [
{"name": "qemu-bridge-helper",
"path": "/work/armbru/tmp/inst-qemu/libexec/qemu-bridge-helper"},
{"name": "qemu-pr-helper",
"path": "/work/armbru/tmp/inst-qemu/libexec/qemu-pr-helper"},
{"name": "qemu-ebpf-rss-helper",
"path": "/work/armbru/qemu/bld-x86/qemu-ebpf-rss-helper"}]}
Ther first two are bogus. /work/armbru/tmp/inst-qemu/... is where "make
install" would put things. I last ran "make install" almost three
months ago.
The last one is accurate in this particular scenario. More on why
below.
It won't be in a deployment where the actual installation paths do not
match the ones we told configure at build time. Yes, people do that.
Please read my entire reply before you comment on this paragraph.
>> >> > This was discussed earlier, for example in
>> >> > https://lists.nongnu.org/archive/html/qemu-devel/2021-06/msg02248.html
>> >> >
>> >> >>
>> >> >> I suspect you're trying to address the problem at the wrong level.
>> >> >
>> >> > What is the proper solution for the problem from your point of view?
>> >>
>> >> I'll explain in more detail, but first I'd like you to answer my
>> >> question below.
>> >>
>> >> >> Similar versioning issues exist with other helpers. We've been doing
>> >> >> fine without QEMU providing unreliable advice on where they might sit in
>> >> >> the file system. What makes this one different?
>> >> >
>> >> > This one is required to be *fully synchronized* with the existing build of QEMU.
>> >> > Other helpers are probably less restrictive and do not have common
>> >> > structures definitions with the QEMU, otherwise they would face the
>> >> > same problem.
>> >> >
>> >> >>
>> >> >> >> What happens when you use the wrong helper?
>> >> >
>> >> > Our intention is that libvirt should never use the wrong RSS helper.
>> >> > But it does not have any ability to check which helper is compatible
>> >> > with the QEMU.
>> >> > QEMU can easily recognize the correct one.
>> >>
>> >> You did not actually answer my question :)
>> >>
>> >> So let's try again: if libvirt does use the wrong RSS helper, how does
>> >> the system behave?
>> >
>> > The receive-side scaling may work incorrectly, i.e. finally may move
>> > incoming packets to a virtqueue different than expected one.
>>
>> Then I'm confused about the purpose of "the stamp" mentioned below. Can
>> you enlighten me?
>
> The stamp is a string (common for qemu executable and RSS helper
> executable during build) that qemu can later retrieve from the helper
> in run-time and ensure this helper is fully compatible with this build
> of qemu (in terms of eBPF operation). The helper is built with the
> same C headers (related to ebpf operation) as the qemu, the qemu is
> able to receive file descriptors created by the helper (of ebpf
> program and ebpf data structure's maps) from libvirt and deal with
> them as if it has created them.
query-helper-paths looks for the helper a number of locations, and
reports the first one. This is fundamentally unrealiable.
For qemu-ebpf-rss-helper, it additionally searches for a stamp symbol in
the ELF symbol table. That's what makes it reliable in the sense of
"won't report crap". It's still unreliable in the sense of "may not
find the helper", see above.
Searching the ELF symbol table requires ELF. I suspect your meson.build
doesn't reflect that.
It also requires the symbol table to be present. Statically linked
programs don't have one, if I remember correctly.
>> >> >> >>
>> >> >> > UB - in most cases, eBPF program will work with wrong configurations.
>> >> >> > That's why the stamp was added.
>> >> >> >
>> >> >> > query-helper-paths checks the stamp only for RSS helper.
>> >> >>
>> >> >> I have no idea what you're talking about :)
>> >> >>
>> >> >> My best guess is that you're trying to tell me that attempting to work
>> >> >> with the wrong helper will fail cleanly due to some stamp check. That
>> >> >> would be nice.
Looks like my best guess was correct.
Let's take a step back.
Management applications run qemu-system-FOO and helpers. They know
where to find qemu-system-FOO. It stands to reason that they also know
where to find the matching helpers. I fail to see why they need help
from qemu-system-FOO via QMP. Even if they need help, qemu-system-FOO
can't give it, because it cannot know for sure. It is wherever the
system administrator / distro put it.
When the system administrator / distro put stuff in unusual places, they
get to configure the programs that use it. In this case, that's the
management application.
How QEMU may be able to help is refuse to run with the wrong helper.
If qemu-system-FOO talks directly to the helper, you can have the helper
present the stamp for qemu-system-FOO to accept or reject. Or vice
versa, doesn't matter.
Or you can provide means for the management application to retrieve a
stamp from all programs involved, so it can fail unless they match.
There are much easier ways for that than searching through ELF symbol
tables. Have a command line option to print it. For qemu-system-FOO,
you can also have a QMP command to query it.
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 0/5] ebpf: Added ebpf helper for libvirtd.
2021-07-13 15:37 [PATCH 0/5] ebpf: Added ebpf helper for libvirtd Andrew Melnychenko
` (4 preceding siblings ...)
2021-07-13 15:37 ` [PATCH 5/5] qmp: Added qemu-ebpf-rss-path command Andrew Melnychenko
@ 2021-07-22 8:37 ` Andrew Melnichenko
2021-08-16 11:57 ` Yuri Benditovich
2021-08-20 3:43 ` Jason Wang
6 siblings, 1 reply; 60+ messages in thread
From: Andrew Melnichenko @ 2021-07-22 8:37 UTC (permalink / raw)
To: Michael S. Tsirkin, Yuri Benditovich, Jason Wang,
Markus Armbruster, Eric Blake, Daniel P. Berrangé
Cc: Yan Vugenfirer, qemu-devel
[-- Attachment #1: Type: text/plain, Size: 3126 bytes --]
ping
On Tue, Jul 13, 2021 at 6:38 PM Andrew Melnychenko <andrew@daynix.com>
wrote:
> Libvirt usually launches qemu with strict permissions.
> To enable eBPF RSS steering, qemu-ebpf-rss-helper was added.
>
> Added property "ebpf_rss_fds" for "virtio-net" that allows to
> initialize eBPF RSS context with passed program & maps fds.
>
> Added qemu-ebpf-rss-helper - simple helper that loads eBPF
> context and passes fds through unix socket.
> Libvirt should call the helper and pass fds to qemu through
> "ebpf_rss_fds" property.
>
> Added explicit target OS check for libbpf dependency in meson.
> eBPF RSS works only with Linux TAP, so there is no reason to
> build eBPF loader/helper for non-Linux.
>
> Changed Qemu updates eBPF maps to array mmaping. Mmaping allows
> bypassing unprivileged BPF map update. Also, instead of 3 maps
> (config, key and indirection table) there is one map that
> combines everything.
>
> Added helper stamp. To check that helper was build with qemu,
> qemu would check helper symbols that should contain the stamp.
> It was done similar to qemu modules, but checking was performed
> by the helper's ELF parsing.
>
> Overall, libvirt process should not be aware of the "interface"
> of eBPF RSS, it will not be aware of eBPF maps/program "type" and
> their quantity. That's why qemu and the helper should be from
> the same build and be "synchronized". Technically each qemu may
> have its own helper. That's why "query-helper-paths" qmp command
> was added. Qemu should return the path to the helper that suits
> and libvirt should use "that" helper for "that" emulator.
>
> qmp sample:
> C: { "execute": "query-helper-paths" }
> S: { "return": [
> {
> "name": "qemu-ebpf-rss-helper",
> "path": "/usr/local/libexec/qemu-ebpf-rss-helper"
> }
> ]
> }
>
> Changes since v1:
> * Mmap() used instead if bpf_map_update_elem().
> * Added helper stamp.
>
> Andrew Melnychenko (5):
> ebpf: Added eBPF initialization by fds and map update.
> virtio-net: Added property to load eBPF RSS with fds.
> qmp: Added the helper stamp check.
> ebpf_rss_helper: Added helper for eBPF RSS.
> qmp: Added qemu-ebpf-rss-path command.
>
> ebpf/ebpf_rss-stub.c | 6 +
> ebpf/ebpf_rss.c | 120 ++++---
> ebpf/ebpf_rss.h | 8 +-
> ebpf/qemu-ebpf-rss-helper.c | 130 +++++++
> ebpf/rss.bpf.skeleton.h | 557 +++++++++++++++---------------
> hw/net/virtio-net.c | 77 ++++-
> include/hw/virtio/virtio-net.h | 1 +
> meson.build | 47 ++-
> monitor/meson.build | 1 +
> monitor/qemu-helper-stamp-utils.c | 297 ++++++++++++++++
> monitor/qemu-helper-stamp-utils.h | 24 ++
> monitor/qmp-cmds.c | 32 ++
> qapi/misc.json | 33 ++
> tools/ebpf/rss.bpf.c | 67 ++--
> 14 files changed, 990 insertions(+), 410 deletions(-)
> create mode 100644 ebpf/qemu-ebpf-rss-helper.c
> create mode 100644 monitor/qemu-helper-stamp-utils.c
> create mode 100644 monitor/qemu-helper-stamp-utils.h
>
> --
> 2.31.1
>
>
[-- Attachment #2: Type: text/html, Size: 3969 bytes --]
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 0/5] ebpf: Added ebpf helper for libvirtd.
2021-07-22 8:37 ` [PATCH 0/5] ebpf: Added ebpf helper for libvirtd Andrew Melnichenko
@ 2021-08-16 11:57 ` Yuri Benditovich
2021-08-17 5:49 ` Jason Wang
0 siblings, 1 reply; 60+ messages in thread
From: Yuri Benditovich @ 2021-08-16 11:57 UTC (permalink / raw)
To: Jason Wang
Cc: Andrew Melnichenko, Daniel P. Berrangé,
Michael S. Tsirkin, qemu-devel, Markus Armbruster,
Yan Vugenfirer, Eric Blake
Jason,
Can you please review the series?
Thanks,
Yuri
On Thu, Jul 22, 2021 at 11:38 AM Andrew Melnichenko <andrew@daynix.com> wrote:
>
> ping
>
> On Tue, Jul 13, 2021 at 6:38 PM Andrew Melnychenko <andrew@daynix.com> wrote:
>>
>> Libvirt usually launches qemu with strict permissions.
>> To enable eBPF RSS steering, qemu-ebpf-rss-helper was added.
>>
>> Added property "ebpf_rss_fds" for "virtio-net" that allows to
>> initialize eBPF RSS context with passed program & maps fds.
>>
>> Added qemu-ebpf-rss-helper - simple helper that loads eBPF
>> context and passes fds through unix socket.
>> Libvirt should call the helper and pass fds to qemu through
>> "ebpf_rss_fds" property.
>>
>> Added explicit target OS check for libbpf dependency in meson.
>> eBPF RSS works only with Linux TAP, so there is no reason to
>> build eBPF loader/helper for non-Linux.
>>
>> Changed Qemu updates eBPF maps to array mmaping. Mmaping allows
>> bypassing unprivileged BPF map update. Also, instead of 3 maps
>> (config, key and indirection table) there is one map that
>> combines everything.
>>
>> Added helper stamp. To check that helper was build with qemu,
>> qemu would check helper symbols that should contain the stamp.
>> It was done similar to qemu modules, but checking was performed
>> by the helper's ELF parsing.
>>
>> Overall, libvirt process should not be aware of the "interface"
>> of eBPF RSS, it will not be aware of eBPF maps/program "type" and
>> their quantity. That's why qemu and the helper should be from
>> the same build and be "synchronized". Technically each qemu may
>> have its own helper. That's why "query-helper-paths" qmp command
>> was added. Qemu should return the path to the helper that suits
>> and libvirt should use "that" helper for "that" emulator.
>>
>> qmp sample:
>> C: { "execute": "query-helper-paths" }
>> S: { "return": [
>> {
>> "name": "qemu-ebpf-rss-helper",
>> "path": "/usr/local/libexec/qemu-ebpf-rss-helper"
>> }
>> ]
>> }
>>
>> Changes since v1:
>> * Mmap() used instead if bpf_map_update_elem().
>> * Added helper stamp.
>>
>> Andrew Melnychenko (5):
>> ebpf: Added eBPF initialization by fds and map update.
>> virtio-net: Added property to load eBPF RSS with fds.
>> qmp: Added the helper stamp check.
>> ebpf_rss_helper: Added helper for eBPF RSS.
>> qmp: Added qemu-ebpf-rss-path command.
>>
>> ebpf/ebpf_rss-stub.c | 6 +
>> ebpf/ebpf_rss.c | 120 ++++---
>> ebpf/ebpf_rss.h | 8 +-
>> ebpf/qemu-ebpf-rss-helper.c | 130 +++++++
>> ebpf/rss.bpf.skeleton.h | 557 +++++++++++++++---------------
>> hw/net/virtio-net.c | 77 ++++-
>> include/hw/virtio/virtio-net.h | 1 +
>> meson.build | 47 ++-
>> monitor/meson.build | 1 +
>> monitor/qemu-helper-stamp-utils.c | 297 ++++++++++++++++
>> monitor/qemu-helper-stamp-utils.h | 24 ++
>> monitor/qmp-cmds.c | 32 ++
>> qapi/misc.json | 33 ++
>> tools/ebpf/rss.bpf.c | 67 ++--
>> 14 files changed, 990 insertions(+), 410 deletions(-)
>> create mode 100644 ebpf/qemu-ebpf-rss-helper.c
>> create mode 100644 monitor/qemu-helper-stamp-utils.c
>> create mode 100644 monitor/qemu-helper-stamp-utils.h
>>
>> --
>> 2.31.1
>>
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 0/5] ebpf: Added ebpf helper for libvirtd.
2021-08-16 11:57 ` Yuri Benditovich
@ 2021-08-17 5:49 ` Jason Wang
0 siblings, 0 replies; 60+ messages in thread
From: Jason Wang @ 2021-08-17 5:49 UTC (permalink / raw)
To: Yuri Benditovich
Cc: Andrew Melnichenko, Daniel P. Berrangé,
Michael S. Tsirkin, qemu-devel, Markus Armbruster,
Yan Vugenfirer, Eric Blake
On Mon, Aug 16, 2021 at 7:58 PM Yuri Benditovich
<yuri.benditovich@daynix.com> wrote:
>
> Jason,
> Can you please review the series?
WIll do it this week.
Thanks
>
> Thanks,
> Yuri
>
> On Thu, Jul 22, 2021 at 11:38 AM Andrew Melnichenko <andrew@daynix.com> wrote:
> >
> > ping
> >
> > On Tue, Jul 13, 2021 at 6:38 PM Andrew Melnychenko <andrew@daynix.com> wrote:
> >>
> >> Libvirt usually launches qemu with strict permissions.
> >> To enable eBPF RSS steering, qemu-ebpf-rss-helper was added.
> >>
> >> Added property "ebpf_rss_fds" for "virtio-net" that allows to
> >> initialize eBPF RSS context with passed program & maps fds.
> >>
> >> Added qemu-ebpf-rss-helper - simple helper that loads eBPF
> >> context and passes fds through unix socket.
> >> Libvirt should call the helper and pass fds to qemu through
> >> "ebpf_rss_fds" property.
> >>
> >> Added explicit target OS check for libbpf dependency in meson.
> >> eBPF RSS works only with Linux TAP, so there is no reason to
> >> build eBPF loader/helper for non-Linux.
> >>
> >> Changed Qemu updates eBPF maps to array mmaping. Mmaping allows
> >> bypassing unprivileged BPF map update. Also, instead of 3 maps
> >> (config, key and indirection table) there is one map that
> >> combines everything.
> >>
> >> Added helper stamp. To check that helper was build with qemu,
> >> qemu would check helper symbols that should contain the stamp.
> >> It was done similar to qemu modules, but checking was performed
> >> by the helper's ELF parsing.
> >>
> >> Overall, libvirt process should not be aware of the "interface"
> >> of eBPF RSS, it will not be aware of eBPF maps/program "type" and
> >> their quantity. That's why qemu and the helper should be from
> >> the same build and be "synchronized". Technically each qemu may
> >> have its own helper. That's why "query-helper-paths" qmp command
> >> was added. Qemu should return the path to the helper that suits
> >> and libvirt should use "that" helper for "that" emulator.
> >>
> >> qmp sample:
> >> C: { "execute": "query-helper-paths" }
> >> S: { "return": [
> >> {
> >> "name": "qemu-ebpf-rss-helper",
> >> "path": "/usr/local/libexec/qemu-ebpf-rss-helper"
> >> }
> >> ]
> >> }
> >>
> >> Changes since v1:
> >> * Mmap() used instead if bpf_map_update_elem().
> >> * Added helper stamp.
> >>
> >> Andrew Melnychenko (5):
> >> ebpf: Added eBPF initialization by fds and map update.
> >> virtio-net: Added property to load eBPF RSS with fds.
> >> qmp: Added the helper stamp check.
> >> ebpf_rss_helper: Added helper for eBPF RSS.
> >> qmp: Added qemu-ebpf-rss-path command.
> >>
> >> ebpf/ebpf_rss-stub.c | 6 +
> >> ebpf/ebpf_rss.c | 120 ++++---
> >> ebpf/ebpf_rss.h | 8 +-
> >> ebpf/qemu-ebpf-rss-helper.c | 130 +++++++
> >> ebpf/rss.bpf.skeleton.h | 557 +++++++++++++++---------------
> >> hw/net/virtio-net.c | 77 ++++-
> >> include/hw/virtio/virtio-net.h | 1 +
> >> meson.build | 47 ++-
> >> monitor/meson.build | 1 +
> >> monitor/qemu-helper-stamp-utils.c | 297 ++++++++++++++++
> >> monitor/qemu-helper-stamp-utils.h | 24 ++
> >> monitor/qmp-cmds.c | 32 ++
> >> qapi/misc.json | 33 ++
> >> tools/ebpf/rss.bpf.c | 67 ++--
> >> 14 files changed, 990 insertions(+), 410 deletions(-)
> >> create mode 100644 ebpf/qemu-ebpf-rss-helper.c
> >> create mode 100644 monitor/qemu-helper-stamp-utils.c
> >> create mode 100644 monitor/qemu-helper-stamp-utils.h
> >>
> >> --
> >> 2.31.1
> >>
>
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 0/5] ebpf: Added ebpf helper for libvirtd.
2021-07-13 15:37 [PATCH 0/5] ebpf: Added ebpf helper for libvirtd Andrew Melnychenko
` (5 preceding siblings ...)
2021-07-22 8:37 ` [PATCH 0/5] ebpf: Added ebpf helper for libvirtd Andrew Melnichenko
@ 2021-08-20 3:43 ` Jason Wang
6 siblings, 0 replies; 60+ messages in thread
From: Jason Wang @ 2021-08-20 3:43 UTC (permalink / raw)
To: Andrew Melnychenko, mst, yuri.benditovich, armbru, eblake, berrange
Cc: yan, qemu-devel
在 2021/7/13 下午11:37, Andrew Melnychenko 写道:
> Libvirt usually launches qemu with strict permissions.
> To enable eBPF RSS steering, qemu-ebpf-rss-helper was added.
>
> Added property "ebpf_rss_fds" for "virtio-net" that allows to
> initialize eBPF RSS context with passed program & maps fds.
>
> Added qemu-ebpf-rss-helper - simple helper that loads eBPF
> context and passes fds through unix socket.
> Libvirt should call the helper and pass fds to qemu through
> "ebpf_rss_fds" property.
>
> Added explicit target OS check for libbpf dependency in meson.
> eBPF RSS works only with Linux TAP, so there is no reason to
> build eBPF loader/helper for non-Linux.
>
> Changed Qemu updates eBPF maps to array mmaping. Mmaping allows
> bypassing unprivileged BPF map update. Also, instead of 3 maps
> (config, key and indirection table) there is one map that
> combines everything.
>
> Added helper stamp. To check that helper was build with qemu,
> qemu would check helper symbols that should contain the stamp.
> It was done similar to qemu modules, but checking was performed
> by the helper's ELF parsing.
>
> Overall, libvirt process should not be aware of the "interface"
> of eBPF RSS, it will not be aware of eBPF maps/program "type" and
> their quantity. That's why qemu and the helper should be from
> the same build and be "synchronized". Technically each qemu may
> have its own helper. That's why "query-helper-paths" qmp command
> was added. Qemu should return the path to the helper that suits
> and libvirt should use "that" helper for "that" emulator.
>
> qmp sample:
> C: { "execute": "query-helper-paths" }
> S: { "return": [
> {
> "name": "qemu-ebpf-rss-helper",
> "path": "/usr/local/libexec/qemu-ebpf-rss-helper"
> }
> ]
> }
>
> Changes since v1:
> * Mmap() used instead if bpf_map_update_elem().
Some questions:
1) Is the mmap() part a stable ABI?
2) Is there a change that we may use other kinds of bpf map that can be
mmaped?
3) What's the advantages of using bpf mmap() over the rx-filter-event?
Thanks
> * Added helper stamp.
>
> Andrew Melnychenko (5):
> ebpf: Added eBPF initialization by fds and map update.
> virtio-net: Added property to load eBPF RSS with fds.
> qmp: Added the helper stamp check.
> ebpf_rss_helper: Added helper for eBPF RSS.
> qmp: Added qemu-ebpf-rss-path command.
>
> ebpf/ebpf_rss-stub.c | 6 +
> ebpf/ebpf_rss.c | 120 ++++---
> ebpf/ebpf_rss.h | 8 +-
> ebpf/qemu-ebpf-rss-helper.c | 130 +++++++
> ebpf/rss.bpf.skeleton.h | 557 +++++++++++++++---------------
> hw/net/virtio-net.c | 77 ++++-
> include/hw/virtio/virtio-net.h | 1 +
> meson.build | 47 ++-
> monitor/meson.build | 1 +
> monitor/qemu-helper-stamp-utils.c | 297 ++++++++++++++++
> monitor/qemu-helper-stamp-utils.h | 24 ++
> monitor/qmp-cmds.c | 32 ++
> qapi/misc.json | 33 ++
> tools/ebpf/rss.bpf.c | 67 ++--
> 14 files changed, 990 insertions(+), 410 deletions(-)
> create mode 100644 ebpf/qemu-ebpf-rss-helper.c
> create mode 100644 monitor/qemu-helper-stamp-utils.c
> create mode 100644 monitor/qemu-helper-stamp-utils.h
>
^ permalink raw reply [flat|nested] 60+ messages in thread
* [PATCH 3/5] qmp: Added the helper stamp check.
2023-02-19 16:20 [PATCH 0/5] eBPF RSS Helper support Andrew Melnychenko
@ 2023-02-19 16:20 ` Andrew Melnychenko
2023-02-20 9:49 ` Daniel P. Berrangé
0 siblings, 1 reply; 60+ messages in thread
From: Andrew Melnychenko @ 2023-02-19 16:20 UTC (permalink / raw)
To: jasowang, mst, pbonzini, marcandre.lureau, berrange, thuth,
philmd, armbru, eblake, qemu-devel, toke, mprivozn
Cc: yuri.benditovich, yan
Added a function to check the stamp in the helper.
eBPF helper should have a special symbol that generates during the build.
QEMU checks the helper and determines that it fits, so the helper
will produce proper output.
Signed-off-by: Andrew Melnychenko <andrew@daynix.com>
---
meson.build | 10 +
monitor/meson.build | 1 +
monitor/qemu-ebpf-rss-helper-stamp-utils.c | 322 +++++++++++++++++++++
monitor/qemu-ebpf-rss-helper-stamp-utils.h | 39 +++
4 files changed, 372 insertions(+)
create mode 100644 monitor/qemu-ebpf-rss-helper-stamp-utils.c
create mode 100644 monitor/qemu-ebpf-rss-helper-stamp-utils.h
diff --git a/meson.build b/meson.build
index a76c855312..b409912aed 100644
--- a/meson.build
+++ b/meson.build
@@ -2868,6 +2868,16 @@ foreach d : hx_headers
endforeach
genh += hxdep
+ebpf_rss_helper_stamp = custom_target(
+ 'qemu-ebpf-rss-helper-stamp.h',
+ output : 'qemu-ebpf-rss-helper-stamp.h',
+ input : 'ebpf/rss.bpf.skeleton.h',
+ command : [python, '-c', 'import hashlib; print(\'#define QEMU_EBPF_RSS_HELPER_STAMP qemuEbpfRssHelperStamp_{}\'.format(hashlib.sha1(open(\'@INPUT@\', \'rb\').read()).hexdigest()))'],
+ capture: true,
+)
+
+genh += ebpf_rss_helper_stamp
+
###################
# Collect sources #
###################
diff --git a/monitor/meson.build b/monitor/meson.build
index ccb4d1a8e6..36de73414b 100644
--- a/monitor/meson.build
+++ b/monitor/meson.build
@@ -6,6 +6,7 @@ softmmu_ss.add(files(
'hmp.c',
))
softmmu_ss.add([spice_headers, files('qmp-cmds.c')])
+softmmu_ss.add(files('qemu-ebpf-rss-helper-stamp-utils.c'))
specific_ss.add(when: 'CONFIG_SOFTMMU',
if_true: [files( 'hmp-cmds-target.c', 'hmp-target.c'), spice])
diff --git a/monitor/qemu-ebpf-rss-helper-stamp-utils.c b/monitor/qemu-ebpf-rss-helper-stamp-utils.c
new file mode 100644
index 0000000000..23efc36ef0
--- /dev/null
+++ b/monitor/qemu-ebpf-rss-helper-stamp-utils.c
@@ -0,0 +1,322 @@
+/*
+ * QEMU helper stamp check utils.
+ *
+ * Developed by Daynix Computing LTD (http://www.daynix.com)
+ *
+ * Authors:
+ * Andrew Melnychenko <andrew@daynix.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2. See
+ * the COPYING file in the top-level directory.
+ *
+ * Description: This file mostly implements helper stamp checking.
+ * The stamp is implemented in a similar way as in qemu modules.
+ * The helper should contain a specific symbol.
+ * Not in a similar way is symbol checking - here we parse
+ * the ELF file. For now, only eBPF helper contains
+ * the stamp, and the stamp is generated from
+ * sha1 ebpf/rss.bpf.skeleton.h (see meson.build).
+ */
+
+#include "qemu/osdep.h"
+#include "elf.h"
+#include "qemu-ebpf-rss-helper-stamp-utils.h"
+
+#include <glib/gstdio.h>
+
+#ifdef CONFIG_LINUX
+
+static void *file_allocate_and_read(int fd, off_t off, size_t size)
+{
+ void *data;
+ int err;
+
+ if (fd < 0) {
+ return NULL;
+ }
+
+ err = lseek(fd, off, SEEK_SET);
+ if (err < 0) {
+ return NULL;
+ }
+
+ data = g_new0(char, size);
+ if (data == NULL) {
+ return NULL;
+ }
+
+ err = read(fd, data, size);
+ if (err < 0) {
+ g_free(data);
+ return NULL;
+ }
+
+ return data;
+}
+
+static Elf64_Shdr *elf64_get_section_table(int fd, Elf64_Ehdr *elf_header)
+{
+ if (elf_header == NULL) {
+ return NULL;
+ }
+ return (Elf64_Shdr *)file_allocate_and_read(fd, elf_header->e_shoff,
+ elf_header->e_shnum * elf_header->e_shentsize);
+}
+
+static Elf32_Shdr *elf32_get_section_table(int fd, Elf32_Ehdr *elf_header)
+{
+ if (elf_header == NULL) {
+ return NULL;
+ }
+ return (Elf32_Shdr *)file_allocate_and_read(fd, elf_header->e_shoff,
+ elf_header->e_shnum * elf_header->e_shentsize);
+}
+
+static void *elf64_get_section_data(int fd, const Elf64_Shdr* section_header)
+{
+ if (fd < 0 || section_header == NULL) {
+ return NULL;
+ }
+ return file_allocate_and_read(fd, section_header->sh_offset,
+ section_header->sh_size);
+}
+
+static void *elf32_get_section_data(int fd, const Elf32_Shdr* section_header)
+{
+ if (fd < 0 || section_header == NULL) {
+ return NULL;
+ }
+ return file_allocate_and_read(fd, section_header->sh_offset,
+ section_header->sh_size);
+}
+
+static bool elf64_check_symbol_in_symbol_table(int fd,
+ Elf64_Shdr *section_table,
+ Elf64_Shdr *symbol_section,
+ const char *symbol)
+{
+ Elf64_Sym *symbol_table;
+ char *string_table;
+ uint32_t i;
+ bool ret = false;
+
+ symbol_table = (Elf64_Sym *) elf64_get_section_data(fd, symbol_section);
+ if (symbol_table == NULL) {
+ return false;
+ }
+
+ string_table = (char *) elf64_get_section_data(
+ fd, section_table + symbol_section->sh_link);
+ if (string_table == NULL) {
+ g_free(symbol_table);
+ return false;
+ }
+
+ for (i = 0; i < (symbol_section->sh_size / sizeof(Elf64_Sym)); ++i) {
+ if (strncmp((string_table + symbol_table[i].st_name),
+ symbol, strlen(symbol)) == 0)
+ {
+ ret = true;
+ break;
+ }
+ }
+
+ g_free(string_table);
+ g_free(symbol_table);
+ return ret;
+}
+
+static bool elf32_check_symbol_in_symbol_table(int fd,
+ Elf32_Shdr *section_table,
+ Elf32_Shdr *symbol_section,
+ const char *symbol)
+{
+ Elf32_Sym *symbol_table;
+ char *string_table;
+ uint32_t i;
+ bool ret = false;
+
+ symbol_table = (Elf32_Sym *) elf32_get_section_data(fd, symbol_section);
+ if (symbol_table == NULL) {
+ return false;
+ }
+
+ string_table = (char *) elf32_get_section_data(fd,
+ section_table + symbol_section->sh_link);
+ if (string_table == NULL) {
+ g_free(symbol_table);
+ return false;
+ }
+
+ for (i = 0; i < (symbol_section->sh_size / sizeof(Elf32_Sym)); ++i) {
+ if (strncmp((string_table + symbol_table[i].st_name),
+ symbol, strlen(symbol)) == 0)
+ {
+ ret = true;
+ break;
+ }
+ }
+
+ g_free(string_table);
+ g_free(symbol_table);
+ return ret;
+}
+
+static bool elf64_check_stamp(int fd, Elf64_Ehdr *elf_header, const char *stamp)
+{
+ Elf64_Shdr *section_table;
+ size_t i;
+ bool ret = false;
+
+ section_table = elf64_get_section_table(fd, elf_header);
+ if (section_table == NULL) {
+ return false;
+ }
+
+ for (i = 0; i < elf_header->e_shnum; ++i) {
+ if ((section_table[i].sh_type == SHT_SYMTAB)
+ || (section_table[i].sh_type == SHT_DYNSYM)) {
+ if (elf64_check_symbol_in_symbol_table(fd, section_table,
+ section_table + i, stamp)) {
+ ret = true;
+ break;
+ }
+ }
+ }
+
+ g_free(section_table);
+ return ret;
+}
+
+static bool elf32_check_stamp(int fd, Elf32_Ehdr *elf_header, const char *stamp)
+{
+ Elf32_Shdr *section_table;
+ size_t i;
+ bool ret = false;
+
+ section_table = elf32_get_section_table(fd, elf_header);
+ if (section_table == NULL) {
+ return false;
+ }
+
+ for (i = 0; i < elf_header->e_shnum; ++i) {
+ if ((section_table[i].sh_type == SHT_SYMTAB)
+ || (section_table[i].sh_type == SHT_DYNSYM)) {
+ if (elf32_check_symbol_in_symbol_table(fd, section_table,
+ section_table + i, stamp)) {
+ ret = true;
+ break;
+ }
+ }
+ }
+
+ g_free(section_table);
+ return ret;
+}
+
+static bool qemu_check_helper_stamp(const char *path, const char *stamp)
+{
+ int fd;
+ bool ret = false;
+ Elf64_Ehdr *elf_header;
+
+ fd = open(path, O_RDONLY | O_SYNC);
+ if (fd < 0) {
+ return false;
+ }
+
+ elf_header = (Elf64_Ehdr *)file_allocate_and_read(
+ fd, 0, sizeof(Elf64_Ehdr));
+ if (elf_header == NULL) {
+ goto error;
+ }
+
+ if (strncmp((char *)elf_header->e_ident, ELFMAG, SELFMAG)) {
+ g_free(elf_header);
+ goto error;
+ }
+
+ if (elf_header->e_ident[EI_CLASS] == ELFCLASS64) {
+ ret = elf64_check_stamp(fd, elf_header, stamp);
+ } else if (elf_header->e_ident[EI_CLASS] == ELFCLASS32) {
+ ret = elf32_check_stamp(fd, (Elf32_Ehdr *)elf_header, stamp);
+ }
+
+ g_free(elf_header);
+error:
+ close(fd);
+ return ret;
+}
+
+#else
+
+static bool qemu_check_helper_stamp(const char *path, const char *stamp)
+{
+ return false;
+}
+
+#endif
+
+char *qemu_find_default_ebpf_rss_helper(void)
+{
+ char *qemu_exec = NULL;
+ char *qemu_dir = NULL;
+ char *helper = NULL;
+
+ helper = g_build_filename(CONFIG_QEMU_HELPERDIR,
+ QEMU_DEFAULT_EBPF_RSS_HELPER_BIN_NAME, NULL);
+ if (g_access(helper, X_OK) == 0
+ && qemu_check_helper_stamp(helper, QEMU_EBPF_RSS_HELPER_STAMP_STR)) {
+ return helper;
+ }
+ g_free(helper);
+
+#ifdef CONFIG_LINUX
+ qemu_exec = g_file_read_link("/proc/self/exe", NULL);
+#else
+ qemu_exec = NULL;
+#endif
+ if (qemu_exec != NULL) {
+ qemu_dir = g_path_get_dirname(qemu_exec);
+ g_free(qemu_exec);
+ helper = g_build_filename(qemu_dir,
+ QEMU_DEFAULT_EBPF_RSS_HELPER_BIN_NAME, NULL);
+ g_free(qemu_dir);
+ if (g_access(helper, X_OK) == 0
+ && qemu_check_helper_stamp(helper, QEMU_EBPF_RSS_HELPER_STAMP_STR)) {
+ return helper;
+ }
+ g_free(helper);
+ }
+
+ return NULL;
+}
+
+char *qemu_check_suggested_ebpf_rss_helper(const char *path)
+{
+ char *helperbin = NULL;
+ struct stat statbuf; /* NOTE: use GStatBuf? */
+
+ /* check is dir or file */
+ if (g_stat(path, &statbuf) < 0) {
+ return NULL;
+ }
+
+ if (statbuf.st_mode & S_IFDIR) {
+ /* is dir */
+ helperbin = g_build_filename(path,
+ QEMU_DEFAULT_EBPF_RSS_HELPER_BIN_NAME, NULL);
+
+ } else if (statbuf.st_mode & S_IFREG) {
+ /* is file */
+ helperbin = g_strdup(path);
+ }
+
+ if (qemu_check_helper_stamp(helperbin, QEMU_EBPF_RSS_HELPER_STAMP_STR)) {
+ return helperbin;
+ }
+
+ g_free(helperbin);
+
+ return NULL;
+}
diff --git a/monitor/qemu-ebpf-rss-helper-stamp-utils.h b/monitor/qemu-ebpf-rss-helper-stamp-utils.h
new file mode 100644
index 0000000000..7490568aa1
--- /dev/null
+++ b/monitor/qemu-ebpf-rss-helper-stamp-utils.h
@@ -0,0 +1,39 @@
+/*
+ * QEMU helper stamp check utils.
+ *
+ * Developed by Daynix Computing LTD (http://www.daynix.com)
+ *
+ * Authors:
+ * Andrew Melnychenko <andrew@daynix.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2. See
+ * the COPYING file in the top-level directory.
+ */
+
+#ifndef QEMU_QEMU_HELPER_STAMP_UTILS_H
+#define QEMU_QEMU_HELPER_STAMP_UTILS_H
+
+#include "qemu-ebpf-rss-helper-stamp.h" /* generated stamp per build */
+
+#define QEMU_EBPF_RSS_HELPER_STAMP_STR stringify(QEMU_EBPF_RSS_HELPER_STAMP)
+
+#define QEMU_DEFAULT_EBPF_RSS_HELPER_BIN_NAME "qemu-ebpf-rss-helper"
+
+/**
+ * Trying to find the helper with a valid stamp in HELPERDIR
+ * or next to the QEMU binary.
+ * @return path to the eBPF RSS helper bin or NULL(helper not found).
+ */
+char *qemu_find_default_ebpf_rss_helper(void);
+
+/**
+ * Check the helper by the suggested path. The helper should have a valid stamp.
+ * @param path - it can be either a file or directory path.
+ * For the file - checks the stamp of the file.
+ * For the directory - looks for QEMU_DEFAULT_EBPF_RSS_HELPER_BIN_NAME
+ * and checks the stamp of that file.
+ * @return path to valid eBPF RSS helper bin or NULL.
+ */
+char *qemu_check_suggested_ebpf_rss_helper(const char *path);
+
+#endif /* QEMU_QEMU_HELPER_STAMP_UTILS_H */
--
2.39.1
^ permalink raw reply related [flat|nested] 60+ messages in thread
* Re: [PATCH 3/5] qmp: Added the helper stamp check.
2023-02-19 16:20 ` [PATCH 3/5] qmp: Added the helper stamp check Andrew Melnychenko
@ 2023-02-20 9:49 ` Daniel P. Berrangé
2023-02-27 3:45 ` Andrew Melnichenko
2023-02-28 9:56 ` Yuri Benditovich
0 siblings, 2 replies; 60+ messages in thread
From: Daniel P. Berrangé @ 2023-02-20 9:49 UTC (permalink / raw)
To: Andrew Melnychenko
Cc: jasowang, mst, pbonzini, marcandre.lureau, thuth, philmd, armbru,
eblake, qemu-devel, toke, mprivozn, yuri.benditovich, yan
On Sun, Feb 19, 2023 at 06:20:58PM +0200, Andrew Melnychenko wrote:
> Added a function to check the stamp in the helper.
> eBPF helper should have a special symbol that generates during the build.
> QEMU checks the helper and determines that it fits, so the helper
> will produce proper output.
I think this is quite limiting for in place upgrades.
Consider this scenario
* Host has QEMU 8.1.0 installed
* VM is running QEMU 8.1.0
* QEMU 8.1.1 is released with a bug fix in the EBF program
* Host is upgraded to QEMU 8.1.1
* User attempts to hotplug a NIC to the running VM
IIUC this last step is going to fail because we'll be loading
the EBF program from 8.1.1 and so its hash is different from
that expected by the QEMU 8.1.0 that the pre-existing VM is
running.
If some changes to the EBF program are not going to be back
compatible from the POV of the QEMU process, should we instead
be versioning the EBF program. eg so new QEMU will ship both
the old and new versions of the EBF program.
With regards,
Daniel
--
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org -o- https://fstop138.berrange.com :|
|: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 3/5] qmp: Added the helper stamp check.
2023-02-20 9:49 ` Daniel P. Berrangé
@ 2023-02-27 3:45 ` Andrew Melnichenko
2023-02-27 14:06 ` Toke Høiland-Jørgensen
2023-02-28 9:56 ` Yuri Benditovich
1 sibling, 1 reply; 60+ messages in thread
From: Andrew Melnichenko @ 2023-02-27 3:45 UTC (permalink / raw)
To: Daniel P. Berrangé
Cc: jasowang, mst, pbonzini, marcandre.lureau, thuth, philmd, armbru,
eblake, qemu-devel, toke, mprivozn, yuri.benditovich, yan
Hi all,
On Mon, Feb 20, 2023 at 11:50 AM Daniel P. Berrangé <berrange@redhat.com> wrote:
>
> On Sun, Feb 19, 2023 at 06:20:58PM +0200, Andrew Melnychenko wrote:
> > Added a function to check the stamp in the helper.
> > eBPF helper should have a special symbol that generates during the build.
> > QEMU checks the helper and determines that it fits, so the helper
> > will produce proper output.
>
> I think this is quite limiting for in place upgrades.
>
> Consider this scenario
>
> * Host has QEMU 8.1.0 installed
> * VM is running QEMU 8.1.0
> * QEMU 8.1.1 is released with a bug fix in the EBF program
> * Host is upgraded to QEMU 8.1.1
> * User attempts to hotplug a NIC to the running VM
>
> IIUC this last step is going to fail because we'll be loading
> the EBF program from 8.1.1 and so its hash is different from
> that expected by the QEMU 8.1.0 that the pre-existing VM is
> running.
>
> If some changes to the EBF program are not going to be back
> compatible from the POV of the QEMU process, should we instead
> be versioning the EBF program. eg so new QEMU will ship both
> the old and new versions of the EBF program.
>
I think it's too complicated to maintain backward compatibility with
eBPF programs in "one package".
As we can see, the eBPF skeleton may be changed not only by bugfix but
with changes in libbpf(g.e. libbpf 1.0.1+).
This may lead to issues when with a newer environment you can't
consistently recreate the "old" skeleton.
To solve the case - we need to "save" the halper from an old package somehow.
>
> With regards,
> Daniel
> --
> |: https://berrange.com -o- https://www.flickr.com/photos/dberrange :|
> |: https://libvirt.org -o- https://fstop138.berrange.com :|
> |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
>
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 3/5] qmp: Added the helper stamp check.
2023-02-27 3:45 ` Andrew Melnichenko
@ 2023-02-27 14:06 ` Toke Høiland-Jørgensen
0 siblings, 0 replies; 60+ messages in thread
From: Toke Høiland-Jørgensen @ 2023-02-27 14:06 UTC (permalink / raw)
To: Andrew Melnichenko, Daniel P. Berrangé
Cc: jasowang, mst, pbonzini, marcandre.lureau, thuth, philmd, armbru,
eblake, qemu-devel, mprivozn, yuri.benditovich, yan
Andrew Melnichenko <andrew@daynix.com> writes:
> Hi all,
>
> On Mon, Feb 20, 2023 at 11:50 AM Daniel P. Berrangé <berrange@redhat.com> wrote:
>>
>> On Sun, Feb 19, 2023 at 06:20:58PM +0200, Andrew Melnychenko wrote:
>> > Added a function to check the stamp in the helper.
>> > eBPF helper should have a special symbol that generates during the build.
>> > QEMU checks the helper and determines that it fits, so the helper
>> > will produce proper output.
>>
>> I think this is quite limiting for in place upgrades.
>>
>> Consider this scenario
>>
>> * Host has QEMU 8.1.0 installed
>> * VM is running QEMU 8.1.0
>> * QEMU 8.1.1 is released with a bug fix in the EBF program
>> * Host is upgraded to QEMU 8.1.1
>> * User attempts to hotplug a NIC to the running VM
>>
>> IIUC this last step is going to fail because we'll be loading
>> the EBF program from 8.1.1 and so its hash is different from
>> that expected by the QEMU 8.1.0 that the pre-existing VM is
>> running.
>>
>> If some changes to the EBF program are not going to be back
>> compatible from the POV of the QEMU process, should we instead
>> be versioning the EBF program. eg so new QEMU will ship both
>> the old and new versions of the EBF program.
>>
>
> I think it's too complicated to maintain backward compatibility with
> eBPF programs in "one package".
> As we can see, the eBPF skeleton may be changed not only by bugfix but
> with changes in libbpf(g.e. libbpf 1.0.1+).
Hmm, what change in libbpf 1.0.1 affects the skeleton format?
> This may lead to issues when with a newer environment you can't
> consistently recreate the "old" skeleton.
This should be detectable, though? As in, if you know that a new version
breaks compatibility you just bump the version number regardless of the
underlying application version?
-Toke
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 3/5] qmp: Added the helper stamp check.
2023-02-20 9:49 ` Daniel P. Berrangé
2023-02-27 3:45 ` Andrew Melnichenko
@ 2023-02-28 9:56 ` Yuri Benditovich
2023-02-28 18:04 ` Daniel P. Berrangé
1 sibling, 1 reply; 60+ messages in thread
From: Yuri Benditovich @ 2023-02-28 9:56 UTC (permalink / raw)
To: Daniel P. Berrangé
Cc: Andrew Melnychenko, jasowang, mst, pbonzini, marcandre.lureau,
thuth, philmd, armbru, eblake, qemu-devel, toke, mprivozn, yan
[-- Attachment #1: Type: text/plain, Size: 2395 bytes --]
On Mon, Feb 20, 2023 at 11:50 AM Daniel P. Berrangé <berrange@redhat.com>
wrote:
> On Sun, Feb 19, 2023 at 06:20:58PM +0200, Andrew Melnychenko wrote:
> > Added a function to check the stamp in the helper.
> > eBPF helper should have a special symbol that generates during the build.
> > QEMU checks the helper and determines that it fits, so the helper
> > will produce proper output.
>
> I think this is quite limiting for in place upgrades.
>
> Consider this scenario
>
> * Host has QEMU 8.1.0 installed
> * VM is running QEMU 8.1.0
> * QEMU 8.1.1 is released with a bug fix in the EBF program
> * Host is upgraded to QEMU 8.1.1
> * User attempts to hotplug a NIC to the running VM
>
> IIUC this last step is going to fail because we'll be loading
> the EBF program from 8.1.1 and so its hash is different from
> that expected by the QEMU 8.1.0 that the pre-existing VM is
> running.
>
> Indeed we did not take in account the in-place upgrade.
> If some changes to the EBF program are not going to be back
> compatible from the POV of the QEMU process, should we instead
> be versioning the EBF program. eg so new QEMU will ship both
> the old and new versions of the EBF program.
>
>
This does not seem to be an elegant option: QEMU theoretically can include
different eBPF programs but it hardly can interface with each one of them.
The code of QEMU (access to eBPF maps etc) includes header files which eBPF
of the day is being built with them.
I see 2 options to address this issue (of course there are more)
1. Build and install qemu-rss-helper-<hash> executable. Libvirt will always
have a correct name, so for the running instance it will use
qemu-rss-helper-<old-hash>, for the new instance it will use
qemu-rss-helper-<new-hash>
2. Build the helper executable and link it inside qemu as a blob. Libvirt
will always retrieve the executable to the temporary file name and use it.
So the retrieved helper will always be compatible with QEMU. I'm not sure
what is the most portable way to do that.
Daniel,
Does one of these seem suitable?
> With regards,
> Daniel
> --
> |: https://berrange.com -o-
> https://www.flickr.com/photos/dberrange :|
> |: https://libvirt.org -o-
> https://fstop138.berrange.com :|
> |: https://entangle-photo.org -o-
> https://www.instagram.com/dberrange :|
>
>
[-- Attachment #2: Type: text/html, Size: 3670 bytes --]
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 3/5] qmp: Added the helper stamp check.
2023-02-28 9:56 ` Yuri Benditovich
@ 2023-02-28 18:04 ` Daniel P. Berrangé
2023-02-28 19:01 ` Toke Høiland-Jørgensen
` (2 more replies)
0 siblings, 3 replies; 60+ messages in thread
From: Daniel P. Berrangé @ 2023-02-28 18:04 UTC (permalink / raw)
To: Yuri Benditovich
Cc: Andrew Melnychenko, jasowang, mst, pbonzini, marcandre.lureau,
thuth, philmd, armbru, eblake, qemu-devel, toke, mprivozn, yan
On Tue, Feb 28, 2023 at 11:56:27AM +0200, Yuri Benditovich wrote:
> On Mon, Feb 20, 2023 at 11:50 AM Daniel P. Berrangé <berrange@redhat.com>
> wrote:
>
> > On Sun, Feb 19, 2023 at 06:20:58PM +0200, Andrew Melnychenko wrote:
> > > Added a function to check the stamp in the helper.
> > > eBPF helper should have a special symbol that generates during the build.
> > > QEMU checks the helper and determines that it fits, so the helper
> > > will produce proper output.
> >
> > I think this is quite limiting for in place upgrades.
> >
> > Consider this scenario
> >
> > * Host has QEMU 8.1.0 installed
> > * VM is running QEMU 8.1.0
> > * QEMU 8.1.1 is released with a bug fix in the EBF program
> > * Host is upgraded to QEMU 8.1.1
> > * User attempts to hotplug a NIC to the running VM
> >
> > IIUC this last step is going to fail because we'll be loading
> > the EBF program from 8.1.1 and so its hash is different from
> > that expected by the QEMU 8.1.0 that the pre-existing VM is
> > running.
> >
> > Indeed we did not take in account the in-place upgrade.
>
>
>
> > If some changes to the EBF program are not going to be back
> > compatible from the POV of the QEMU process, should we instead
> > be versioning the EBF program. eg so new QEMU will ship both
> > the old and new versions of the EBF program.
>
> This does not seem to be an elegant option: QEMU theoretically can include
> different eBPF programs but it hardly can interface with each one of them.
> The code of QEMU (access to eBPF maps etc) includes header files which eBPF
> of the day is being built with them.
>
> I see 2 options to address this issue (of course there are more)
> 1. Build and install qemu-rss-helper-<hash> executable. Libvirt will always
> have a correct name, so for the running instance it will use
> qemu-rss-helper-<old-hash>, for the new instance it will use
> qemu-rss-helper-<new-hash>
We'll get an ever growing number of program variants we need to
build & distribute with each new QEMU release.
> 2. Build the helper executable and link it inside qemu as a blob. Libvirt
> will always retrieve the executable to the temporary file name and use it.
> So the retrieved helper will always be compatible with QEMU. I'm not sure
> what is the most portable way to do that.
QEMU is considered an untrusted process, so there's no way we're going
to ask it to give us an ELF binary and then execute that in privileged
context.
> Does one of these seem suitable?
Neither feels very appealing to me.
I've been trying to understand the eBPF code we're dealing with in a
little more detail.
IIUC, QEMU, or rather the virtio-net driver needs to receive one FD
for the BPF program, and one or more FDs for the BPF maps that the
program uses. Currently it uses 3 maps, so needs 3 map FDs on top of
the program FD.
The helper program that is proposed here calls ebpf_rss_load() to
load the program and get back a struct which gives access to the
4 FDs, which are then sent to the mgmt app, which forwards them
onto QEMU.
The ebpf_rss_load() method is making use of various structs that
are specific to the RSS program implementation, but does not seems
to do anything especially interesting. It calls into rss_bpf__open()
which eventually gets around to calling rss_bpf__create_skeleton
which is where the interesting stuff happens.
This rss_bpf__create_skeleton() method is implemented in terms of
totally generic libbpf APIs, and has the actual blob that is the
BPF program.
Looking at what this does, I feel it should be trivial for a mgmt
app to implement equivalent logic to rss_bpf__create_skeleton in a
generic manner, if we could just expose the program blob and the
map names to the mgmt app. eg a simple json file
{
"maps": [
"tap_rss_map_configurations",
"tap_rss_map_indirection_table",
"tap_rss_map_toeplitz_key",
],
"program": "....the big blob encoded in base64..."
}
if we installed that file are /usr/share/qemu/bpf/net-rss.json
then when a QEMU process is started, the mgmt app capture the
data in this JSON file. It now has enough info to create the
EBPF programs needed and pass the FDs over to QEMU. This would
be robust against QEMU software upgrades, and not tied to the
specific EBPF program imlp. We can add or remove maps / change
their names etc any time, as the details in the JSON descriptor
can be updated. This avoids need for any special helper program
to be provided by QEMU with the problems that is throwing up
for us.
What am I missing ? This seems pretty straightforward to
achieve from what I see.
With regards,
Daniel
--
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org -o- https://fstop138.berrange.com :|
|: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 3/5] qmp: Added the helper stamp check.
2023-02-28 18:04 ` Daniel P. Berrangé
@ 2023-02-28 19:01 ` Toke Høiland-Jørgensen
2023-02-28 19:03 ` Daniel P. Berrangé
2023-02-28 19:01 ` Daniel P. Berrangé
2023-03-01 6:49 ` Yuri Benditovich
2 siblings, 1 reply; 60+ messages in thread
From: Toke Høiland-Jørgensen @ 2023-02-28 19:01 UTC (permalink / raw)
To: Daniel P. Berrangé, Yuri Benditovich
Cc: Andrew Melnychenko, jasowang, mst, pbonzini, marcandre.lureau,
thuth, philmd, armbru, eblake, qemu-devel, mprivozn, yan
Daniel P. Berrangé <berrange@redhat.com> writes:
> On Tue, Feb 28, 2023 at 11:56:27AM +0200, Yuri Benditovich wrote:
>> On Mon, Feb 20, 2023 at 11:50 AM Daniel P. Berrangé <berrange@redhat.com>
>> wrote:
>>
>> > On Sun, Feb 19, 2023 at 06:20:58PM +0200, Andrew Melnychenko wrote:
>> > > Added a function to check the stamp in the helper.
>> > > eBPF helper should have a special symbol that generates during the build.
>> > > QEMU checks the helper and determines that it fits, so the helper
>> > > will produce proper output.
>> >
>> > I think this is quite limiting for in place upgrades.
>> >
>> > Consider this scenario
>> >
>> > * Host has QEMU 8.1.0 installed
>> > * VM is running QEMU 8.1.0
>> > * QEMU 8.1.1 is released with a bug fix in the EBF program
>> > * Host is upgraded to QEMU 8.1.1
>> > * User attempts to hotplug a NIC to the running VM
>> >
>> > IIUC this last step is going to fail because we'll be loading
>> > the EBF program from 8.1.1 and so its hash is different from
>> > that expected by the QEMU 8.1.0 that the pre-existing VM is
>> > running.
>> >
>> > Indeed we did not take in account the in-place upgrade.
>>
>>
>>
>> > If some changes to the EBF program are not going to be back
>> > compatible from the POV of the QEMU process, should we instead
>> > be versioning the EBF program. eg so new QEMU will ship both
>> > the old and new versions of the EBF program.
>>
>> This does not seem to be an elegant option: QEMU theoretically can include
>> different eBPF programs but it hardly can interface with each one of them.
>> The code of QEMU (access to eBPF maps etc) includes header files which eBPF
>> of the day is being built with them.
>>
>> I see 2 options to address this issue (of course there are more)
>> 1. Build and install qemu-rss-helper-<hash> executable. Libvirt will always
>> have a correct name, so for the running instance it will use
>> qemu-rss-helper-<old-hash>, for the new instance it will use
>> qemu-rss-helper-<new-hash>
>
> We'll get an ever growing number of program variants we need to
> build & distribute with each new QEMU release.
>
>> 2. Build the helper executable and link it inside qemu as a blob. Libvirt
>> will always retrieve the executable to the temporary file name and use it.
>> So the retrieved helper will always be compatible with QEMU. I'm not sure
>> what is the most portable way to do that.
>
> QEMU is considered an untrusted process, so there's no way we're going
> to ask it to give us an ELF binary and then execute that in privileged
> context.
>
>> Does one of these seem suitable?
>
> Neither feels very appealing to me.
>
> I've been trying to understand the eBPF code we're dealing with in a
> little more detail.
>
> IIUC, QEMU, or rather the virtio-net driver needs to receive one FD
> for the BPF program, and one or more FDs for the BPF maps that the
> program uses. Currently it uses 3 maps, so needs 3 map FDs on top of
> the program FD.
>
> The helper program that is proposed here calls ebpf_rss_load() to
> load the program and get back a struct which gives access to the
> 4 FDs, which are then sent to the mgmt app, which forwards them
> onto QEMU.
>
> The ebpf_rss_load() method is making use of various structs that
> are specific to the RSS program implementation, but does not seems
> to do anything especially interesting. It calls into rss_bpf__open()
> which eventually gets around to calling rss_bpf__create_skeleton
> which is where the interesting stuff happens.
>
> This rss_bpf__create_skeleton() method is implemented in terms of
> totally generic libbpf APIs, and has the actual blob that is the
> BPF program.
>
> Looking at what this does, I feel it should be trivial for a mgmt
> app to implement equivalent logic to rss_bpf__create_skeleton in a
> generic manner, if we could just expose the program blob and the
> map names to the mgmt app. eg a simple json file
>
> {
> "maps": [
> "tap_rss_map_configurations",
> "tap_rss_map_indirection_table",
> "tap_rss_map_toeplitz_key",
> ],
> "program": "....the big blob encoded in base64..."
> }
>
> if we installed that file are /usr/share/qemu/bpf/net-rss.json
> then when a QEMU process is started, the mgmt app capture the
> data in this JSON file. It now has enough info to create the
> EBPF programs needed and pass the FDs over to QEMU. This would
> be robust against QEMU software upgrades, and not tied to the
> specific EBPF program imlp. We can add or remove maps / change
> their names etc any time, as the details in the JSON descriptor
> can be updated. This avoids need for any special helper program
> to be provided by QEMU with the problems that is throwing up
> for us.
Just to interject a note on this here: the skeleton code is mostly a
convenience feature used to embed BPF programs into the calling binary.
It is perfectly possible to just have the BPF object file itself reside
directly in the file system and just use the regular libbpf APIs to load
it. Some things get a bit more cumbersome (mostly setting values of
global variables, if the BPF program uses those).
So the JSON example above could just be a regular compiled-from-clang
BPF object file, and the management program can load that, inspect its
contents using the libbpf APIs and pass the file descriptors on to Qemu.
It's even possible to embed version information into this so that Qemu
can check if it understands the format and bail out if it doesn't - just
stick a version field in the configuration map as the first entry :)
-Toke
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 3/5] qmp: Added the helper stamp check.
2023-02-28 19:01 ` Toke Høiland-Jørgensen
@ 2023-02-28 19:03 ` Daniel P. Berrangé
2023-02-28 22:21 ` Toke Høiland-Jørgensen
0 siblings, 1 reply; 60+ messages in thread
From: Daniel P. Berrangé @ 2023-02-28 19:03 UTC (permalink / raw)
To: Toke Høiland-Jørgensen
Cc: Yuri Benditovich, Andrew Melnychenko, jasowang, mst, pbonzini,
marcandre.lureau, thuth, philmd, armbru, eblake, qemu-devel,
mprivozn, yan
On Tue, Feb 28, 2023 at 08:01:51PM +0100, Toke Høiland-Jørgensen wrote:
> Daniel P. Berrangé <berrange@redhat.com> writes:
>
> Just to interject a note on this here: the skeleton code is mostly a
> convenience feature used to embed BPF programs into the calling binary.
> It is perfectly possible to just have the BPF object file itself reside
> directly in the file system and just use the regular libbpf APIs to load
> it. Some things get a bit more cumbersome (mostly setting values of
> global variables, if the BPF program uses those).
>
> So the JSON example above could just be a regular compiled-from-clang
> BPF object file, and the management program can load that, inspect its
> contents using the libbpf APIs and pass the file descriptors on to Qemu.
> It's even possible to embed version information into this so that Qemu
> can check if it understands the format and bail out if it doesn't - just
> stick a version field in the configuration map as the first entry :)
If all you have is the BPF object file is it possible to interrogate
it to get a list of all the maps, and get FDs associated for them ?
I had a look at the libbpf API and wasn't sure about that, it seemed
like you had to know the required maps upfront ? If it is possible
to auto-discover everything you need, soley from the BPF object file
as input, then just dealing with that in isolation would feel simpler.
With regards,
Daniel
--
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org -o- https://fstop138.berrange.com :|
|: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 3/5] qmp: Added the helper stamp check.
2023-02-28 19:03 ` Daniel P. Berrangé
@ 2023-02-28 22:21 ` Toke Høiland-Jørgensen
2023-03-01 9:30 ` Daniel P. Berrangé
0 siblings, 1 reply; 60+ messages in thread
From: Toke Høiland-Jørgensen @ 2023-02-28 22:21 UTC (permalink / raw)
To: Daniel P. Berrangé
Cc: Yuri Benditovich, Andrew Melnychenko, jasowang, mst, pbonzini,
marcandre.lureau, thuth, philmd, armbru, eblake, qemu-devel,
mprivozn, yan
Daniel P. Berrangé <berrange@redhat.com> writes:
> On Tue, Feb 28, 2023 at 08:01:51PM +0100, Toke Høiland-Jørgensen wrote:
>> Daniel P. Berrangé <berrange@redhat.com> writes:
>>
>> Just to interject a note on this here: the skeleton code is mostly a
>> convenience feature used to embed BPF programs into the calling binary.
>> It is perfectly possible to just have the BPF object file itself reside
>> directly in the file system and just use the regular libbpf APIs to load
>> it. Some things get a bit more cumbersome (mostly setting values of
>> global variables, if the BPF program uses those).
>>
>> So the JSON example above could just be a regular compiled-from-clang
>> BPF object file, and the management program can load that, inspect its
>> contents using the libbpf APIs and pass the file descriptors on to Qemu.
>> It's even possible to embed version information into this so that Qemu
>> can check if it understands the format and bail out if it doesn't - just
>> stick a version field in the configuration map as the first entry :)
>
> If all you have is the BPF object file is it possible to interrogate
> it to get a list of all the maps, and get FDs associated for them ?
> I had a look at the libbpf API and wasn't sure about that, it seemed
> like you had to know the required maps upfront ? If it is possible
> to auto-discover everything you need, soley from the BPF object file
> as input, then just dealing with that in isolation would feel simpler.
It is. You load the object file, and bpf_object__for_each_map() lets you
discover which maps it contains, with the different bpf_map__*() APIs
telling you the properties of that map (and you can modify them too
before loading the object if needed).
The only thing that's not in the object file is any initial data you
want to put into the map(s). But except for read-only maps that can be
added by userspace after loading the maps, so you could just let Qemu do
that...
> It occurrs to me that exposing the BPF program as data rather than
> via binary will make more practical to integrate this into KubeVirt's
> architecture. In their deployment setup both QEMU and libvirt are
> running unprivileged inside a container. For any advanced nmetworking
> a completely separate component creates the TAP device and passes it
> into the container running QEMU. I don't think that the separate
> precisely matched helper binary would be something they can use, but
> it might be possible to expose a data file providing the BPF program
> blob and describing its maps.
Well, "a data file providing the BPF program blob and describing its
maps" is basically what a BPF .o file is. It just happens to be encoded
in ELF format :)
You can embed it into some other data structure and have libbpf load it
from a blob in memory as well as from the filesystem, though; that is
basically what the skeleton file does (notice the big character string
at the end, that's just the original .o file contents).
-Toke
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 3/5] qmp: Added the helper stamp check.
2023-02-28 22:21 ` Toke Høiland-Jørgensen
@ 2023-03-01 9:30 ` Daniel P. Berrangé
2023-03-01 14:53 ` Toke Høiland-Jørgensen
0 siblings, 1 reply; 60+ messages in thread
From: Daniel P. Berrangé @ 2023-03-01 9:30 UTC (permalink / raw)
To: Toke Høiland-Jørgensen
Cc: Yuri Benditovich, Andrew Melnychenko, jasowang, mst, pbonzini,
marcandre.lureau, thuth, philmd, armbru, eblake, qemu-devel,
mprivozn, yan
On Tue, Feb 28, 2023 at 11:21:56PM +0100, Toke Høiland-Jørgensen wrote:
> Daniel P. Berrangé <berrange@redhat.com> writes:
>
> > On Tue, Feb 28, 2023 at 08:01:51PM +0100, Toke Høiland-Jørgensen wrote:
> >> Daniel P. Berrangé <berrange@redhat.com> writes:
> >>
> >> Just to interject a note on this here: the skeleton code is mostly a
> >> convenience feature used to embed BPF programs into the calling binary.
> >> It is perfectly possible to just have the BPF object file itself reside
> >> directly in the file system and just use the regular libbpf APIs to load
> >> it. Some things get a bit more cumbersome (mostly setting values of
> >> global variables, if the BPF program uses those).
> >>
> >> So the JSON example above could just be a regular compiled-from-clang
> >> BPF object file, and the management program can load that, inspect its
> >> contents using the libbpf APIs and pass the file descriptors on to Qemu.
> >> It's even possible to embed version information into this so that Qemu
> >> can check if it understands the format and bail out if it doesn't - just
> >> stick a version field in the configuration map as the first entry :)
> >
> > If all you have is the BPF object file is it possible to interrogate
> > it to get a list of all the maps, and get FDs associated for them ?
> > I had a look at the libbpf API and wasn't sure about that, it seemed
> > like you had to know the required maps upfront ? If it is possible
> > to auto-discover everything you need, soley from the BPF object file
> > as input, then just dealing with that in isolation would feel simpler.
>
> It is. You load the object file, and bpf_object__for_each_map() lets you
> discover which maps it contains, with the different bpf_map__*() APIs
> telling you the properties of that map (and you can modify them too
> before loading the object if needed).
>
> The only thing that's not in the object file is any initial data you
> want to put into the map(s). But except for read-only maps that can be
> added by userspace after loading the maps, so you could just let Qemu do
> that...
>
> > It occurrs to me that exposing the BPF program as data rather than
> > via binary will make more practical to integrate this into KubeVirt's
> > architecture. In their deployment setup both QEMU and libvirt are
> > running unprivileged inside a container. For any advanced nmetworking
> > a completely separate component creates the TAP device and passes it
> > into the container running QEMU. I don't think that the separate
> > precisely matched helper binary would be something they can use, but
> > it might be possible to expose a data file providing the BPF program
> > blob and describing its maps.
>
> Well, "a data file providing the BPF program blob and describing its
> maps" is basically what a BPF .o file is. It just happens to be encoded
> in ELF format :)
>
> You can embed it into some other data structure and have libbpf load it
> from a blob in memory as well as from the filesystem, though; that is
> basically what the skeleton file does (notice the big character string
> at the end, that's just the original .o file contents).
Ok, in that case I'm really wondering why any of this helper program
stuff was proposed. I recall the rationale was that it was impossible
for an external program to load the BPF object on behalf of QEMU,
because it would not know how todo that without QEMU specific
knowledge.
It looks like we can simply expose the BPF object blob to mgmt apps
directly and get rid of this helper program entirely.
With regards,
Daniel
--
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org -o- https://fstop138.berrange.com :|
|: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 3/5] qmp: Added the helper stamp check.
2023-03-01 9:30 ` Daniel P. Berrangé
@ 2023-03-01 14:53 ` Toke Høiland-Jørgensen
2023-03-01 15:05 ` Daniel P. Berrangé
0 siblings, 1 reply; 60+ messages in thread
From: Toke Høiland-Jørgensen @ 2023-03-01 14:53 UTC (permalink / raw)
To: Daniel P. Berrangé
Cc: Yuri Benditovich, Andrew Melnychenko, jasowang, mst, pbonzini,
marcandre.lureau, thuth, philmd, armbru, eblake, qemu-devel,
mprivozn, yan
Daniel P. Berrangé <berrange@redhat.com> writes:
> On Tue, Feb 28, 2023 at 11:21:56PM +0100, Toke Høiland-Jørgensen wrote:
>> Daniel P. Berrangé <berrange@redhat.com> writes:
>>
>> > On Tue, Feb 28, 2023 at 08:01:51PM +0100, Toke Høiland-Jørgensen wrote:
>> >> Daniel P. Berrangé <berrange@redhat.com> writes:
>> >>
>> >> Just to interject a note on this here: the skeleton code is mostly a
>> >> convenience feature used to embed BPF programs into the calling binary.
>> >> It is perfectly possible to just have the BPF object file itself reside
>> >> directly in the file system and just use the regular libbpf APIs to load
>> >> it. Some things get a bit more cumbersome (mostly setting values of
>> >> global variables, if the BPF program uses those).
>> >>
>> >> So the JSON example above could just be a regular compiled-from-clang
>> >> BPF object file, and the management program can load that, inspect its
>> >> contents using the libbpf APIs and pass the file descriptors on to Qemu.
>> >> It's even possible to embed version information into this so that Qemu
>> >> can check if it understands the format and bail out if it doesn't - just
>> >> stick a version field in the configuration map as the first entry :)
>> >
>> > If all you have is the BPF object file is it possible to interrogate
>> > it to get a list of all the maps, and get FDs associated for them ?
>> > I had a look at the libbpf API and wasn't sure about that, it seemed
>> > like you had to know the required maps upfront ? If it is possible
>> > to auto-discover everything you need, soley from the BPF object file
>> > as input, then just dealing with that in isolation would feel simpler.
>>
>> It is. You load the object file, and bpf_object__for_each_map() lets you
>> discover which maps it contains, with the different bpf_map__*() APIs
>> telling you the properties of that map (and you can modify them too
>> before loading the object if needed).
>>
>> The only thing that's not in the object file is any initial data you
>> want to put into the map(s). But except for read-only maps that can be
>> added by userspace after loading the maps, so you could just let Qemu do
>> that...
>>
>> > It occurrs to me that exposing the BPF program as data rather than
>> > via binary will make more practical to integrate this into KubeVirt's
>> > architecture. In their deployment setup both QEMU and libvirt are
>> > running unprivileged inside a container. For any advanced nmetworking
>> > a completely separate component creates the TAP device and passes it
>> > into the container running QEMU. I don't think that the separate
>> > precisely matched helper binary would be something they can use, but
>> > it might be possible to expose a data file providing the BPF program
>> > blob and describing its maps.
>>
>> Well, "a data file providing the BPF program blob and describing its
>> maps" is basically what a BPF .o file is. It just happens to be encoded
>> in ELF format :)
>>
>> You can embed it into some other data structure and have libbpf load it
>> from a blob in memory as well as from the filesystem, though; that is
>> basically what the skeleton file does (notice the big character string
>> at the end, that's just the original .o file contents).
>
> Ok, in that case I'm really wondering why any of this helper program
> stuff was proposed. I recall the rationale was that it was impossible
> for an external program to load the BPF object on behalf of QEMU,
> because it would not know how todo that without QEMU specific
> knowledge.
I'm not sure either. Was there some bits that initially needed to be set
before the program was loaded (read-only maps or something)? Also,
upstream does encourage the use of skeletons for embedding into
applications, so it's not an unreasonable thing to start with if you
don't have the kind of deployment constraints that Qemu does in this
case.
> It looks like we can simply expose the BPF object blob to mgmt apps
> directly and get rid of this helper program entirely.
I believe so, yes. You'd still need to be sure that the BPF object file
itself comes from a trusted place, but hopefully it should be enough to
load it from a known filesystem path? (Sorry if this is a stupid
question, I only have a fuzzy idea of how all the pieces fit together
here).
-Toke
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 3/5] qmp: Added the helper stamp check.
2023-03-01 14:53 ` Toke Høiland-Jørgensen
@ 2023-03-01 15:05 ` Daniel P. Berrangé
2023-03-01 22:40 ` Toke Høiland-Jørgensen
0 siblings, 1 reply; 60+ messages in thread
From: Daniel P. Berrangé @ 2023-03-01 15:05 UTC (permalink / raw)
To: Toke Høiland-Jørgensen
Cc: Yuri Benditovich, Andrew Melnychenko, jasowang, mst, pbonzini,
marcandre.lureau, thuth, philmd, armbru, eblake, qemu-devel,
mprivozn, yan
On Wed, Mar 01, 2023 at 03:53:47PM +0100, Toke Høiland-Jørgensen wrote:
> Daniel P. Berrangé <berrange@redhat.com> writes:
>
> > On Tue, Feb 28, 2023 at 11:21:56PM +0100, Toke Høiland-Jørgensen wrote:
> >> Daniel P. Berrangé <berrange@redhat.com> writes:
> >>
> >> > On Tue, Feb 28, 2023 at 08:01:51PM +0100, Toke Høiland-Jørgensen wrote:
> >> >> Daniel P. Berrangé <berrange@redhat.com> writes:
> >> >>
> >> >> Just to interject a note on this here: the skeleton code is mostly a
> >> >> convenience feature used to embed BPF programs into the calling binary.
> >> >> It is perfectly possible to just have the BPF object file itself reside
> >> >> directly in the file system and just use the regular libbpf APIs to load
> >> >> it. Some things get a bit more cumbersome (mostly setting values of
> >> >> global variables, if the BPF program uses those).
> >> >>
> >> >> So the JSON example above could just be a regular compiled-from-clang
> >> >> BPF object file, and the management program can load that, inspect its
> >> >> contents using the libbpf APIs and pass the file descriptors on to Qemu.
> >> >> It's even possible to embed version information into this so that Qemu
> >> >> can check if it understands the format and bail out if it doesn't - just
> >> >> stick a version field in the configuration map as the first entry :)
> >> >
> >> > If all you have is the BPF object file is it possible to interrogate
> >> > it to get a list of all the maps, and get FDs associated for them ?
> >> > I had a look at the libbpf API and wasn't sure about that, it seemed
> >> > like you had to know the required maps upfront ? If it is possible
> >> > to auto-discover everything you need, soley from the BPF object file
> >> > as input, then just dealing with that in isolation would feel simpler.
> >>
> >> It is. You load the object file, and bpf_object__for_each_map() lets you
> >> discover which maps it contains, with the different bpf_map__*() APIs
> >> telling you the properties of that map (and you can modify them too
> >> before loading the object if needed).
> >>
> >> The only thing that's not in the object file is any initial data you
> >> want to put into the map(s). But except for read-only maps that can be
> >> added by userspace after loading the maps, so you could just let Qemu do
> >> that...
> >>
> >> > It occurrs to me that exposing the BPF program as data rather than
> >> > via binary will make more practical to integrate this into KubeVirt's
> >> > architecture. In their deployment setup both QEMU and libvirt are
> >> > running unprivileged inside a container. For any advanced nmetworking
> >> > a completely separate component creates the TAP device and passes it
> >> > into the container running QEMU. I don't think that the separate
> >> > precisely matched helper binary would be something they can use, but
> >> > it might be possible to expose a data file providing the BPF program
> >> > blob and describing its maps.
> >>
> >> Well, "a data file providing the BPF program blob and describing its
> >> maps" is basically what a BPF .o file is. It just happens to be encoded
> >> in ELF format :)
> >>
> >> You can embed it into some other data structure and have libbpf load it
> >> from a blob in memory as well as from the filesystem, though; that is
> >> basically what the skeleton file does (notice the big character string
> >> at the end, that's just the original .o file contents).
> >
> > Ok, in that case I'm really wondering why any of this helper program
> > stuff was proposed. I recall the rationale was that it was impossible
> > for an external program to load the BPF object on behalf of QEMU,
> > because it would not know how todo that without QEMU specific
> > knowledge.
>
> I'm not sure either. Was there some bits that initially needed to be set
> before the program was loaded (read-only maps or something)? Also,
> upstream does encourage the use of skeletons for embedding into
> applications, so it's not an unreasonable thing to start with if you
> don't have the kind of deployment constraints that Qemu does in this
> case.
>
> > It looks like we can simply expose the BPF object blob to mgmt apps
> > directly and get rid of this helper program entirely.
>
> I believe so, yes. You'd still need to be sure that the BPF object file
> itself comes from a trusted place, but hopefully it should be enough to
> load it from a known filesystem path? (Sorry if this is a stupid
> question, I only have a fuzzy idea of how all the pieces fit together
> here).
It could be from a well known location on the filesystem, but might
be better to make it possible to query it from QMP, which is mostly
safe *provided* you've not yet started guest CPUs running. It could
be queried at startup and then cached for future use.
With regards,
Daniel
--
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org -o- https://fstop138.berrange.com :|
|: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 3/5] qmp: Added the helper stamp check.
2023-03-01 15:05 ` Daniel P. Berrangé
@ 2023-03-01 22:40 ` Toke Høiland-Jørgensen
2023-03-22 13:26 ` Andrew Melnichenko
0 siblings, 1 reply; 60+ messages in thread
From: Toke Høiland-Jørgensen @ 2023-03-01 22:40 UTC (permalink / raw)
To: Daniel P. Berrangé
Cc: Yuri Benditovich, Andrew Melnychenko, jasowang, mst, pbonzini,
marcandre.lureau, thuth, philmd, armbru, eblake, qemu-devel,
mprivozn, yan
Daniel P. Berrangé <berrange@redhat.com> writes:
> On Wed, Mar 01, 2023 at 03:53:47PM +0100, Toke Høiland-Jørgensen wrote:
>> Daniel P. Berrangé <berrange@redhat.com> writes:
>>
>> > On Tue, Feb 28, 2023 at 11:21:56PM +0100, Toke Høiland-Jørgensen wrote:
>> >> Daniel P. Berrangé <berrange@redhat.com> writes:
>> >>
>> >> > On Tue, Feb 28, 2023 at 08:01:51PM +0100, Toke Høiland-Jørgensen wrote:
>> >> >> Daniel P. Berrangé <berrange@redhat.com> writes:
>> >> >>
>> >> >> Just to interject a note on this here: the skeleton code is mostly a
>> >> >> convenience feature used to embed BPF programs into the calling binary.
>> >> >> It is perfectly possible to just have the BPF object file itself reside
>> >> >> directly in the file system and just use the regular libbpf APIs to load
>> >> >> it. Some things get a bit more cumbersome (mostly setting values of
>> >> >> global variables, if the BPF program uses those).
>> >> >>
>> >> >> So the JSON example above could just be a regular compiled-from-clang
>> >> >> BPF object file, and the management program can load that, inspect its
>> >> >> contents using the libbpf APIs and pass the file descriptors on to Qemu.
>> >> >> It's even possible to embed version information into this so that Qemu
>> >> >> can check if it understands the format and bail out if it doesn't - just
>> >> >> stick a version field in the configuration map as the first entry :)
>> >> >
>> >> > If all you have is the BPF object file is it possible to interrogate
>> >> > it to get a list of all the maps, and get FDs associated for them ?
>> >> > I had a look at the libbpf API and wasn't sure about that, it seemed
>> >> > like you had to know the required maps upfront ? If it is possible
>> >> > to auto-discover everything you need, soley from the BPF object file
>> >> > as input, then just dealing with that in isolation would feel simpler.
>> >>
>> >> It is. You load the object file, and bpf_object__for_each_map() lets you
>> >> discover which maps it contains, with the different bpf_map__*() APIs
>> >> telling you the properties of that map (and you can modify them too
>> >> before loading the object if needed).
>> >>
>> >> The only thing that's not in the object file is any initial data you
>> >> want to put into the map(s). But except for read-only maps that can be
>> >> added by userspace after loading the maps, so you could just let Qemu do
>> >> that...
>> >>
>> >> > It occurrs to me that exposing the BPF program as data rather than
>> >> > via binary will make more practical to integrate this into KubeVirt's
>> >> > architecture. In their deployment setup both QEMU and libvirt are
>> >> > running unprivileged inside a container. For any advanced nmetworking
>> >> > a completely separate component creates the TAP device and passes it
>> >> > into the container running QEMU. I don't think that the separate
>> >> > precisely matched helper binary would be something they can use, but
>> >> > it might be possible to expose a data file providing the BPF program
>> >> > blob and describing its maps.
>> >>
>> >> Well, "a data file providing the BPF program blob and describing its
>> >> maps" is basically what a BPF .o file is. It just happens to be encoded
>> >> in ELF format :)
>> >>
>> >> You can embed it into some other data structure and have libbpf load it
>> >> from a blob in memory as well as from the filesystem, though; that is
>> >> basically what the skeleton file does (notice the big character string
>> >> at the end, that's just the original .o file contents).
>> >
>> > Ok, in that case I'm really wondering why any of this helper program
>> > stuff was proposed. I recall the rationale was that it was impossible
>> > for an external program to load the BPF object on behalf of QEMU,
>> > because it would not know how todo that without QEMU specific
>> > knowledge.
>>
>> I'm not sure either. Was there some bits that initially needed to be set
>> before the program was loaded (read-only maps or something)? Also,
>> upstream does encourage the use of skeletons for embedding into
>> applications, so it's not an unreasonable thing to start with if you
>> don't have the kind of deployment constraints that Qemu does in this
>> case.
>>
>> > It looks like we can simply expose the BPF object blob to mgmt apps
>> > directly and get rid of this helper program entirely.
>>
>> I believe so, yes. You'd still need to be sure that the BPF object file
>> itself comes from a trusted place, but hopefully it should be enough to
>> load it from a known filesystem path? (Sorry if this is a stupid
>> question, I only have a fuzzy idea of how all the pieces fit together
>> here).
>
> It could be from a well known location on the filesystem, but might
> be better to make it possible to query it from QMP, which is mostly
> safe *provided* you've not yet started guest CPUs running. It could
> be queried at startup and then cached for future use.
Right, I don't have a strong opinion about the exact mechanism, just
wanted to convey a general "loading an untrusted BPF program is bad"
kind of vibe ;)
-Toke
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 3/5] qmp: Added the helper stamp check.
2023-03-01 22:40 ` Toke Høiland-Jørgensen
@ 2023-03-22 13:26 ` Andrew Melnichenko
2023-03-22 15:59 ` Daniel P. Berrangé
0 siblings, 1 reply; 60+ messages in thread
From: Andrew Melnichenko @ 2023-03-22 13:26 UTC (permalink / raw)
To: Toke Høiland-Jørgensen
Cc: Daniel P. Berrangé,
Yuri Benditovich, jasowang, mst, pbonzini, marcandre.lureau,
thuth, philmd, armbru, eblake, qemu-devel, mprivozn, yan
Hi all,
I've researched an issue a bit. The solution with passing eBPF blob
and loading in the Libvirt looks promising.
Overall, the possible solution looks like this:
* Libvirt checks virtio-net properties and understands that eBPF
steering may be required.
* Libvirt requests eBPF blob through QMP.
* Libvirt loads blob for virtio-net and passes fds from eBPF to QEMU.
I think that it's a good idea to pass only eBPF blob without
additional metainformation. Most metainfo that we need could be
retrieved from eBPF blob, and the only question is to pass fds
sequence to QEMU.
I propose to pass them as they appear in the blob itself, like
"virtio-net-pci,ebpf_rss_fds=<prog>,<map1>,<map2>,<map3>...".
Also, I think it's a good idea to make a "general" QMP request for
eBPF blobs. Something like "request_ebpf <arg>"(g.e "request_ebpf
virtio-net-rss").
I'll prepare new RFC patches if you have questions or something to
discuss, please let me know.
On Thu, Mar 2, 2023 at 12:40 AM Toke Høiland-Jørgensen <toke@redhat.com> wrote:
>
> Daniel P. Berrangé <berrange@redhat.com> writes:
>
> > On Wed, Mar 01, 2023 at 03:53:47PM +0100, Toke Høiland-Jørgensen wrote:
> >> Daniel P. Berrangé <berrange@redhat.com> writes:
> >>
> >> > On Tue, Feb 28, 2023 at 11:21:56PM +0100, Toke Høiland-Jørgensen wrote:
> >> >> Daniel P. Berrangé <berrange@redhat.com> writes:
> >> >>
> >> >> > On Tue, Feb 28, 2023 at 08:01:51PM +0100, Toke Høiland-Jørgensen wrote:
> >> >> >> Daniel P. Berrangé <berrange@redhat.com> writes:
> >> >> >>
> >> >> >> Just to interject a note on this here: the skeleton code is mostly a
> >> >> >> convenience feature used to embed BPF programs into the calling binary.
> >> >> >> It is perfectly possible to just have the BPF object file itself reside
> >> >> >> directly in the file system and just use the regular libbpf APIs to load
> >> >> >> it. Some things get a bit more cumbersome (mostly setting values of
> >> >> >> global variables, if the BPF program uses those).
> >> >> >>
> >> >> >> So the JSON example above could just be a regular compiled-from-clang
> >> >> >> BPF object file, and the management program can load that, inspect its
> >> >> >> contents using the libbpf APIs and pass the file descriptors on to Qemu.
> >> >> >> It's even possible to embed version information into this so that Qemu
> >> >> >> can check if it understands the format and bail out if it doesn't - just
> >> >> >> stick a version field in the configuration map as the first entry :)
> >> >> >
> >> >> > If all you have is the BPF object file is it possible to interrogate
> >> >> > it to get a list of all the maps, and get FDs associated for them ?
> >> >> > I had a look at the libbpf API and wasn't sure about that, it seemed
> >> >> > like you had to know the required maps upfront ? If it is possible
> >> >> > to auto-discover everything you need, soley from the BPF object file
> >> >> > as input, then just dealing with that in isolation would feel simpler.
> >> >>
> >> >> It is. You load the object file, and bpf_object__for_each_map() lets you
> >> >> discover which maps it contains, with the different bpf_map__*() APIs
> >> >> telling you the properties of that map (and you can modify them too
> >> >> before loading the object if needed).
> >> >>
> >> >> The only thing that's not in the object file is any initial data you
> >> >> want to put into the map(s). But except for read-only maps that can be
> >> >> added by userspace after loading the maps, so you could just let Qemu do
> >> >> that...
> >> >>
> >> >> > It occurrs to me that exposing the BPF program as data rather than
> >> >> > via binary will make more practical to integrate this into KubeVirt's
> >> >> > architecture. In their deployment setup both QEMU and libvirt are
> >> >> > running unprivileged inside a container. For any advanced nmetworking
> >> >> > a completely separate component creates the TAP device and passes it
> >> >> > into the container running QEMU. I don't think that the separate
> >> >> > precisely matched helper binary would be something they can use, but
> >> >> > it might be possible to expose a data file providing the BPF program
> >> >> > blob and describing its maps.
> >> >>
> >> >> Well, "a data file providing the BPF program blob and describing its
> >> >> maps" is basically what a BPF .o file is. It just happens to be encoded
> >> >> in ELF format :)
> >> >>
> >> >> You can embed it into some other data structure and have libbpf load it
> >> >> from a blob in memory as well as from the filesystem, though; that is
> >> >> basically what the skeleton file does (notice the big character string
> >> >> at the end, that's just the original .o file contents).
> >> >
> >> > Ok, in that case I'm really wondering why any of this helper program
> >> > stuff was proposed. I recall the rationale was that it was impossible
> >> > for an external program to load the BPF object on behalf of QEMU,
> >> > because it would not know how todo that without QEMU specific
> >> > knowledge.
> >>
> >> I'm not sure either. Was there some bits that initially needed to be set
> >> before the program was loaded (read-only maps or something)? Also,
> >> upstream does encourage the use of skeletons for embedding into
> >> applications, so it's not an unreasonable thing to start with if you
> >> don't have the kind of deployment constraints that Qemu does in this
> >> case.
> >>
> >> > It looks like we can simply expose the BPF object blob to mgmt apps
> >> > directly and get rid of this helper program entirely.
> >>
> >> I believe so, yes. You'd still need to be sure that the BPF object file
> >> itself comes from a trusted place, but hopefully it should be enough to
> >> load it from a known filesystem path? (Sorry if this is a stupid
> >> question, I only have a fuzzy idea of how all the pieces fit together
> >> here).
> >
> > It could be from a well known location on the filesystem, but might
> > be better to make it possible to query it from QMP, which is mostly
> > safe *provided* you've not yet started guest CPUs running. It could
> > be queried at startup and then cached for future use.
>
> Right, I don't have a strong opinion about the exact mechanism, just
> wanted to convey a general "loading an untrusted BPF program is bad"
> kind of vibe ;)
>
> -Toke
>
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 3/5] qmp: Added the helper stamp check.
2023-03-22 13:26 ` Andrew Melnichenko
@ 2023-03-22 15:59 ` Daniel P. Berrangé
0 siblings, 0 replies; 60+ messages in thread
From: Daniel P. Berrangé @ 2023-03-22 15:59 UTC (permalink / raw)
To: Andrew Melnichenko
Cc: Toke Høiland-Jørgensen, Yuri Benditovich, jasowang,
mst, pbonzini, marcandre.lureau, thuth, philmd, armbru, eblake,
qemu-devel, mprivozn, yan
On Wed, Mar 22, 2023 at 03:26:59PM +0200, Andrew Melnichenko wrote:
> Hi all,
> I've researched an issue a bit. The solution with passing eBPF blob
> and loading in the Libvirt looks promising.
> Overall, the possible solution looks like this:
> * Libvirt checks virtio-net properties and understands that eBPF
> steering may be required.
> * Libvirt requests eBPF blob through QMP.
> * Libvirt loads blob for virtio-net and passes fds from eBPF to QEMU.
>
> I think that it's a good idea to pass only eBPF blob without
> additional metainformation. Most metainfo that we need could be
> retrieved from eBPF blob, and the only question is to pass fds
> sequence to QEMU.
> I propose to pass them as they appear in the blob itself, like
> "virtio-net-pci,ebpf_rss_fds=<prog>,<map1>,<map2>,<map3>...".
Using ',' for separating FDs is a bad idea, because ',' is already
used for separating QemuOpts arguments.
With -netdev we use ':' for spearating FDs with vhostfds= and fds=
arguments, so I'd suggest following that practice.
> Also, I think it's a good idea to make a "general" QMP request for
> eBPF blobs. Something like "request_ebpf <arg>"(g.e "request_ebpf
> virtio-net-rss").
That's reasonable as a future proofing idea I think.
With regards,
Daniel
--
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org -o- https://fstop138.berrange.com :|
|: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 3/5] qmp: Added the helper stamp check.
2023-02-28 18:04 ` Daniel P. Berrangé
2023-02-28 19:01 ` Toke Høiland-Jørgensen
@ 2023-02-28 19:01 ` Daniel P. Berrangé
2023-03-01 6:49 ` Yuri Benditovich
2 siblings, 0 replies; 60+ messages in thread
From: Daniel P. Berrangé @ 2023-02-28 19:01 UTC (permalink / raw)
To: Yuri Benditovich, Andrew Melnychenko, jasowang, mst, pbonzini,
marcandre.lureau, thuth, philmd, armbru, eblake, qemu-devel,
toke, mprivozn, yan
On Tue, Feb 28, 2023 at 06:04:51PM +0000, Daniel P. Berrangé wrote:
> On Tue, Feb 28, 2023 at 11:56:27AM +0200, Yuri Benditovich wrote:
> > On Mon, Feb 20, 2023 at 11:50 AM Daniel P. Berrangé <berrange@redhat.com>
> > wrote:
> >
> > > On Sun, Feb 19, 2023 at 06:20:58PM +0200, Andrew Melnychenko wrote:
> > > > Added a function to check the stamp in the helper.
> > > > eBPF helper should have a special symbol that generates during the build.
> > > > QEMU checks the helper and determines that it fits, so the helper
> > > > will produce proper output.
> > >
> > > I think this is quite limiting for in place upgrades.
> > >
> > > Consider this scenario
> > >
> > > * Host has QEMU 8.1.0 installed
> > > * VM is running QEMU 8.1.0
> > > * QEMU 8.1.1 is released with a bug fix in the EBF program
> > > * Host is upgraded to QEMU 8.1.1
> > > * User attempts to hotplug a NIC to the running VM
> > >
> > > IIUC this last step is going to fail because we'll be loading
> > > the EBF program from 8.1.1 and so its hash is different from
> > > that expected by the QEMU 8.1.0 that the pre-existing VM is
> > > running.
> > >
> > > Indeed we did not take in account the in-place upgrade.
> >
> >
> >
> > > If some changes to the EBF program are not going to be back
> > > compatible from the POV of the QEMU process, should we instead
> > > be versioning the EBF program. eg so new QEMU will ship both
> > > the old and new versions of the EBF program.
> >
> > This does not seem to be an elegant option: QEMU theoretically can include
> > different eBPF programs but it hardly can interface with each one of them.
> > The code of QEMU (access to eBPF maps etc) includes header files which eBPF
> > of the day is being built with them.
> >
> > I see 2 options to address this issue (of course there are more)
> > 1. Build and install qemu-rss-helper-<hash> executable. Libvirt will always
> > have a correct name, so for the running instance it will use
> > qemu-rss-helper-<old-hash>, for the new instance it will use
> > qemu-rss-helper-<new-hash>
>
> We'll get an ever growing number of program variants we need to
> build & distribute with each new QEMU release.
>
> > 2. Build the helper executable and link it inside qemu as a blob. Libvirt
> > will always retrieve the executable to the temporary file name and use it.
> > So the retrieved helper will always be compatible with QEMU. I'm not sure
> > what is the most portable way to do that.
>
> QEMU is considered an untrusted process, so there's no way we're going
> to ask it to give us an ELF binary and then execute that in privileged
> context.
>
> > Does one of these seem suitable?
>
> Neither feels very appealing to me.
>
> I've been trying to understand the eBPF code we're dealing with in a
> little more detail.
>
> IIUC, QEMU, or rather the virtio-net driver needs to receive one FD
> for the BPF program, and one or more FDs for the BPF maps that the
> program uses. Currently it uses 3 maps, so needs 3 map FDs on top of
> the program FD.
>
> The helper program that is proposed here calls ebpf_rss_load() to
> load the program and get back a struct which gives access to the
> 4 FDs, which are then sent to the mgmt app, which forwards them
> onto QEMU.
>
> The ebpf_rss_load() method is making use of various structs that
> are specific to the RSS program implementation, but does not seems
> to do anything especially interesting. It calls into rss_bpf__open()
> which eventually gets around to calling rss_bpf__create_skeleton
> which is where the interesting stuff happens.
>
> This rss_bpf__create_skeleton() method is implemented in terms of
> totally generic libbpf APIs, and has the actual blob that is the
> BPF program.
>
> Looking at what this does, I feel it should be trivial for a mgmt
> app to implement equivalent logic to rss_bpf__create_skeleton in a
> generic manner, if we could just expose the program blob and the
> map names to the mgmt app. eg a simple json file
>
> {
> "maps": [
> "tap_rss_map_configurations",
> "tap_rss_map_indirection_table",
> "tap_rss_map_toeplitz_key",
> ],
> "program": "....the big blob encoded in base64..."
> }
>
> if we installed that file are /usr/share/qemu/bpf/net-rss.json
> then when a QEMU process is started, the mgmt app capture the
> data in this JSON file. It now has enough info to create the
> EBPF programs needed and pass the FDs over to QEMU. This would
> be robust against QEMU software upgrades, and not tied to the
> specific EBPF program imlp. We can add or remove maps / change
> their names etc any time, as the details in the JSON descriptor
> can be updated. This avoids need for any special helper program
> to be provided by QEMU with the problems that is throwing up
> for us.
It occurrs to me that exposing the BPF program as data rather than
via binary will make more practical to integrate this into KubeVirt's
architecture. In their deployment setup both QEMU and libvirt are
running unprivileged inside a container. For any advanced nmetworking
a completely separate component creates the TAP device and passes it
into the container running QEMU. I don't think that the separate
precisely matched helper binary would be something they can use, but
it might be possible to expose a data file providing the BPF program
blob and describing its maps.
With regards,
Daniel
--
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org -o- https://fstop138.berrange.com :|
|: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 3/5] qmp: Added the helper stamp check.
2023-02-28 18:04 ` Daniel P. Berrangé
2023-02-28 19:01 ` Toke Høiland-Jørgensen
2023-02-28 19:01 ` Daniel P. Berrangé
@ 2023-03-01 6:49 ` Yuri Benditovich
2023-03-01 9:31 ` Daniel P. Berrangé
2 siblings, 1 reply; 60+ messages in thread
From: Yuri Benditovich @ 2023-03-01 6:49 UTC (permalink / raw)
To: Daniel P. Berrangé
Cc: Andrew Melnychenko, jasowang, mst, pbonzini, marcandre.lureau,
thuth, philmd, armbru, eblake, qemu-devel, toke, mprivozn, yan
[-- Attachment #1: Type: text/plain, Size: 5870 bytes --]
On Tue, Feb 28, 2023 at 8:05 PM Daniel P. Berrangé <berrange@redhat.com>
wrote:
> On Tue, Feb 28, 2023 at 11:56:27AM +0200, Yuri Benditovich wrote:
> > On Mon, Feb 20, 2023 at 11:50 AM Daniel P. Berrangé <berrange@redhat.com
> >
> > wrote:
> >
> > > On Sun, Feb 19, 2023 at 06:20:58PM +0200, Andrew Melnychenko wrote:
> > > > Added a function to check the stamp in the helper.
> > > > eBPF helper should have a special symbol that generates during the
> build.
> > > > QEMU checks the helper and determines that it fits, so the helper
> > > > will produce proper output.
> > >
> > > I think this is quite limiting for in place upgrades.
> > >
> > > Consider this scenario
> > >
> > > * Host has QEMU 8.1.0 installed
> > > * VM is running QEMU 8.1.0
> > > * QEMU 8.1.1 is released with a bug fix in the EBF program
> > > * Host is upgraded to QEMU 8.1.1
> > > * User attempts to hotplug a NIC to the running VM
> > >
> > > IIUC this last step is going to fail because we'll be loading
> > > the EBF program from 8.1.1 and so its hash is different from
> > > that expected by the QEMU 8.1.0 that the pre-existing VM is
> > > running.
> > >
> > > Indeed we did not take in account the in-place upgrade.
> >
> >
> >
> > > If some changes to the EBF program are not going to be back
> > > compatible from the POV of the QEMU process, should we instead
> > > be versioning the EBF program. eg so new QEMU will ship both
> > > the old and new versions of the EBF program.
> >
> > This does not seem to be an elegant option: QEMU theoretically can
> include
> > different eBPF programs but it hardly can interface with each one of
> them.
> > The code of QEMU (access to eBPF maps etc) includes header files which
> eBPF
> > of the day is being built with them.
> >
> > I see 2 options to address this issue (of course there are more)
> > 1. Build and install qemu-rss-helper-<hash> executable. Libvirt will
> always
> > have a correct name, so for the running instance it will use
> > qemu-rss-helper-<old-hash>, for the new instance it will use
> > qemu-rss-helper-<new-hash>
>
> We'll get an ever growing number of program variants we need to
> build & distribute with each new QEMU release.
>
New release of the qemu-rss-helper-<new-hash> will be created in fact only
when the eBPF binary is updated.
This does not happen on each release. But yes, this looks like versioning
of all the shared libraries.
>
> > 2. Build the helper executable and link it inside qemu as a blob. Libvirt
> > will always retrieve the executable to the temporary file name and use
> it.
> > So the retrieved helper will always be compatible with QEMU. I'm not sure
> > what is the most portable way to do that.
>
> QEMU is considered an untrusted process, so there's no way we're going
> to ask it to give us an ELF binary and then execute that in privileged
> context.
>
> > Does one of these seem suitable?
>
> Neither feels very appealing to me.
>
> I've been trying to understand the eBPF code we're dealing with in a
> little more detail.
>
> IIUC, QEMU, or rather the virtio-net driver needs to receive one FD
> for the BPF program, and one or more FDs for the BPF maps that the
> program uses. Currently it uses 3 maps, so needs 3 map FDs on top of
> the program FD.
>
> The helper program that is proposed here calls ebpf_rss_load() to
> load the program and get back a struct which gives access to the
> 4 FDs, which are then sent to the mgmt app, which forwards them
> onto QEMU.
>
> The ebpf_rss_load() method is making use of various structs that
> are specific to the RSS program implementation, but does not seems
> to do anything especially interesting. It calls into rss_bpf__open()
> which eventually gets around to calling rss_bpf__create_skeleton
> which is where the interesting stuff happens.
>
> This rss_bpf__create_skeleton() method is implemented in terms of
> totally generic libbpf APIs, and has the actual blob that is the
> BPF program.
>
> Looking at what this does, I feel it should be trivial for a mgmt
> app to implement equivalent logic to rss_bpf__create_skeleton in a
> generic manner, if we could just expose the program blob and the
> map names to the mgmt app. eg a simple json file
>
> {
> "maps": [
> "tap_rss_map_configurations",
> "tap_rss_map_indirection_table",
> "tap_rss_map_toeplitz_key",
> ],
> "program": "....the big blob encoded in base64..."
> }
>
> if we installed that file are /usr/share/qemu/bpf/net-rss.json
> then when a QEMU process is started, the mgmt app capture the
> data in this JSON file. It now has enough info to create the
> EBPF programs needed and pass the FDs over to QEMU. This would
> be robust against QEMU software upgrades, and not tied to the
> specific EBPF program imlp. We can add or remove maps / change
> their names etc any time, as the details in the JSON descriptor
> can be updated. This avoids need for any special helper program
> to be provided by QEMU with the problems that is throwing up
> for us.
>
If I understand correctly, the libvirt will have the same problem in the
in-place update scenario with the JSON file
Let's say that there is no virtio-net device at the initial start and the
libvirt does not need to load the JSON file.
On the later hot plug of virtio-net it will go to the JSON file after the
update, correct?
>
> What am I missing ? This seems pretty straightforward to
> achieve from what I see.
>
> With regards,
> Daniel
> --
> |: https://berrange.com -o-
> https://www.flickr.com/photos/dberrange :|
> |: https://libvirt.org -o-
> https://fstop138.berrange.com :|
> |: https://entangle-photo.org -o-
> https://www.instagram.com/dberrange :|
>
>
[-- Attachment #2: Type: text/html, Size: 7692 bytes --]
^ permalink raw reply [flat|nested] 60+ messages in thread
* Re: [PATCH 3/5] qmp: Added the helper stamp check.
2023-03-01 6:49 ` Yuri Benditovich
@ 2023-03-01 9:31 ` Daniel P. Berrangé
0 siblings, 0 replies; 60+ messages in thread
From: Daniel P. Berrangé @ 2023-03-01 9:31 UTC (permalink / raw)
To: Yuri Benditovich
Cc: Andrew Melnychenko, jasowang, mst, pbonzini, marcandre.lureau,
thuth, philmd, armbru, eblake, qemu-devel, toke, mprivozn, yan
On Wed, Mar 01, 2023 at 08:49:42AM +0200, Yuri Benditovich wrote:
> On Tue, Feb 28, 2023 at 8:05 PM Daniel P. Berrangé <berrange@redhat.com>
> wrote:
>
> > On Tue, Feb 28, 2023 at 11:56:27AM +0200, Yuri Benditovich wrote:
> > > On Mon, Feb 20, 2023 at 11:50 AM Daniel P. Berrangé <berrange@redhat.com
> > >
> > > wrote:
> > >
> > > > On Sun, Feb 19, 2023 at 06:20:58PM +0200, Andrew Melnychenko wrote:
> > > > > Added a function to check the stamp in the helper.
> > > > > eBPF helper should have a special symbol that generates during the
> > build.
> > > > > QEMU checks the helper and determines that it fits, so the helper
> > > > > will produce proper output.
> > > >
> > > > I think this is quite limiting for in place upgrades.
> > > >
> > > > Consider this scenario
> > > >
> > > > * Host has QEMU 8.1.0 installed
> > > > * VM is running QEMU 8.1.0
> > > > * QEMU 8.1.1 is released with a bug fix in the EBF program
> > > > * Host is upgraded to QEMU 8.1.1
> > > > * User attempts to hotplug a NIC to the running VM
> > > >
> > > > IIUC this last step is going to fail because we'll be loading
> > > > the EBF program from 8.1.1 and so its hash is different from
> > > > that expected by the QEMU 8.1.0 that the pre-existing VM is
> > > > running.
> > > >
> > > > Indeed we did not take in account the in-place upgrade.
> > >
> > >
> > >
> > > > If some changes to the EBF program are not going to be back
> > > > compatible from the POV of the QEMU process, should we instead
> > > > be versioning the EBF program. eg so new QEMU will ship both
> > > > the old and new versions of the EBF program.
> > >
> > > This does not seem to be an elegant option: QEMU theoretically can
> > include
> > > different eBPF programs but it hardly can interface with each one of
> > them.
> > > The code of QEMU (access to eBPF maps etc) includes header files which
> > eBPF
> > > of the day is being built with them.
> > >
> > > I see 2 options to address this issue (of course there are more)
> > > 1. Build and install qemu-rss-helper-<hash> executable. Libvirt will
> > always
> > > have a correct name, so for the running instance it will use
> > > qemu-rss-helper-<old-hash>, for the new instance it will use
> > > qemu-rss-helper-<new-hash>
> >
> > We'll get an ever growing number of program variants we need to
> > build & distribute with each new QEMU release.
> >
>
> New release of the qemu-rss-helper-<new-hash> will be created in fact only
> when the eBPF binary is updated.
> This does not happen on each release. But yes, this looks like versioning
> of all the shared libraries.
>
>
> >
> > > 2. Build the helper executable and link it inside qemu as a blob. Libvirt
> > > will always retrieve the executable to the temporary file name and use
> > it.
> > > So the retrieved helper will always be compatible with QEMU. I'm not sure
> > > what is the most portable way to do that.
> >
> > QEMU is considered an untrusted process, so there's no way we're going
> > to ask it to give us an ELF binary and then execute that in privileged
> > context.
> >
> > > Does one of these seem suitable?
> >
> > Neither feels very appealing to me.
> >
> > I've been trying to understand the eBPF code we're dealing with in a
> > little more detail.
> >
> > IIUC, QEMU, or rather the virtio-net driver needs to receive one FD
> > for the BPF program, and one or more FDs for the BPF maps that the
> > program uses. Currently it uses 3 maps, so needs 3 map FDs on top of
> > the program FD.
> >
> > The helper program that is proposed here calls ebpf_rss_load() to
> > load the program and get back a struct which gives access to the
> > 4 FDs, which are then sent to the mgmt app, which forwards them
> > onto QEMU.
> >
> > The ebpf_rss_load() method is making use of various structs that
> > are specific to the RSS program implementation, but does not seems
> > to do anything especially interesting. It calls into rss_bpf__open()
> > which eventually gets around to calling rss_bpf__create_skeleton
> > which is where the interesting stuff happens.
> >
> > This rss_bpf__create_skeleton() method is implemented in terms of
> > totally generic libbpf APIs, and has the actual blob that is the
> > BPF program.
> >
> > Looking at what this does, I feel it should be trivial for a mgmt
> > app to implement equivalent logic to rss_bpf__create_skeleton in a
> > generic manner, if we could just expose the program blob and the
> > map names to the mgmt app. eg a simple json file
> >
> > {
> > "maps": [
> > "tap_rss_map_configurations",
> > "tap_rss_map_indirection_table",
> > "tap_rss_map_toeplitz_key",
> > ],
> > "program": "....the big blob encoded in base64..."
> > }
> >
> > if we installed that file are /usr/share/qemu/bpf/net-rss.json
> > then when a QEMU process is started, the mgmt app capture the
> > data in this JSON file. It now has enough info to create the
> > EBPF programs needed and pass the FDs over to QEMU. This would
> > be robust against QEMU software upgrades, and not tied to the
> > specific EBPF program imlp. We can add or remove maps / change
> > their names etc any time, as the details in the JSON descriptor
> > can be updated. This avoids need for any special helper program
> > to be provided by QEMU with the problems that is throwing up
> > for us.
> >
>
> If I understand correctly, the libvirt will have the same problem in the
> in-place update scenario with the JSON file
> Let's say that there is no virtio-net device at the initial start and the
> libvirt does not need to load the JSON file.
> On the later hot plug of virtio-net it will go to the JSON file after the
> update, correct?
Libvirt does work to detect capabilities of a QEMU binary before starting
a guest, and caches this info for later use when the VM is running. I'm
suggesting that libvirt can detect the BPF program info at this time and
cache it, regardless of whether it initially needs to use the BPF program.
With regards,
Daniel
--
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org -o- https://fstop138.berrange.com :|
|: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
^ permalink raw reply [flat|nested] 60+ messages in thread