* [PATCH v2] block/rbd: Add support for rbd image encryption
@ 2021-06-27 11:46 Or Ozeri
2021-06-27 13:46 ` Ilya Dryomov
0 siblings, 1 reply; 3+ messages in thread
From: Or Ozeri @ 2021-06-27 11:46 UTC (permalink / raw)
To: qemu-devel
Cc: kwolf, berrange, qemu-block, dannyh, oro, idryomov, to.my.trociny
Starting from ceph Pacific, RBD has built-in support for image-level encryption.
Currently supported formats are LUKS version 1 and 2.
There are 2 new relevant librbd APIs for controlling encryption, both expect an
open image context:
rbd_encryption_format: formats an image (i.e. writes the LUKS header)
rbd_encryption_load: loads encryptor/decryptor to the image IO stack
This commit extends the qemu rbd driver API to support the above.
Signed-off-by: Or Ozeri <oro@il.ibm.com>
---
v2: handle encryption info only for the case where encryption is not loaded
---
block/rbd.c | 361 ++++++++++++++++++++++++++++++++++++++++++-
qapi/block-core.json | 110 ++++++++++++-
2 files changed, 465 insertions(+), 6 deletions(-)
diff --git a/block/rbd.c b/block/rbd.c
index f098a89c7b..8edd1be49e 100644
--- a/block/rbd.c
+++ b/block/rbd.c
@@ -73,6 +73,18 @@
#define LIBRBD_USE_IOVEC 0
#endif
+#define RBD_ENCRYPTION_LUKS_HEADER_VERIFICATION_LEN 8
+
+static const char rbd_luks_header_verification[
+ RBD_ENCRYPTION_LUKS_HEADER_VERIFICATION_LEN] = {
+ 'L', 'U', 'K', 'S', 0xBA, 0xBE, 0, 1
+};
+
+static const char rbd_luks2_header_verification[
+ RBD_ENCRYPTION_LUKS_HEADER_VERIFICATION_LEN] = {
+ 'L', 'U', 'K', 'S', 0xBA, 0xBE, 0, 2
+};
+
typedef enum {
RBD_AIO_READ,
RBD_AIO_WRITE,
@@ -341,6 +353,203 @@ static void qemu_rbd_memset(RADOSCB *rcb, int64_t offs)
}
}
+#ifdef LIBRBD_SUPPORTS_ENCRYPTION
+static int qemu_rbd_convert_luks_options(
+ RbdEncryptionOptionsLUKSBase *luks_opts,
+ char **passphrase,
+ size_t *passphrase_len,
+ Error **errp)
+{
+ return qcrypto_secret_lookup(luks_opts->key_secret, (uint8_t **)passphrase,
+ passphrase_len, errp);
+}
+
+static int qemu_rbd_convert_luks_create_options(
+ RbdEncryptionCreateOptionsLUKSBase *luks_opts,
+ rbd_encryption_algorithm_t *alg,
+ char **passphrase,
+ size_t *passphrase_len,
+ Error **errp)
+{
+ int r = 0;
+
+ r = qemu_rbd_convert_luks_options(
+ qapi_RbdEncryptionCreateOptionsLUKSBase_base(luks_opts),
+ passphrase, passphrase_len, errp);
+ if (r < 0) {
+ return r;
+ }
+
+ if (luks_opts->has_cipher_alg) {
+ switch (luks_opts->cipher_alg) {
+ case QCRYPTO_CIPHER_ALG_AES_128: {
+ *alg = RBD_ENCRYPTION_ALGORITHM_AES128;
+ break;
+ }
+ case QCRYPTO_CIPHER_ALG_AES_256: {
+ *alg = RBD_ENCRYPTION_ALGORITHM_AES256;
+ break;
+ }
+ default: {
+ r = -ENOTSUP;
+ error_setg_errno(errp, -r, "unknown encryption algorithm: %u",
+ luks_opts->cipher_alg);
+ return r;
+ }
+ }
+ } else {
+ /* default alg */
+ *alg = RBD_ENCRYPTION_ALGORITHM_AES256;
+ }
+
+ return 0;
+}
+
+static int qemu_rbd_encryption_format(rbd_image_t image,
+ RbdEncryptionCreateOptions *encrypt,
+ Error **errp)
+{
+ int r = 0;
+ g_autofree char *passphrase = NULL;
+ size_t passphrase_len;
+ rbd_encryption_format_t format;
+ rbd_encryption_options_t opts;
+ rbd_encryption_luks1_format_options_t luks_opts;
+ rbd_encryption_luks2_format_options_t luks2_opts;
+ size_t opts_size;
+ uint64_t raw_size, effective_size;
+
+ r = rbd_get_size(image, &raw_size);
+ if (r < 0) {
+ error_setg_errno(errp, -r, "cannot get raw image size");
+ return r;
+ }
+
+ switch (encrypt->format) {
+ case RBD_IMAGE_ENCRYPTION_FORMAT_LUKS: {
+ memset(&luks_opts, 0, sizeof(luks_opts));
+ format = RBD_ENCRYPTION_FORMAT_LUKS1;
+ opts = &luks_opts;
+ opts_size = sizeof(luks_opts);
+ r = qemu_rbd_convert_luks_create_options(
+ qapi_RbdEncryptionCreateOptionsLUKS_base(&encrypt->u.luks),
+ &luks_opts.alg, &passphrase, &passphrase_len, errp);
+ if (r < 0) {
+ return r;
+ }
+ luks_opts.passphrase = passphrase;
+ luks_opts.passphrase_size = passphrase_len;
+ break;
+ }
+ case RBD_IMAGE_ENCRYPTION_FORMAT_LUKS2: {
+ memset(&luks2_opts, 0, sizeof(luks2_opts));
+ format = RBD_ENCRYPTION_FORMAT_LUKS2;
+ opts = &luks2_opts;
+ opts_size = sizeof(luks2_opts);
+ r = qemu_rbd_convert_luks_create_options(
+ qapi_RbdEncryptionCreateOptionsLUKS2_base(
+ &encrypt->u.luks2),
+ &luks2_opts.alg, &passphrase, &passphrase_len, errp);
+ if (r < 0) {
+ return r;
+ }
+ luks2_opts.passphrase = passphrase;
+ luks2_opts.passphrase_size = passphrase_len;
+ break;
+ }
+ default: {
+ r = -ENOTSUP;
+ error_setg_errno(
+ errp, -r, "unknown image encryption format: %u",
+ encrypt->format);
+ return r;
+ }
+ }
+
+ r = rbd_encryption_format(image, format, opts, opts_size);
+ if (r < 0) {
+ error_setg_errno(errp, -r, "encryption format fail");
+ return r;
+ }
+
+ r = rbd_get_size(image, &effective_size);
+ if (r < 0) {
+ error_setg_errno(errp, -r, "cannot get effective image size");
+ return r;
+ }
+
+ r = rbd_resize(image, raw_size + (raw_size - effective_size));
+ if (r < 0) {
+ error_setg_errno(errp, -r, "cannot resize image after format");
+ return r;
+ }
+
+ return 0;
+}
+
+static int qemu_rbd_encryption_load(rbd_image_t image,
+ RbdEncryptionOptions *encrypt,
+ Error **errp)
+{
+ int r = 0;
+ g_autofree char *passphrase = NULL;
+ size_t passphrase_len;
+ rbd_encryption_luks1_format_options_t luks_opts;
+ rbd_encryption_luks2_format_options_t luks2_opts;
+ rbd_encryption_format_t format;
+ rbd_encryption_options_t opts;
+ size_t opts_size;
+
+ switch (encrypt->format) {
+ case RBD_IMAGE_ENCRYPTION_FORMAT_LUKS: {
+ memset(&luks_opts, 0, sizeof(luks_opts));
+ format = RBD_ENCRYPTION_FORMAT_LUKS1;
+ opts = &luks_opts;
+ opts_size = sizeof(luks_opts);
+ r = qemu_rbd_convert_luks_options(
+ qapi_RbdEncryptionOptionsLUKS_base(&encrypt->u.luks),
+ &passphrase, &passphrase_len, errp);
+ if (r < 0) {
+ return r;
+ }
+ luks_opts.passphrase = passphrase;
+ luks_opts.passphrase_size = passphrase_len;
+ break;
+ }
+ case RBD_IMAGE_ENCRYPTION_FORMAT_LUKS2: {
+ memset(&luks2_opts, 0, sizeof(luks2_opts));
+ format = RBD_ENCRYPTION_FORMAT_LUKS2;
+ opts = &luks2_opts;
+ opts_size = sizeof(luks2_opts);
+ r = qemu_rbd_convert_luks_options(
+ qapi_RbdEncryptionOptionsLUKS2_base(&encrypt->u.luks2),
+ &passphrase, &passphrase_len, errp);
+ if (r < 0) {
+ return r;
+ }
+ luks2_opts.passphrase = passphrase;
+ luks2_opts.passphrase_size = passphrase_len;
+ break;
+ }
+ default: {
+ r = -ENOTSUP;
+ error_setg_errno(
+ errp, -r, "unknown image encryption format: %u",
+ encrypt->format);
+ return r;
+ }
+ }
+
+ r = rbd_encryption_load(image, format, opts, opts_size);
+ if (r < 0) {
+ error_setg_errno(errp, -r, "encryption load fail");
+ return r;
+ }
+
+ return 0;
+}
+#endif
+
/* FIXME Deprecate and remove keypairs or make it available in QMP. */
static int qemu_rbd_do_create(BlockdevCreateOptions *options,
const char *keypairs, const char *password_secret,
@@ -358,6 +567,13 @@ static int qemu_rbd_do_create(BlockdevCreateOptions *options,
return -EINVAL;
}
+#ifndef LIBRBD_SUPPORTS_ENCRYPTION
+ if (opts->has_encrypt) {
+ error_setg(errp, "RBD library does not support image encryption");
+ return -ENOTSUP;
+ }
+#endif
+
if (opts->has_cluster_size) {
int64_t objsize = opts->cluster_size;
if ((objsize - 1) & objsize) { /* not a power of 2? */
@@ -383,6 +599,28 @@ static int qemu_rbd_do_create(BlockdevCreateOptions *options,
goto out;
}
+#ifdef LIBRBD_SUPPORTS_ENCRYPTION
+ if (opts->has_encrypt) {
+ rbd_image_t image;
+
+ ret = rbd_open(io_ctx, opts->location->image, &image, NULL);
+ if (ret < 0) {
+ error_setg_errno(errp, -ret,
+ "error opening image '%s' for encryption format",
+ opts->location->image);
+ goto out;
+ }
+
+ ret = qemu_rbd_encryption_format(image, opts->encrypt, errp);
+ rbd_close(image);
+ if (ret < 0) {
+ /* encryption format fail, try removing the image */
+ rbd_remove(io_ctx, opts->location->image);
+ goto out;
+ }
+ }
+#endif
+
ret = 0;
out:
rados_ioctx_destroy(io_ctx);
@@ -395,6 +633,43 @@ static int qemu_rbd_co_create(BlockdevCreateOptions *options, Error **errp)
return qemu_rbd_do_create(options, NULL, NULL, errp);
}
+static int qemu_rbd_extract_encryption_create_options(
+ QemuOpts *opts,
+ RbdEncryptionCreateOptions **spec,
+ Error **errp)
+{
+ QDict *opts_qdict;
+ QDict *encrypt_qdict;
+ Visitor *v;
+ int ret = 0;
+
+ opts_qdict = qemu_opts_to_qdict(opts, NULL);
+ qdict_extract_subqdict(opts_qdict, &encrypt_qdict, "encrypt.");
+ qobject_unref(opts_qdict);
+ if (!qdict_size(encrypt_qdict)) {
+ *spec = NULL;
+ goto exit;
+ }
+
+ /* Convert options into a QAPI object */
+ v = qobject_input_visitor_new_flat_confused(encrypt_qdict, errp);
+ if (!v) {
+ ret = -EINVAL;
+ goto exit;
+ }
+
+ visit_type_RbdEncryptionCreateOptions(v, NULL, spec, errp);
+ visit_free(v);
+ if (!*spec) {
+ ret = -EINVAL;
+ goto exit;
+ }
+
+exit:
+ qobject_unref(encrypt_qdict);
+ return ret;
+}
+
static int coroutine_fn qemu_rbd_co_create_opts(BlockDriver *drv,
const char *filename,
QemuOpts *opts,
@@ -403,6 +678,7 @@ static int coroutine_fn qemu_rbd_co_create_opts(BlockDriver *drv,
BlockdevCreateOptions *create_options;
BlockdevCreateOptionsRbd *rbd_opts;
BlockdevOptionsRbd *loc;
+ RbdEncryptionCreateOptions *encrypt = NULL;
Error *local_err = NULL;
const char *keypairs, *password_secret;
QDict *options = NULL;
@@ -431,6 +707,13 @@ static int coroutine_fn qemu_rbd_co_create_opts(BlockDriver *drv,
goto exit;
}
+ ret = qemu_rbd_extract_encryption_create_options(opts, &encrypt, errp);
+ if (ret < 0) {
+ goto exit;
+ }
+ rbd_opts->encrypt = encrypt;
+ rbd_opts->has_encrypt = !!encrypt;
+
/*
* Caution: while qdict_get_try_str() is fine, getting non-string
* types would require more care. When @options come from -blockdev
@@ -756,12 +1039,24 @@ static int qemu_rbd_open(BlockDriverState *bs, QDict *options, int flags,
goto failed_open;
}
+ if (opts->has_encrypt) {
+#ifdef LIBRBD_SUPPORTS_ENCRYPTION
+ r = qemu_rbd_encryption_load(s->image, opts->encrypt, errp);
+ if (r < 0) {
+ goto failed_post_open;
+ }
+#else
+ r = -ENOTSUP;
+ error_setg(errp, "RBD library does not support image encryption");
+ goto failed_post_open;
+#endif
+ }
+
r = rbd_get_size(s->image, &s->image_size);
if (r < 0) {
error_setg_errno(errp, -r, "error getting image size from %s",
s->image_name);
- rbd_close(s->image);
- goto failed_open;
+ goto failed_post_open;
}
/* If we are using an rbd snapshot, we must be r/o, otherwise
@@ -769,8 +1064,7 @@ static int qemu_rbd_open(BlockDriverState *bs, QDict *options, int flags,
if (s->snap != NULL) {
r = bdrv_apply_auto_read_only(bs, "rbd snapshots are read-only", errp);
if (r < 0) {
- rbd_close(s->image);
- goto failed_open;
+ goto failed_post_open;
}
}
@@ -780,6 +1074,8 @@ static int qemu_rbd_open(BlockDriverState *bs, QDict *options, int flags,
r = 0;
goto out;
+failed_post_open:
+ rbd_close(s->image);
failed_open:
rados_ioctx_destroy(s->io_ctx);
g_free(s->snap);
@@ -1050,6 +1346,46 @@ static int qemu_rbd_getinfo(BlockDriverState *bs, BlockDriverInfo *bdi)
return 0;
}
+static ImageInfoSpecific *qemu_rbd_get_specific_info(BlockDriverState *bs,
+ Error **errp)
+{
+ BDRVRBDState *s = bs->opaque;
+ ImageInfoSpecific *spec_info;
+ char buf[RBD_ENCRYPTION_LUKS_HEADER_VERIFICATION_LEN] = {0};
+ int r;
+
+ if (s->image_size >= RBD_ENCRYPTION_LUKS_HEADER_VERIFICATION_LEN) {
+ r = rbd_read(s->image, 0,
+ RBD_ENCRYPTION_LUKS_HEADER_VERIFICATION_LEN, buf);
+ if (r < 0) {
+ error_setg_errno(errp, -r, "cannot read image start for probe");
+ return NULL;
+ }
+ }
+
+ spec_info = g_new(ImageInfoSpecific, 1);
+ *spec_info = (ImageInfoSpecific){
+ .type = IMAGE_INFO_SPECIFIC_KIND_RBD,
+ .u.rbd.data = g_new0(ImageInfoSpecificRbd, 1),
+ };
+
+ if (memcmp(buf, rbd_luks_header_verification,
+ RBD_ENCRYPTION_LUKS_HEADER_VERIFICATION_LEN) == 0) {
+ spec_info->u.rbd.data->encryption_format =
+ RBD_IMAGE_ENCRYPTION_FORMAT_LUKS;
+ spec_info->u.rbd.data->has_encryption_format = true;
+ } else if (memcmp(buf, rbd_luks2_header_verification,
+ RBD_ENCRYPTION_LUKS_HEADER_VERIFICATION_LEN) == 0) {
+ spec_info->u.rbd.data->encryption_format =
+ RBD_IMAGE_ENCRYPTION_FORMAT_LUKS2;
+ spec_info->u.rbd.data->has_encryption_format = true;
+ } else {
+ spec_info->u.rbd.data->has_encryption_format = false;
+ }
+
+ return spec_info;
+}
+
static int64_t qemu_rbd_getlength(BlockDriverState *bs)
{
BDRVRBDState *s = bs->opaque;
@@ -1243,6 +1579,22 @@ static QemuOptsList qemu_rbd_create_opts = {
.type = QEMU_OPT_STRING,
.help = "ID of secret providing the password",
},
+ {
+ .name = "encrypt.format",
+ .type = QEMU_OPT_STRING,
+ .help = "Encrypt the image, format choices: 'luks', 'luks2'",
+ },
+ {
+ .name = "encrypt.cipher-alg",
+ .type = QEMU_OPT_STRING,
+ .help = "Name of encryption cipher algorithm"
+ " (allowed values: aes-128, aes-256)",
+ },
+ {
+ .name = "encrypt.key-secret",
+ .type = QEMU_OPT_STRING,
+ .help = "ID of secret providing LUKS passphrase",
+ },
{ /* end of list */ }
}
};
@@ -1272,6 +1624,7 @@ static BlockDriver bdrv_rbd = {
.bdrv_co_create_opts = qemu_rbd_co_create_opts,
.bdrv_has_zero_init = bdrv_has_zero_init_1,
.bdrv_get_info = qemu_rbd_getinfo,
+ .bdrv_get_specific_info = qemu_rbd_get_specific_info,
.create_opts = &qemu_rbd_create_opts,
.bdrv_getlength = qemu_rbd_getlength,
.bdrv_co_truncate = qemu_rbd_co_truncate,
diff --git a/qapi/block-core.json b/qapi/block-core.json
index 6d227924d0..6cf67d796e 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -127,6 +127,18 @@
'extents': ['ImageInfo']
} }
+##
+# @ImageInfoSpecificRbd:
+#
+# @encryption-format: Image encryption format
+#
+# Since: 6.1
+##
+{ 'struct': 'ImageInfoSpecificRbd',
+ 'data': {
+ '*encryption-format': 'RbdImageEncryptionFormat'
+ } }
+
##
# @ImageInfoSpecific:
#
@@ -141,7 +153,8 @@
# If we need to add block driver specific parameters for
# LUKS in future, then we'll subclass QCryptoBlockInfoLUKS
# to define a ImageInfoSpecificLUKS
- 'luks': 'QCryptoBlockInfoLUKS'
+ 'luks': 'QCryptoBlockInfoLUKS',
+ 'rbd': 'ImageInfoSpecificRbd'
} }
##
@@ -3609,6 +3622,94 @@
{ 'enum': 'RbdAuthMode',
'data': [ 'cephx', 'none' ] }
+##
+# @RbdImageEncryptionFormat:
+#
+# Since: 6.1
+##
+{ 'enum': 'RbdImageEncryptionFormat',
+ 'data': [ 'luks', 'luks2' ] }
+
+##
+# @RbdEncryptionOptionsLUKSBase:
+#
+# @key-secret: ID of a QCryptoSecret object providing a passphrase
+# for unlocking the encryption
+#
+# Since: 6.1
+##
+{ 'struct': 'RbdEncryptionOptionsLUKSBase',
+ 'data': { 'key-secret': 'str' } }
+
+##
+# @RbdEncryptionCreateOptionsLUKSBase:
+#
+# @cipher-alg: The encryption algorithm
+#
+# Since: 6.1
+##
+{ 'struct': 'RbdEncryptionCreateOptionsLUKSBase',
+ 'base': 'RbdEncryptionOptionsLUKSBase',
+ 'data': { '*cipher-alg': 'QCryptoCipherAlgorithm' } }
+
+##
+# @RbdEncryptionOptionsLUKS:
+#
+# Since: 6.1
+##
+{ 'struct': 'RbdEncryptionOptionsLUKS',
+ 'base': 'RbdEncryptionOptionsLUKSBase',
+ 'data': { } }
+
+##
+# @RbdEncryptionOptionsLUKS2:
+#
+# Since: 6.1
+##
+{ 'struct': 'RbdEncryptionOptionsLUKS2',
+ 'base': 'RbdEncryptionOptionsLUKSBase',
+ 'data': { } }
+
+##
+# @RbdEncryptionCreateOptionsLUKS:
+#
+# Since: 6.1
+##
+{ 'struct': 'RbdEncryptionCreateOptionsLUKS',
+ 'base': 'RbdEncryptionCreateOptionsLUKSBase',
+ 'data': { } }
+
+##
+# @RbdEncryptionCreateOptionsLUKS2:
+#
+# Since: 6.1
+##
+{ 'struct': 'RbdEncryptionCreateOptionsLUKS2',
+ 'base': 'RbdEncryptionCreateOptionsLUKSBase',
+ 'data': { } }
+
+##
+# @RbdEncryptionOptions:
+#
+# Since: 6.1
+##
+{ 'union': 'RbdEncryptionOptions',
+ 'base': { 'format': 'RbdImageEncryptionFormat' },
+ 'discriminator': 'format',
+ 'data': { 'luks': 'RbdEncryptionOptionsLUKS',
+ 'luks2': 'RbdEncryptionOptionsLUKS2' } }
+
+##
+# @RbdEncryptionCreateOptions:
+#
+# Since: 6.1
+##
+{ 'union': 'RbdEncryptionCreateOptions',
+ 'base': { 'format': 'RbdImageEncryptionFormat' },
+ 'discriminator': 'format',
+ 'data': { 'luks': 'RbdEncryptionCreateOptionsLUKS',
+ 'luks2': 'RbdEncryptionCreateOptionsLUKS2' } }
+
##
# @BlockdevOptionsRbd:
#
@@ -3624,6 +3725,8 @@
#
# @snapshot: Ceph snapshot name.
#
+# @encrypt: Image encryption options. (Since 6.1)
+#
# @user: Ceph id name.
#
# @auth-client-required: Acceptable authentication modes.
@@ -3646,6 +3749,7 @@
'image': 'str',
'*conf': 'str',
'*snapshot': 'str',
+ '*encrypt': 'RbdEncryptionOptions',
'*user': 'str',
'*auth-client-required': ['RbdAuthMode'],
'*key-secret': 'str',
@@ -4418,13 +4522,15 @@
# point to a snapshot.
# @size: Size of the virtual disk in bytes
# @cluster-size: RBD object size
+# @encrypt: Image encryption options. (Since 6.1)
#
# Since: 2.12
##
{ 'struct': 'BlockdevCreateOptionsRbd',
'data': { 'location': 'BlockdevOptionsRbd',
'size': 'size',
- '*cluster-size' : 'size' } }
+ '*cluster-size' : 'size',
+ '*encrypt' : 'RbdEncryptionCreateOptions' } }
##
# @BlockdevVmdkSubformat:
--
2.27.0
^ permalink raw reply related [flat|nested] 3+ messages in thread
* Re: [PATCH v2] block/rbd: Add support for rbd image encryption
2021-06-27 11:46 [PATCH v2] block/rbd: Add support for rbd image encryption Or Ozeri
@ 2021-06-27 13:46 ` Ilya Dryomov
2021-07-02 11:37 ` Kevin Wolf
0 siblings, 1 reply; 3+ messages in thread
From: Ilya Dryomov @ 2021-06-27 13:46 UTC (permalink / raw)
To: Or Ozeri
Cc: kwolf, Daniel P. Berrangé,
qemu-block, qemu-devel, Mykola Golub, Danny Harnik
On Sun, Jun 27, 2021 at 1:46 PM Or Ozeri <oro@il.ibm.com> wrote:
>
> Starting from ceph Pacific, RBD has built-in support for image-level encryption.
> Currently supported formats are LUKS version 1 and 2.
>
> There are 2 new relevant librbd APIs for controlling encryption, both expect an
> open image context:
>
> rbd_encryption_format: formats an image (i.e. writes the LUKS header)
> rbd_encryption_load: loads encryptor/decryptor to the image IO stack
>
> This commit extends the qemu rbd driver API to support the above.
>
> Signed-off-by: Or Ozeri <oro@il.ibm.com>
> ---
> v2: handle encryption info only for the case where encryption is not loaded
> ---
> block/rbd.c | 361 ++++++++++++++++++++++++++++++++++++++++++-
> qapi/block-core.json | 110 ++++++++++++-
> 2 files changed, 465 insertions(+), 6 deletions(-)
>
> diff --git a/block/rbd.c b/block/rbd.c
> index f098a89c7b..8edd1be49e 100644
> --- a/block/rbd.c
> +++ b/block/rbd.c
> @@ -73,6 +73,18 @@
> #define LIBRBD_USE_IOVEC 0
> #endif
>
> +#define RBD_ENCRYPTION_LUKS_HEADER_VERIFICATION_LEN 8
> +
> +static const char rbd_luks_header_verification[
> + RBD_ENCRYPTION_LUKS_HEADER_VERIFICATION_LEN] = {
> + 'L', 'U', 'K', 'S', 0xBA, 0xBE, 0, 1
> +};
> +
> +static const char rbd_luks2_header_verification[
> + RBD_ENCRYPTION_LUKS_HEADER_VERIFICATION_LEN] = {
> + 'L', 'U', 'K', 'S', 0xBA, 0xBE, 0, 2
> +};
> +
> typedef enum {
> RBD_AIO_READ,
> RBD_AIO_WRITE,
> @@ -341,6 +353,203 @@ static void qemu_rbd_memset(RADOSCB *rcb, int64_t offs)
> }
> }
>
> +#ifdef LIBRBD_SUPPORTS_ENCRYPTION
> +static int qemu_rbd_convert_luks_options(
> + RbdEncryptionOptionsLUKSBase *luks_opts,
> + char **passphrase,
> + size_t *passphrase_len,
> + Error **errp)
> +{
> + return qcrypto_secret_lookup(luks_opts->key_secret, (uint8_t **)passphrase,
> + passphrase_len, errp);
> +}
> +
> +static int qemu_rbd_convert_luks_create_options(
> + RbdEncryptionCreateOptionsLUKSBase *luks_opts,
> + rbd_encryption_algorithm_t *alg,
> + char **passphrase,
> + size_t *passphrase_len,
> + Error **errp)
> +{
> + int r = 0;
> +
> + r = qemu_rbd_convert_luks_options(
> + qapi_RbdEncryptionCreateOptionsLUKSBase_base(luks_opts),
> + passphrase, passphrase_len, errp);
> + if (r < 0) {
> + return r;
> + }
> +
> + if (luks_opts->has_cipher_alg) {
> + switch (luks_opts->cipher_alg) {
> + case QCRYPTO_CIPHER_ALG_AES_128: {
> + *alg = RBD_ENCRYPTION_ALGORITHM_AES128;
> + break;
> + }
> + case QCRYPTO_CIPHER_ALG_AES_256: {
> + *alg = RBD_ENCRYPTION_ALGORITHM_AES256;
> + break;
> + }
> + default: {
> + r = -ENOTSUP;
> + error_setg_errno(errp, -r, "unknown encryption algorithm: %u",
> + luks_opts->cipher_alg);
> + return r;
> + }
> + }
> + } else {
> + /* default alg */
> + *alg = RBD_ENCRYPTION_ALGORITHM_AES256;
> + }
> +
> + return 0;
> +}
> +
> +static int qemu_rbd_encryption_format(rbd_image_t image,
> + RbdEncryptionCreateOptions *encrypt,
> + Error **errp)
> +{
> + int r = 0;
> + g_autofree char *passphrase = NULL;
> + size_t passphrase_len;
> + rbd_encryption_format_t format;
> + rbd_encryption_options_t opts;
> + rbd_encryption_luks1_format_options_t luks_opts;
> + rbd_encryption_luks2_format_options_t luks2_opts;
> + size_t opts_size;
> + uint64_t raw_size, effective_size;
> +
> + r = rbd_get_size(image, &raw_size);
> + if (r < 0) {
> + error_setg_errno(errp, -r, "cannot get raw image size");
> + return r;
> + }
> +
> + switch (encrypt->format) {
> + case RBD_IMAGE_ENCRYPTION_FORMAT_LUKS: {
> + memset(&luks_opts, 0, sizeof(luks_opts));
> + format = RBD_ENCRYPTION_FORMAT_LUKS1;
> + opts = &luks_opts;
> + opts_size = sizeof(luks_opts);
> + r = qemu_rbd_convert_luks_create_options(
> + qapi_RbdEncryptionCreateOptionsLUKS_base(&encrypt->u.luks),
> + &luks_opts.alg, &passphrase, &passphrase_len, errp);
> + if (r < 0) {
> + return r;
> + }
> + luks_opts.passphrase = passphrase;
> + luks_opts.passphrase_size = passphrase_len;
> + break;
> + }
> + case RBD_IMAGE_ENCRYPTION_FORMAT_LUKS2: {
> + memset(&luks2_opts, 0, sizeof(luks2_opts));
> + format = RBD_ENCRYPTION_FORMAT_LUKS2;
> + opts = &luks2_opts;
> + opts_size = sizeof(luks2_opts);
> + r = qemu_rbd_convert_luks_create_options(
> + qapi_RbdEncryptionCreateOptionsLUKS2_base(
> + &encrypt->u.luks2),
> + &luks2_opts.alg, &passphrase, &passphrase_len, errp);
> + if (r < 0) {
> + return r;
> + }
> + luks2_opts.passphrase = passphrase;
> + luks2_opts.passphrase_size = passphrase_len;
> + break;
> + }
> + default: {
> + r = -ENOTSUP;
> + error_setg_errno(
> + errp, -r, "unknown image encryption format: %u",
> + encrypt->format);
> + return r;
> + }
> + }
> +
> + r = rbd_encryption_format(image, format, opts, opts_size);
> + if (r < 0) {
> + error_setg_errno(errp, -r, "encryption format fail");
> + return r;
> + }
> +
> + r = rbd_get_size(image, &effective_size);
> + if (r < 0) {
> + error_setg_errno(errp, -r, "cannot get effective image size");
> + return r;
> + }
> +
> + r = rbd_resize(image, raw_size + (raw_size - effective_size));
> + if (r < 0) {
> + error_setg_errno(errp, -r, "cannot resize image after format");
> + return r;
> + }
> +
> + return 0;
> +}
> +
> +static int qemu_rbd_encryption_load(rbd_image_t image,
> + RbdEncryptionOptions *encrypt,
> + Error **errp)
> +{
> + int r = 0;
> + g_autofree char *passphrase = NULL;
> + size_t passphrase_len;
> + rbd_encryption_luks1_format_options_t luks_opts;
> + rbd_encryption_luks2_format_options_t luks2_opts;
> + rbd_encryption_format_t format;
> + rbd_encryption_options_t opts;
> + size_t opts_size;
> +
> + switch (encrypt->format) {
> + case RBD_IMAGE_ENCRYPTION_FORMAT_LUKS: {
> + memset(&luks_opts, 0, sizeof(luks_opts));
> + format = RBD_ENCRYPTION_FORMAT_LUKS1;
> + opts = &luks_opts;
> + opts_size = sizeof(luks_opts);
> + r = qemu_rbd_convert_luks_options(
> + qapi_RbdEncryptionOptionsLUKS_base(&encrypt->u.luks),
> + &passphrase, &passphrase_len, errp);
> + if (r < 0) {
> + return r;
> + }
> + luks_opts.passphrase = passphrase;
> + luks_opts.passphrase_size = passphrase_len;
> + break;
> + }
> + case RBD_IMAGE_ENCRYPTION_FORMAT_LUKS2: {
> + memset(&luks2_opts, 0, sizeof(luks2_opts));
> + format = RBD_ENCRYPTION_FORMAT_LUKS2;
> + opts = &luks2_opts;
> + opts_size = sizeof(luks2_opts);
> + r = qemu_rbd_convert_luks_options(
> + qapi_RbdEncryptionOptionsLUKS2_base(&encrypt->u.luks2),
> + &passphrase, &passphrase_len, errp);
> + if (r < 0) {
> + return r;
> + }
> + luks2_opts.passphrase = passphrase;
> + luks2_opts.passphrase_size = passphrase_len;
> + break;
> + }
> + default: {
> + r = -ENOTSUP;
> + error_setg_errno(
> + errp, -r, "unknown image encryption format: %u",
> + encrypt->format);
> + return r;
> + }
> + }
> +
> + r = rbd_encryption_load(image, format, opts, opts_size);
> + if (r < 0) {
> + error_setg_errno(errp, -r, "encryption load fail");
> + return r;
> + }
> +
> + return 0;
> +}
> +#endif
> +
> /* FIXME Deprecate and remove keypairs or make it available in QMP. */
> static int qemu_rbd_do_create(BlockdevCreateOptions *options,
> const char *keypairs, const char *password_secret,
> @@ -358,6 +567,13 @@ static int qemu_rbd_do_create(BlockdevCreateOptions *options,
> return -EINVAL;
> }
>
> +#ifndef LIBRBD_SUPPORTS_ENCRYPTION
> + if (opts->has_encrypt) {
> + error_setg(errp, "RBD library does not support image encryption");
> + return -ENOTSUP;
> + }
> +#endif
> +
> if (opts->has_cluster_size) {
> int64_t objsize = opts->cluster_size;
> if ((objsize - 1) & objsize) { /* not a power of 2? */
> @@ -383,6 +599,28 @@ static int qemu_rbd_do_create(BlockdevCreateOptions *options,
> goto out;
> }
>
> +#ifdef LIBRBD_SUPPORTS_ENCRYPTION
> + if (opts->has_encrypt) {
> + rbd_image_t image;
> +
> + ret = rbd_open(io_ctx, opts->location->image, &image, NULL);
> + if (ret < 0) {
> + error_setg_errno(errp, -ret,
> + "error opening image '%s' for encryption format",
> + opts->location->image);
> + goto out;
> + }
> +
> + ret = qemu_rbd_encryption_format(image, opts->encrypt, errp);
> + rbd_close(image);
> + if (ret < 0) {
> + /* encryption format fail, try removing the image */
> + rbd_remove(io_ctx, opts->location->image);
> + goto out;
> + }
> + }
> +#endif
> +
> ret = 0;
> out:
> rados_ioctx_destroy(io_ctx);
> @@ -395,6 +633,43 @@ static int qemu_rbd_co_create(BlockdevCreateOptions *options, Error **errp)
> return qemu_rbd_do_create(options, NULL, NULL, errp);
> }
>
> +static int qemu_rbd_extract_encryption_create_options(
> + QemuOpts *opts,
> + RbdEncryptionCreateOptions **spec,
> + Error **errp)
> +{
> + QDict *opts_qdict;
> + QDict *encrypt_qdict;
> + Visitor *v;
> + int ret = 0;
> +
> + opts_qdict = qemu_opts_to_qdict(opts, NULL);
> + qdict_extract_subqdict(opts_qdict, &encrypt_qdict, "encrypt.");
> + qobject_unref(opts_qdict);
> + if (!qdict_size(encrypt_qdict)) {
> + *spec = NULL;
> + goto exit;
> + }
> +
> + /* Convert options into a QAPI object */
> + v = qobject_input_visitor_new_flat_confused(encrypt_qdict, errp);
> + if (!v) {
> + ret = -EINVAL;
> + goto exit;
> + }
> +
> + visit_type_RbdEncryptionCreateOptions(v, NULL, spec, errp);
> + visit_free(v);
> + if (!*spec) {
> + ret = -EINVAL;
> + goto exit;
> + }
> +
> +exit:
> + qobject_unref(encrypt_qdict);
> + return ret;
> +}
> +
> static int coroutine_fn qemu_rbd_co_create_opts(BlockDriver *drv,
> const char *filename,
> QemuOpts *opts,
> @@ -403,6 +678,7 @@ static int coroutine_fn qemu_rbd_co_create_opts(BlockDriver *drv,
> BlockdevCreateOptions *create_options;
> BlockdevCreateOptionsRbd *rbd_opts;
> BlockdevOptionsRbd *loc;
> + RbdEncryptionCreateOptions *encrypt = NULL;
> Error *local_err = NULL;
> const char *keypairs, *password_secret;
> QDict *options = NULL;
> @@ -431,6 +707,13 @@ static int coroutine_fn qemu_rbd_co_create_opts(BlockDriver *drv,
> goto exit;
> }
>
> + ret = qemu_rbd_extract_encryption_create_options(opts, &encrypt, errp);
> + if (ret < 0) {
> + goto exit;
> + }
> + rbd_opts->encrypt = encrypt;
> + rbd_opts->has_encrypt = !!encrypt;
> +
> /*
> * Caution: while qdict_get_try_str() is fine, getting non-string
> * types would require more care. When @options come from -blockdev
> @@ -756,12 +1039,24 @@ static int qemu_rbd_open(BlockDriverState *bs, QDict *options, int flags,
> goto failed_open;
> }
>
> + if (opts->has_encrypt) {
> +#ifdef LIBRBD_SUPPORTS_ENCRYPTION
> + r = qemu_rbd_encryption_load(s->image, opts->encrypt, errp);
> + if (r < 0) {
> + goto failed_post_open;
> + }
> +#else
> + r = -ENOTSUP;
> + error_setg(errp, "RBD library does not support image encryption");
> + goto failed_post_open;
> +#endif
> + }
> +
> r = rbd_get_size(s->image, &s->image_size);
> if (r < 0) {
> error_setg_errno(errp, -r, "error getting image size from %s",
> s->image_name);
> - rbd_close(s->image);
> - goto failed_open;
> + goto failed_post_open;
> }
>
> /* If we are using an rbd snapshot, we must be r/o, otherwise
> @@ -769,8 +1064,7 @@ static int qemu_rbd_open(BlockDriverState *bs, QDict *options, int flags,
> if (s->snap != NULL) {
> r = bdrv_apply_auto_read_only(bs, "rbd snapshots are read-only", errp);
> if (r < 0) {
> - rbd_close(s->image);
> - goto failed_open;
> + goto failed_post_open;
> }
> }
>
> @@ -780,6 +1074,8 @@ static int qemu_rbd_open(BlockDriverState *bs, QDict *options, int flags,
> r = 0;
> goto out;
>
> +failed_post_open:
> + rbd_close(s->image);
> failed_open:
> rados_ioctx_destroy(s->io_ctx);
> g_free(s->snap);
> @@ -1050,6 +1346,46 @@ static int qemu_rbd_getinfo(BlockDriverState *bs, BlockDriverInfo *bdi)
> return 0;
> }
>
> +static ImageInfoSpecific *qemu_rbd_get_specific_info(BlockDriverState *bs,
> + Error **errp)
> +{
> + BDRVRBDState *s = bs->opaque;
> + ImageInfoSpecific *spec_info;
> + char buf[RBD_ENCRYPTION_LUKS_HEADER_VERIFICATION_LEN] = {0};
> + int r;
> +
> + if (s->image_size >= RBD_ENCRYPTION_LUKS_HEADER_VERIFICATION_LEN) {
> + r = rbd_read(s->image, 0,
> + RBD_ENCRYPTION_LUKS_HEADER_VERIFICATION_LEN, buf);
> + if (r < 0) {
> + error_setg_errno(errp, -r, "cannot read image start for probe");
> + return NULL;
> + }
> + }
> +
> + spec_info = g_new(ImageInfoSpecific, 1);
> + *spec_info = (ImageInfoSpecific){
> + .type = IMAGE_INFO_SPECIFIC_KIND_RBD,
> + .u.rbd.data = g_new0(ImageInfoSpecificRbd, 1),
> + };
> +
> + if (memcmp(buf, rbd_luks_header_verification,
> + RBD_ENCRYPTION_LUKS_HEADER_VERIFICATION_LEN) == 0) {
> + spec_info->u.rbd.data->encryption_format =
> + RBD_IMAGE_ENCRYPTION_FORMAT_LUKS;
> + spec_info->u.rbd.data->has_encryption_format = true;
> + } else if (memcmp(buf, rbd_luks2_header_verification,
> + RBD_ENCRYPTION_LUKS_HEADER_VERIFICATION_LEN) == 0) {
> + spec_info->u.rbd.data->encryption_format =
> + RBD_IMAGE_ENCRYPTION_FORMAT_LUKS2;
> + spec_info->u.rbd.data->has_encryption_format = true;
> + } else {
> + spec_info->u.rbd.data->has_encryption_format = false;
> + }
> +
> + return spec_info;
> +}
> +
> static int64_t qemu_rbd_getlength(BlockDriverState *bs)
> {
> BDRVRBDState *s = bs->opaque;
> @@ -1243,6 +1579,22 @@ static QemuOptsList qemu_rbd_create_opts = {
> .type = QEMU_OPT_STRING,
> .help = "ID of secret providing the password",
> },
> + {
> + .name = "encrypt.format",
> + .type = QEMU_OPT_STRING,
> + .help = "Encrypt the image, format choices: 'luks', 'luks2'",
> + },
> + {
> + .name = "encrypt.cipher-alg",
> + .type = QEMU_OPT_STRING,
> + .help = "Name of encryption cipher algorithm"
> + " (allowed values: aes-128, aes-256)",
> + },
> + {
> + .name = "encrypt.key-secret",
> + .type = QEMU_OPT_STRING,
> + .help = "ID of secret providing LUKS passphrase",
> + },
> { /* end of list */ }
> }
> };
> @@ -1272,6 +1624,7 @@ static BlockDriver bdrv_rbd = {
> .bdrv_co_create_opts = qemu_rbd_co_create_opts,
> .bdrv_has_zero_init = bdrv_has_zero_init_1,
> .bdrv_get_info = qemu_rbd_getinfo,
> + .bdrv_get_specific_info = qemu_rbd_get_specific_info,
> .create_opts = &qemu_rbd_create_opts,
> .bdrv_getlength = qemu_rbd_getlength,
> .bdrv_co_truncate = qemu_rbd_co_truncate,
> diff --git a/qapi/block-core.json b/qapi/block-core.json
> index 6d227924d0..6cf67d796e 100644
> --- a/qapi/block-core.json
> +++ b/qapi/block-core.json
> @@ -127,6 +127,18 @@
> 'extents': ['ImageInfo']
> } }
>
> +##
> +# @ImageInfoSpecificRbd:
> +#
> +# @encryption-format: Image encryption format
> +#
> +# Since: 6.1
> +##
> +{ 'struct': 'ImageInfoSpecificRbd',
> + 'data': {
> + '*encryption-format': 'RbdImageEncryptionFormat'
> + } }
> +
> ##
> # @ImageInfoSpecific:
> #
> @@ -141,7 +153,8 @@
> # If we need to add block driver specific parameters for
> # LUKS in future, then we'll subclass QCryptoBlockInfoLUKS
> # to define a ImageInfoSpecificLUKS
> - 'luks': 'QCryptoBlockInfoLUKS'
> + 'luks': 'QCryptoBlockInfoLUKS',
> + 'rbd': 'ImageInfoSpecificRbd'
> } }
>
> ##
> @@ -3609,6 +3622,94 @@
> { 'enum': 'RbdAuthMode',
> 'data': [ 'cephx', 'none' ] }
>
> +##
> +# @RbdImageEncryptionFormat:
> +#
> +# Since: 6.1
> +##
> +{ 'enum': 'RbdImageEncryptionFormat',
> + 'data': [ 'luks', 'luks2' ] }
> +
> +##
> +# @RbdEncryptionOptionsLUKSBase:
> +#
> +# @key-secret: ID of a QCryptoSecret object providing a passphrase
> +# for unlocking the encryption
> +#
> +# Since: 6.1
> +##
> +{ 'struct': 'RbdEncryptionOptionsLUKSBase',
> + 'data': { 'key-secret': 'str' } }
> +
> +##
> +# @RbdEncryptionCreateOptionsLUKSBase:
> +#
> +# @cipher-alg: The encryption algorithm
> +#
> +# Since: 6.1
> +##
> +{ 'struct': 'RbdEncryptionCreateOptionsLUKSBase',
> + 'base': 'RbdEncryptionOptionsLUKSBase',
> + 'data': { '*cipher-alg': 'QCryptoCipherAlgorithm' } }
> +
> +##
> +# @RbdEncryptionOptionsLUKS:
> +#
> +# Since: 6.1
> +##
> +{ 'struct': 'RbdEncryptionOptionsLUKS',
> + 'base': 'RbdEncryptionOptionsLUKSBase',
> + 'data': { } }
> +
> +##
> +# @RbdEncryptionOptionsLUKS2:
> +#
> +# Since: 6.1
> +##
> +{ 'struct': 'RbdEncryptionOptionsLUKS2',
> + 'base': 'RbdEncryptionOptionsLUKSBase',
> + 'data': { } }
> +
> +##
> +# @RbdEncryptionCreateOptionsLUKS:
> +#
> +# Since: 6.1
> +##
> +{ 'struct': 'RbdEncryptionCreateOptionsLUKS',
> + 'base': 'RbdEncryptionCreateOptionsLUKSBase',
> + 'data': { } }
> +
> +##
> +# @RbdEncryptionCreateOptionsLUKS2:
> +#
> +# Since: 6.1
> +##
> +{ 'struct': 'RbdEncryptionCreateOptionsLUKS2',
> + 'base': 'RbdEncryptionCreateOptionsLUKSBase',
> + 'data': { } }
> +
> +##
> +# @RbdEncryptionOptions:
> +#
> +# Since: 6.1
> +##
> +{ 'union': 'RbdEncryptionOptions',
> + 'base': { 'format': 'RbdImageEncryptionFormat' },
> + 'discriminator': 'format',
> + 'data': { 'luks': 'RbdEncryptionOptionsLUKS',
> + 'luks2': 'RbdEncryptionOptionsLUKS2' } }
> +
> +##
> +# @RbdEncryptionCreateOptions:
> +#
> +# Since: 6.1
> +##
> +{ 'union': 'RbdEncryptionCreateOptions',
> + 'base': { 'format': 'RbdImageEncryptionFormat' },
> + 'discriminator': 'format',
> + 'data': { 'luks': 'RbdEncryptionCreateOptionsLUKS',
> + 'luks2': 'RbdEncryptionCreateOptionsLUKS2' } }
> +
> ##
> # @BlockdevOptionsRbd:
> #
> @@ -3624,6 +3725,8 @@
> #
> # @snapshot: Ceph snapshot name.
> #
> +# @encrypt: Image encryption options. (Since 6.1)
> +#
> # @user: Ceph id name.
> #
> # @auth-client-required: Acceptable authentication modes.
> @@ -3646,6 +3749,7 @@
> 'image': 'str',
> '*conf': 'str',
> '*snapshot': 'str',
> + '*encrypt': 'RbdEncryptionOptions',
> '*user': 'str',
> '*auth-client-required': ['RbdAuthMode'],
> '*key-secret': 'str',
> @@ -4418,13 +4522,15 @@
> # point to a snapshot.
> # @size: Size of the virtual disk in bytes
> # @cluster-size: RBD object size
> +# @encrypt: Image encryption options. (Since 6.1)
> #
> # Since: 2.12
> ##
> { 'struct': 'BlockdevCreateOptionsRbd',
> 'data': { 'location': 'BlockdevOptionsRbd',
> 'size': 'size',
> - '*cluster-size' : 'size' } }
> + '*cluster-size' : 'size',
> + '*encrypt' : 'RbdEncryptionCreateOptions' } }
>
> ##
> # @BlockdevVmdkSubformat:
> --
> 2.27.0
>
Reviewed-by: Ilya Dryomov <idryomov@gmail.com>
Thanks,
Ilya
^ permalink raw reply [flat|nested] 3+ messages in thread
* Re: [PATCH v2] block/rbd: Add support for rbd image encryption
2021-06-27 13:46 ` Ilya Dryomov
@ 2021-07-02 11:37 ` Kevin Wolf
0 siblings, 0 replies; 3+ messages in thread
From: Kevin Wolf @ 2021-07-02 11:37 UTC (permalink / raw)
To: Ilya Dryomov
Cc: Daniel P. Berrangé,
qemu-block, qemu-devel, Mykola Golub, Or Ozeri, Danny Harnik
Am 27.06.2021 um 15:46 hat Ilya Dryomov geschrieben:
> On Sun, Jun 27, 2021 at 1:46 PM Or Ozeri <oro@il.ibm.com> wrote:
> >
> > Starting from ceph Pacific, RBD has built-in support for image-level encryption.
> > Currently supported formats are LUKS version 1 and 2.
> >
> > There are 2 new relevant librbd APIs for controlling encryption, both expect an
> > open image context:
> >
> > rbd_encryption_format: formats an image (i.e. writes the LUKS header)
> > rbd_encryption_load: loads encryptor/decryptor to the image IO stack
> >
> > This commit extends the qemu rbd driver API to support the above.
> >
> > Signed-off-by: Or Ozeri <oro@il.ibm.com>
> > ---
> > v2: handle encryption info only for the case where encryption is not loaded
> > ---
> > block/rbd.c | 361 ++++++++++++++++++++++++++++++++++++++++++-
> > qapi/block-core.json | 110 ++++++++++++-
> > 2 files changed, 465 insertions(+), 6 deletions(-)
> >
> > diff --git a/block/rbd.c b/block/rbd.c
> > index f098a89c7b..8edd1be49e 100644
> > --- a/block/rbd.c
> > +++ b/block/rbd.c
> > @@ -73,6 +73,18 @@
> > #define LIBRBD_USE_IOVEC 0
> > #endif
> >
> > +#define RBD_ENCRYPTION_LUKS_HEADER_VERIFICATION_LEN 8
> > +
> > +static const char rbd_luks_header_verification[
> > + RBD_ENCRYPTION_LUKS_HEADER_VERIFICATION_LEN] = {
> > + 'L', 'U', 'K', 'S', 0xBA, 0xBE, 0, 1
> > +};
> > +
> > +static const char rbd_luks2_header_verification[
> > + RBD_ENCRYPTION_LUKS_HEADER_VERIFICATION_LEN] = {
> > + 'L', 'U', 'K', 'S', 0xBA, 0xBE, 0, 2
> > +};
> > +
> > typedef enum {
> > RBD_AIO_READ,
> > RBD_AIO_WRITE,
> > @@ -341,6 +353,203 @@ static void qemu_rbd_memset(RADOSCB *rcb, int64_t offs)
> > }
> > }
> >
> > +#ifdef LIBRBD_SUPPORTS_ENCRYPTION
> > +static int qemu_rbd_convert_luks_options(
> > + RbdEncryptionOptionsLUKSBase *luks_opts,
> > + char **passphrase,
> > + size_t *passphrase_len,
> > + Error **errp)
> > +{
> > + return qcrypto_secret_lookup(luks_opts->key_secret, (uint8_t **)passphrase,
> > + passphrase_len, errp);
> > +}
> > +
> > +static int qemu_rbd_convert_luks_create_options(
> > + RbdEncryptionCreateOptionsLUKSBase *luks_opts,
> > + rbd_encryption_algorithm_t *alg,
> > + char **passphrase,
> > + size_t *passphrase_len,
> > + Error **errp)
> > +{
> > + int r = 0;
> > +
> > + r = qemu_rbd_convert_luks_options(
> > + qapi_RbdEncryptionCreateOptionsLUKSBase_base(luks_opts),
> > + passphrase, passphrase_len, errp);
> > + if (r < 0) {
> > + return r;
> > + }
> > +
> > + if (luks_opts->has_cipher_alg) {
> > + switch (luks_opts->cipher_alg) {
> > + case QCRYPTO_CIPHER_ALG_AES_128: {
> > + *alg = RBD_ENCRYPTION_ALGORITHM_AES128;
> > + break;
> > + }
> > + case QCRYPTO_CIPHER_ALG_AES_256: {
> > + *alg = RBD_ENCRYPTION_ALGORITHM_AES256;
> > + break;
> > + }
> > + default: {
> > + r = -ENOTSUP;
> > + error_setg_errno(errp, -r, "unknown encryption algorithm: %u",
> > + luks_opts->cipher_alg);
> > + return r;
> > + }
> > + }
> > + } else {
> > + /* default alg */
> > + *alg = RBD_ENCRYPTION_ALGORITHM_AES256;
> > + }
> > +
> > + return 0;
> > +}
> > +
> > +static int qemu_rbd_encryption_format(rbd_image_t image,
> > + RbdEncryptionCreateOptions *encrypt,
> > + Error **errp)
> > +{
> > + int r = 0;
> > + g_autofree char *passphrase = NULL;
> > + size_t passphrase_len;
> > + rbd_encryption_format_t format;
> > + rbd_encryption_options_t opts;
> > + rbd_encryption_luks1_format_options_t luks_opts;
> > + rbd_encryption_luks2_format_options_t luks2_opts;
> > + size_t opts_size;
> > + uint64_t raw_size, effective_size;
> > +
> > + r = rbd_get_size(image, &raw_size);
> > + if (r < 0) {
> > + error_setg_errno(errp, -r, "cannot get raw image size");
> > + return r;
> > + }
> > +
> > + switch (encrypt->format) {
> > + case RBD_IMAGE_ENCRYPTION_FORMAT_LUKS: {
> > + memset(&luks_opts, 0, sizeof(luks_opts));
> > + format = RBD_ENCRYPTION_FORMAT_LUKS1;
> > + opts = &luks_opts;
> > + opts_size = sizeof(luks_opts);
> > + r = qemu_rbd_convert_luks_create_options(
> > + qapi_RbdEncryptionCreateOptionsLUKS_base(&encrypt->u.luks),
> > + &luks_opts.alg, &passphrase, &passphrase_len, errp);
> > + if (r < 0) {
> > + return r;
> > + }
> > + luks_opts.passphrase = passphrase;
> > + luks_opts.passphrase_size = passphrase_len;
> > + break;
> > + }
> > + case RBD_IMAGE_ENCRYPTION_FORMAT_LUKS2: {
> > + memset(&luks2_opts, 0, sizeof(luks2_opts));
> > + format = RBD_ENCRYPTION_FORMAT_LUKS2;
> > + opts = &luks2_opts;
> > + opts_size = sizeof(luks2_opts);
> > + r = qemu_rbd_convert_luks_create_options(
> > + qapi_RbdEncryptionCreateOptionsLUKS2_base(
> > + &encrypt->u.luks2),
> > + &luks2_opts.alg, &passphrase, &passphrase_len, errp);
> > + if (r < 0) {
> > + return r;
> > + }
> > + luks2_opts.passphrase = passphrase;
> > + luks2_opts.passphrase_size = passphrase_len;
> > + break;
> > + }
> > + default: {
> > + r = -ENOTSUP;
> > + error_setg_errno(
> > + errp, -r, "unknown image encryption format: %u",
> > + encrypt->format);
> > + return r;
> > + }
> > + }
> > +
> > + r = rbd_encryption_format(image, format, opts, opts_size);
> > + if (r < 0) {
> > + error_setg_errno(errp, -r, "encryption format fail");
> > + return r;
> > + }
> > +
> > + r = rbd_get_size(image, &effective_size);
> > + if (r < 0) {
> > + error_setg_errno(errp, -r, "cannot get effective image size");
> > + return r;
> > + }
> > +
> > + r = rbd_resize(image, raw_size + (raw_size - effective_size));
> > + if (r < 0) {
> > + error_setg_errno(errp, -r, "cannot resize image after format");
> > + return r;
> > + }
> > +
> > + return 0;
> > +}
> > +
> > +static int qemu_rbd_encryption_load(rbd_image_t image,
> > + RbdEncryptionOptions *encrypt,
> > + Error **errp)
> > +{
> > + int r = 0;
> > + g_autofree char *passphrase = NULL;
> > + size_t passphrase_len;
> > + rbd_encryption_luks1_format_options_t luks_opts;
> > + rbd_encryption_luks2_format_options_t luks2_opts;
> > + rbd_encryption_format_t format;
> > + rbd_encryption_options_t opts;
> > + size_t opts_size;
> > +
> > + switch (encrypt->format) {
> > + case RBD_IMAGE_ENCRYPTION_FORMAT_LUKS: {
> > + memset(&luks_opts, 0, sizeof(luks_opts));
> > + format = RBD_ENCRYPTION_FORMAT_LUKS1;
> > + opts = &luks_opts;
> > + opts_size = sizeof(luks_opts);
> > + r = qemu_rbd_convert_luks_options(
> > + qapi_RbdEncryptionOptionsLUKS_base(&encrypt->u.luks),
> > + &passphrase, &passphrase_len, errp);
> > + if (r < 0) {
> > + return r;
> > + }
> > + luks_opts.passphrase = passphrase;
> > + luks_opts.passphrase_size = passphrase_len;
> > + break;
> > + }
> > + case RBD_IMAGE_ENCRYPTION_FORMAT_LUKS2: {
> > + memset(&luks2_opts, 0, sizeof(luks2_opts));
> > + format = RBD_ENCRYPTION_FORMAT_LUKS2;
> > + opts = &luks2_opts;
> > + opts_size = sizeof(luks2_opts);
> > + r = qemu_rbd_convert_luks_options(
> > + qapi_RbdEncryptionOptionsLUKS2_base(&encrypt->u.luks2),
> > + &passphrase, &passphrase_len, errp);
> > + if (r < 0) {
> > + return r;
> > + }
> > + luks2_opts.passphrase = passphrase;
> > + luks2_opts.passphrase_size = passphrase_len;
> > + break;
> > + }
> > + default: {
> > + r = -ENOTSUP;
> > + error_setg_errno(
> > + errp, -r, "unknown image encryption format: %u",
> > + encrypt->format);
> > + return r;
> > + }
> > + }
> > +
> > + r = rbd_encryption_load(image, format, opts, opts_size);
> > + if (r < 0) {
> > + error_setg_errno(errp, -r, "encryption load fail");
> > + return r;
> > + }
> > +
> > + return 0;
> > +}
> > +#endif
> > +
> > /* FIXME Deprecate and remove keypairs or make it available in QMP. */
> > static int qemu_rbd_do_create(BlockdevCreateOptions *options,
> > const char *keypairs, const char *password_secret,
> > @@ -358,6 +567,13 @@ static int qemu_rbd_do_create(BlockdevCreateOptions *options,
> > return -EINVAL;
> > }
> >
> > +#ifndef LIBRBD_SUPPORTS_ENCRYPTION
> > + if (opts->has_encrypt) {
> > + error_setg(errp, "RBD library does not support image encryption");
> > + return -ENOTSUP;
> > + }
> > +#endif
> > +
> > if (opts->has_cluster_size) {
> > int64_t objsize = opts->cluster_size;
> > if ((objsize - 1) & objsize) { /* not a power of 2? */
> > @@ -383,6 +599,28 @@ static int qemu_rbd_do_create(BlockdevCreateOptions *options,
> > goto out;
> > }
> >
> > +#ifdef LIBRBD_SUPPORTS_ENCRYPTION
> > + if (opts->has_encrypt) {
> > + rbd_image_t image;
> > +
> > + ret = rbd_open(io_ctx, opts->location->image, &image, NULL);
> > + if (ret < 0) {
> > + error_setg_errno(errp, -ret,
> > + "error opening image '%s' for encryption format",
> > + opts->location->image);
> > + goto out;
> > + }
> > +
> > + ret = qemu_rbd_encryption_format(image, opts->encrypt, errp);
> > + rbd_close(image);
> > + if (ret < 0) {
> > + /* encryption format fail, try removing the image */
> > + rbd_remove(io_ctx, opts->location->image);
> > + goto out;
> > + }
> > + }
> > +#endif
> > +
> > ret = 0;
> > out:
> > rados_ioctx_destroy(io_ctx);
> > @@ -395,6 +633,43 @@ static int qemu_rbd_co_create(BlockdevCreateOptions *options, Error **errp)
> > return qemu_rbd_do_create(options, NULL, NULL, errp);
> > }
> >
> > +static int qemu_rbd_extract_encryption_create_options(
> > + QemuOpts *opts,
> > + RbdEncryptionCreateOptions **spec,
> > + Error **errp)
> > +{
> > + QDict *opts_qdict;
> > + QDict *encrypt_qdict;
> > + Visitor *v;
> > + int ret = 0;
> > +
> > + opts_qdict = qemu_opts_to_qdict(opts, NULL);
> > + qdict_extract_subqdict(opts_qdict, &encrypt_qdict, "encrypt.");
> > + qobject_unref(opts_qdict);
> > + if (!qdict_size(encrypt_qdict)) {
> > + *spec = NULL;
> > + goto exit;
> > + }
> > +
> > + /* Convert options into a QAPI object */
> > + v = qobject_input_visitor_new_flat_confused(encrypt_qdict, errp);
> > + if (!v) {
> > + ret = -EINVAL;
> > + goto exit;
> > + }
> > +
> > + visit_type_RbdEncryptionCreateOptions(v, NULL, spec, errp);
> > + visit_free(v);
> > + if (!*spec) {
> > + ret = -EINVAL;
> > + goto exit;
> > + }
> > +
> > +exit:
> > + qobject_unref(encrypt_qdict);
> > + return ret;
> > +}
> > +
> > static int coroutine_fn qemu_rbd_co_create_opts(BlockDriver *drv,
> > const char *filename,
> > QemuOpts *opts,
> > @@ -403,6 +678,7 @@ static int coroutine_fn qemu_rbd_co_create_opts(BlockDriver *drv,
> > BlockdevCreateOptions *create_options;
> > BlockdevCreateOptionsRbd *rbd_opts;
> > BlockdevOptionsRbd *loc;
> > + RbdEncryptionCreateOptions *encrypt = NULL;
> > Error *local_err = NULL;
> > const char *keypairs, *password_secret;
> > QDict *options = NULL;
> > @@ -431,6 +707,13 @@ static int coroutine_fn qemu_rbd_co_create_opts(BlockDriver *drv,
> > goto exit;
> > }
> >
> > + ret = qemu_rbd_extract_encryption_create_options(opts, &encrypt, errp);
> > + if (ret < 0) {
> > + goto exit;
> > + }
> > + rbd_opts->encrypt = encrypt;
> > + rbd_opts->has_encrypt = !!encrypt;
> > +
> > /*
> > * Caution: while qdict_get_try_str() is fine, getting non-string
> > * types would require more care. When @options come from -blockdev
> > @@ -756,12 +1039,24 @@ static int qemu_rbd_open(BlockDriverState *bs, QDict *options, int flags,
> > goto failed_open;
> > }
> >
> > + if (opts->has_encrypt) {
> > +#ifdef LIBRBD_SUPPORTS_ENCRYPTION
> > + r = qemu_rbd_encryption_load(s->image, opts->encrypt, errp);
> > + if (r < 0) {
> > + goto failed_post_open;
> > + }
> > +#else
> > + r = -ENOTSUP;
> > + error_setg(errp, "RBD library does not support image encryption");
> > + goto failed_post_open;
> > +#endif
> > + }
> > +
> > r = rbd_get_size(s->image, &s->image_size);
> > if (r < 0) {
> > error_setg_errno(errp, -r, "error getting image size from %s",
> > s->image_name);
> > - rbd_close(s->image);
> > - goto failed_open;
> > + goto failed_post_open;
> > }
> >
> > /* If we are using an rbd snapshot, we must be r/o, otherwise
> > @@ -769,8 +1064,7 @@ static int qemu_rbd_open(BlockDriverState *bs, QDict *options, int flags,
> > if (s->snap != NULL) {
> > r = bdrv_apply_auto_read_only(bs, "rbd snapshots are read-only", errp);
> > if (r < 0) {
> > - rbd_close(s->image);
> > - goto failed_open;
> > + goto failed_post_open;
> > }
> > }
> >
> > @@ -780,6 +1074,8 @@ static int qemu_rbd_open(BlockDriverState *bs, QDict *options, int flags,
> > r = 0;
> > goto out;
> >
> > +failed_post_open:
> > + rbd_close(s->image);
> > failed_open:
> > rados_ioctx_destroy(s->io_ctx);
> > g_free(s->snap);
> > @@ -1050,6 +1346,46 @@ static int qemu_rbd_getinfo(BlockDriverState *bs, BlockDriverInfo *bdi)
> > return 0;
> > }
> >
> > +static ImageInfoSpecific *qemu_rbd_get_specific_info(BlockDriverState *bs,
> > + Error **errp)
> > +{
> > + BDRVRBDState *s = bs->opaque;
> > + ImageInfoSpecific *spec_info;
> > + char buf[RBD_ENCRYPTION_LUKS_HEADER_VERIFICATION_LEN] = {0};
> > + int r;
> > +
> > + if (s->image_size >= RBD_ENCRYPTION_LUKS_HEADER_VERIFICATION_LEN) {
> > + r = rbd_read(s->image, 0,
> > + RBD_ENCRYPTION_LUKS_HEADER_VERIFICATION_LEN, buf);
> > + if (r < 0) {
> > + error_setg_errno(errp, -r, "cannot read image start for probe");
> > + return NULL;
> > + }
> > + }
> > +
> > + spec_info = g_new(ImageInfoSpecific, 1);
> > + *spec_info = (ImageInfoSpecific){
> > + .type = IMAGE_INFO_SPECIFIC_KIND_RBD,
> > + .u.rbd.data = g_new0(ImageInfoSpecificRbd, 1),
> > + };
> > +
> > + if (memcmp(buf, rbd_luks_header_verification,
> > + RBD_ENCRYPTION_LUKS_HEADER_VERIFICATION_LEN) == 0) {
> > + spec_info->u.rbd.data->encryption_format =
> > + RBD_IMAGE_ENCRYPTION_FORMAT_LUKS;
> > + spec_info->u.rbd.data->has_encryption_format = true;
> > + } else if (memcmp(buf, rbd_luks2_header_verification,
> > + RBD_ENCRYPTION_LUKS_HEADER_VERIFICATION_LEN) == 0) {
> > + spec_info->u.rbd.data->encryption_format =
> > + RBD_IMAGE_ENCRYPTION_FORMAT_LUKS2;
> > + spec_info->u.rbd.data->has_encryption_format = true;
> > + } else {
> > + spec_info->u.rbd.data->has_encryption_format = false;
> > + }
> > +
> > + return spec_info;
> > +}
> > +
> > static int64_t qemu_rbd_getlength(BlockDriverState *bs)
> > {
> > BDRVRBDState *s = bs->opaque;
> > @@ -1243,6 +1579,22 @@ static QemuOptsList qemu_rbd_create_opts = {
> > .type = QEMU_OPT_STRING,
> > .help = "ID of secret providing the password",
> > },
> > + {
> > + .name = "encrypt.format",
> > + .type = QEMU_OPT_STRING,
> > + .help = "Encrypt the image, format choices: 'luks', 'luks2'",
> > + },
> > + {
> > + .name = "encrypt.cipher-alg",
> > + .type = QEMU_OPT_STRING,
> > + .help = "Name of encryption cipher algorithm"
> > + " (allowed values: aes-128, aes-256)",
> > + },
> > + {
> > + .name = "encrypt.key-secret",
> > + .type = QEMU_OPT_STRING,
> > + .help = "ID of secret providing LUKS passphrase",
> > + },
> > { /* end of list */ }
> > }
> > };
> > @@ -1272,6 +1624,7 @@ static BlockDriver bdrv_rbd = {
> > .bdrv_co_create_opts = qemu_rbd_co_create_opts,
> > .bdrv_has_zero_init = bdrv_has_zero_init_1,
> > .bdrv_get_info = qemu_rbd_getinfo,
> > + .bdrv_get_specific_info = qemu_rbd_get_specific_info,
> > .create_opts = &qemu_rbd_create_opts,
> > .bdrv_getlength = qemu_rbd_getlength,
> > .bdrv_co_truncate = qemu_rbd_co_truncate,
> > diff --git a/qapi/block-core.json b/qapi/block-core.json
> > index 6d227924d0..6cf67d796e 100644
> > --- a/qapi/block-core.json
> > +++ b/qapi/block-core.json
> > @@ -127,6 +127,18 @@
> > 'extents': ['ImageInfo']
> > } }
> >
> > +##
> > +# @ImageInfoSpecificRbd:
> > +#
> > +# @encryption-format: Image encryption format
> > +#
> > +# Since: 6.1
> > +##
> > +{ 'struct': 'ImageInfoSpecificRbd',
> > + 'data': {
> > + '*encryption-format': 'RbdImageEncryptionFormat'
> > + } }
> > +
> > ##
> > # @ImageInfoSpecific:
> > #
> > @@ -141,7 +153,8 @@
> > # If we need to add block driver specific parameters for
> > # LUKS in future, then we'll subclass QCryptoBlockInfoLUKS
> > # to define a ImageInfoSpecificLUKS
> > - 'luks': 'QCryptoBlockInfoLUKS'
> > + 'luks': 'QCryptoBlockInfoLUKS',
> > + 'rbd': 'ImageInfoSpecificRbd'
> > } }
> >
> > ##
> > @@ -3609,6 +3622,94 @@
> > { 'enum': 'RbdAuthMode',
> > 'data': [ 'cephx', 'none' ] }
> >
> > +##
> > +# @RbdImageEncryptionFormat:
> > +#
> > +# Since: 6.1
> > +##
> > +{ 'enum': 'RbdImageEncryptionFormat',
> > + 'data': [ 'luks', 'luks2' ] }
> > +
> > +##
> > +# @RbdEncryptionOptionsLUKSBase:
> > +#
> > +# @key-secret: ID of a QCryptoSecret object providing a passphrase
> > +# for unlocking the encryption
> > +#
> > +# Since: 6.1
> > +##
> > +{ 'struct': 'RbdEncryptionOptionsLUKSBase',
> > + 'data': { 'key-secret': 'str' } }
> > +
> > +##
> > +# @RbdEncryptionCreateOptionsLUKSBase:
> > +#
> > +# @cipher-alg: The encryption algorithm
> > +#
> > +# Since: 6.1
> > +##
> > +{ 'struct': 'RbdEncryptionCreateOptionsLUKSBase',
> > + 'base': 'RbdEncryptionOptionsLUKSBase',
> > + 'data': { '*cipher-alg': 'QCryptoCipherAlgorithm' } }
> > +
> > +##
> > +# @RbdEncryptionOptionsLUKS:
> > +#
> > +# Since: 6.1
> > +##
> > +{ 'struct': 'RbdEncryptionOptionsLUKS',
> > + 'base': 'RbdEncryptionOptionsLUKSBase',
> > + 'data': { } }
> > +
> > +##
> > +# @RbdEncryptionOptionsLUKS2:
> > +#
> > +# Since: 6.1
> > +##
> > +{ 'struct': 'RbdEncryptionOptionsLUKS2',
> > + 'base': 'RbdEncryptionOptionsLUKSBase',
> > + 'data': { } }
> > +
> > +##
> > +# @RbdEncryptionCreateOptionsLUKS:
> > +#
> > +# Since: 6.1
> > +##
> > +{ 'struct': 'RbdEncryptionCreateOptionsLUKS',
> > + 'base': 'RbdEncryptionCreateOptionsLUKSBase',
> > + 'data': { } }
> > +
> > +##
> > +# @RbdEncryptionCreateOptionsLUKS2:
> > +#
> > +# Since: 6.1
> > +##
> > +{ 'struct': 'RbdEncryptionCreateOptionsLUKS2',
> > + 'base': 'RbdEncryptionCreateOptionsLUKSBase',
> > + 'data': { } }
> > +
> > +##
> > +# @RbdEncryptionOptions:
> > +#
> > +# Since: 6.1
> > +##
> > +{ 'union': 'RbdEncryptionOptions',
> > + 'base': { 'format': 'RbdImageEncryptionFormat' },
> > + 'discriminator': 'format',
> > + 'data': { 'luks': 'RbdEncryptionOptionsLUKS',
> > + 'luks2': 'RbdEncryptionOptionsLUKS2' } }
> > +
> > +##
> > +# @RbdEncryptionCreateOptions:
> > +#
> > +# Since: 6.1
> > +##
> > +{ 'union': 'RbdEncryptionCreateOptions',
> > + 'base': { 'format': 'RbdImageEncryptionFormat' },
> > + 'discriminator': 'format',
> > + 'data': { 'luks': 'RbdEncryptionCreateOptionsLUKS',
> > + 'luks2': 'RbdEncryptionCreateOptionsLUKS2' } }
> > +
> > ##
> > # @BlockdevOptionsRbd:
> > #
> > @@ -3624,6 +3725,8 @@
> > #
> > # @snapshot: Ceph snapshot name.
> > #
> > +# @encrypt: Image encryption options. (Since 6.1)
> > +#
> > # @user: Ceph id name.
> > #
> > # @auth-client-required: Acceptable authentication modes.
> > @@ -3646,6 +3749,7 @@
> > 'image': 'str',
> > '*conf': 'str',
> > '*snapshot': 'str',
> > + '*encrypt': 'RbdEncryptionOptions',
> > '*user': 'str',
> > '*auth-client-required': ['RbdAuthMode'],
> > '*key-secret': 'str',
> > @@ -4418,13 +4522,15 @@
> > # point to a snapshot.
> > # @size: Size of the virtual disk in bytes
> > # @cluster-size: RBD object size
> > +# @encrypt: Image encryption options. (Since 6.1)
> > #
> > # Since: 2.12
> > ##
> > { 'struct': 'BlockdevCreateOptionsRbd',
> > 'data': { 'location': 'BlockdevOptionsRbd',
> > 'size': 'size',
> > - '*cluster-size' : 'size' } }
> > + '*cluster-size' : 'size',
> > + '*encrypt' : 'RbdEncryptionCreateOptions' } }
> >
> > ##
> > # @BlockdevVmdkSubformat:
> > --
> > 2.27.0
> >
>
> Reviewed-by: Ilya Dryomov <idryomov@gmail.com>
Thanks, applied to the block branch.
Kevin
^ permalink raw reply [flat|nested] 3+ messages in thread
end of thread, other threads:[~2021-07-02 11:40 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-06-27 11:46 [PATCH v2] block/rbd: Add support for rbd image encryption Or Ozeri
2021-06-27 13:46 ` Ilya Dryomov
2021-07-02 11:37 ` Kevin Wolf
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.