[PATCH nft v2 00/10] add typeof keyword

[PATCH nft v2 00/10] add typeof keyword

[Florian Westphal]

This patch series adds the typeof keyword.

The only dependency is a small change to libnftnl to add two new
UDATA_SET_TYPEOF enum values.

named set can be configured as follows:

set os {
   typeof osf name
   elements = { "Linux", "Windows" }
}

or

nft add set ip filter allowed "{ typeof ip daddr  . tcp dport; }"

... which is the same as the "old" 'type ipv4_addr . inet_service".

The type is stored in the kernel via the udata set infrastructure,
on listing -- if a udata type is present -- nft will validate that this
type matches the set key length.

Note that while 'typeof' can be used with concatenations, they
only work as aliases for known types -- its currently not possible
to use integer/string types via the 'typeof' keyword.

Doing so requires a bit more work to dissect the correct key
geometry on netlink dumps, we can also not fallback in this case,
i.e. if the typeof udata is not there/invalid, we would be
unable to reconstruct the needed subkey size information.

Florian Westphal (10):
      parser: add a helper for concat expression handling
      libnftnl: split nft_ctx_new/free
      src: store expr, not dtype to track data in sets
      src: parser: add syntax to provide size of variable-sized data types
      src: add "typeof" print support
      mnl: round up the map data size too
      src: netlink: remove assertion
      evaluate: print a hint about 'type,width' syntax on 0 keylen
      doc: mention 'typeof' as alternative to 'type' keyword
      tests: add typeof test cases

Pablo Neira Ayuso (1):
      parser: add typeof keyword for declarations

 23 files changed, 582 insertions(+), 154 deletions(-)
 create mode 100644 tests/shell/testcases/maps/dumps/typeof_maps_0.nft
 create mode 100755 tests/shell/testcases/maps/typeof_maps_0
 create mode 100644 tests/shell/testcases/sets/dumps/typeof_sets_0.nft
 create mode 100755 tests/shell/testcases/sets/typeof_sets_0

[PATCH nft v2 01/11] parser: add a helper for concat expression handling

[Florian Westphal]

Cull the repeated copy&paste snippets and add/use a helper for this.

Signed-off-by: Florian Westphal <fw@strlen.de>
---
 src/parser_bison.y | 99 ++++++++++++++++++++--------------------------
 1 file changed, 43 insertions(+), 56 deletions(-)

diff --git a/src/parser_bison.y b/src/parser_bison.y
index 707f46716ed3..0fd9b94c5b2f 100644
--- a/src/parser_bison.y
+++ b/src/parser_bison.y
@@ -102,6 +102,25 @@ static void location_update(struct location *loc, struct location *rhs, int n)
 	}
 }
 
+static struct expr *handle_concat_expr(const struct location *loc,
+					 struct expr *expr,
+					 struct expr *expr_l, struct expr *expr_r,
+					 struct location loc_rhs[3])
+{
+	if (expr->etype != EXPR_CONCAT) {
+		expr = concat_expr_alloc(loc);
+		compound_expr_add(expr, expr_l);
+	} else {
+		location_update(&expr_r->location, loc_rhs, 2);
+
+		expr = expr_l;
+		expr->location = *loc;
+	}
+
+	compound_expr_add(expr, expr_r);
+	return expr;
+}
+
 #define YYLLOC_DEFAULT(Current, Rhs, N)	location_update(&Current, Rhs, N)
 
 #define symbol_value(loc, str) \
@@ -1878,20 +1897,12 @@ data_type_atom_expr	:	type_identifier
 data_type_expr		:	data_type_atom_expr
 			|	data_type_expr	DOT	data_type_atom_expr
 			{
-				if ($1->etype != EXPR_CONCAT) {
-					$$ = concat_expr_alloc(&@$);
-					compound_expr_add($$, $1);
-				} else {
-					struct location rhs[] = {
-						[1]	= @2,
-						[2]	= @3,
-					};
-					location_update(&$3->location, rhs, 2);
-
-					$$ = $1;
-					$$->location = @$;
-				}
-				compound_expr_add($$, $3);
+				struct location rhs[] = {
+					[1]	= @2,
+					[2]	= @3,
+				};
+
+				$$ = handle_concat_expr(&@$, $$, $1, $3, rhs);
 			}
 			;
 
@@ -2992,20 +3003,12 @@ basic_stmt_expr		:	inclusive_or_stmt_expr
 concat_stmt_expr	:	basic_stmt_expr
 			|	concat_stmt_expr	DOT	primary_stmt_expr
 			{
-				if ($$->etype != EXPR_CONCAT) {
-					$$ = concat_expr_alloc(&@$);
-					compound_expr_add($$, $1);
-				} else {
-					struct location rhs[] = {
-						[1]	= @2,
-						[2]	= @3,
-					};
-					location_update(&$3->location, rhs, 2);
-
-					$$ = $1;
-					$$->location = @$;
-				}
-				compound_expr_add($$, $3);
+				struct location rhs[] = {
+					[1]	= @2,
+					[2]	= @3,
+				};
+
+				$$ = handle_concat_expr(&@$, $$, $1, $3, rhs);
 			}
 			;
 
@@ -3525,20 +3528,12 @@ basic_expr		:	inclusive_or_expr
 concat_expr		:	basic_expr
 			|	concat_expr		DOT		basic_expr
 			{
-				if ($$->etype != EXPR_CONCAT) {
-					$$ = concat_expr_alloc(&@$);
-					compound_expr_add($$, $1);
-				} else {
-					struct location rhs[] = {
-						[1]	= @2,
-						[2]	= @3,
-					};
-					location_update(&$3->location, rhs, 2);
-
-					$$ = $1;
-					$$->location = @$;
-				}
-				compound_expr_add($$, $3);
+				struct location rhs[] = {
+					[1]	= @2,
+					[2]	= @3,
+				};
+
+				$$ = handle_concat_expr(&@$, $$, $1, $3, rhs);
 			}
 			;
 
@@ -3946,20 +3941,12 @@ basic_rhs_expr		:	inclusive_or_rhs_expr
 concat_rhs_expr		:	basic_rhs_expr
 			|	concat_rhs_expr	DOT	basic_rhs_expr
 			{
-				if ($$->etype != EXPR_CONCAT) {
-					$$ = concat_expr_alloc(&@$);
-					compound_expr_add($$, $1);
-				} else {
-					struct location rhs[] = {
-						[1]	= @2,
-						[2]	= @3,
-					};
-					location_update(&$3->location, rhs, 2);
-
-					$$ = $1;
-					$$->location = @$;
-				}
-				compound_expr_add($$, $3);
+				struct location rhs[] = {
+					[1]	= @2,
+					[2]	= @3,
+				};
+
+				$$ = handle_concat_expr(&@$, $$, $1, $3, rhs);
 			}
 			;
 
-- 
2.23.0

[PATCH nft v2 02/11] libnftnl: split nft_ctx_new/free

[Florian Westphal]

a followup patch doesn't need e.g. symbol files to be loaded,
so split those functions to create a helper that can be used
internally by nft to create a more minimal nft context.

Signed-off-by: Florian Westphal <fw@strlen.de>
---
 include/nftables.h |  3 +++
 src/libnftables.c  | 33 ++++++++++++++++++++++++---------
 2 files changed, 27 insertions(+), 9 deletions(-)

diff --git a/include/nftables.h b/include/nftables.h
index 90d331960ef2..503adf0ca1cc 100644
--- a/include/nftables.h
+++ b/include/nftables.h
@@ -213,6 +213,9 @@ int nft_print(struct output_ctx *octx, const char *fmt, ...)
 int nft_gmp_print(struct output_ctx *octx, const char *fmt, ...)
 	__attribute__((format(printf, 2, 0)));
 
+struct nft_ctx *__nft_ctx_new(void);
+void __nft_ctx_free(struct nft_ctx *ctx);
+
 #define __NFT_OUTPUT_NOTSUPP	UINT_MAX
 
 #endif /* NFTABLES_NFTABLES_H */
diff --git a/src/libnftables.c b/src/libnftables.c
index cd2fcf2fd522..3c0d3a18ffe5 100644
--- a/src/libnftables.c
+++ b/src/libnftables.c
@@ -134,8 +134,7 @@ static void nft_ctx_netlink_init(struct nft_ctx *ctx)
 	ctx->nf_sock = nft_mnl_socket_open();
 }
 
-EXPORT_SYMBOL(nft_ctx_new);
-struct nft_ctx *nft_ctx_new(uint32_t flags)
+struct nft_ctx *__nft_ctx_new(void)
 {
 	static bool init_once;
 	struct nft_ctx *ctx;
@@ -149,17 +148,27 @@ struct nft_ctx *nft_ctx_new(uint32_t flags)
 	}
 
 	ctx = xzalloc(sizeof(struct nft_ctx));
-	nft_init(ctx);
 
 	ctx->state = xzalloc(sizeof(struct parser_state));
 	nft_ctx_add_include_path(ctx, DEFAULT_INCLUDE_PATH);
-	ctx->parser_max_errors	= 10;
 	init_list_head(&ctx->cache.list);
 	ctx->top_scope = scope_alloc();
-	ctx->flags = flags;
 	ctx->output.output_fp = stdout;
 	ctx->output.error_fp = stderr;
 
+	return ctx;
+}
+
+EXPORT_SYMBOL(nft_ctx_new);
+struct nft_ctx *nft_ctx_new(uint32_t flags)
+{
+	struct nft_ctx *ctx = __nft_ctx_new();
+
+	nft_init(ctx);
+
+	ctx->parser_max_errors = 10;
+	ctx->flags = flags;
+
 	if (flags == NFT_CTX_DEFAULT)
 		nft_ctx_netlink_init(ctx);
 
@@ -282,23 +291,29 @@ const char *nft_ctx_get_error_buffer(struct nft_ctx *ctx)
 	return get_cookie_buffer(&ctx->output.error_cookie);
 }
 
-EXPORT_SYMBOL(nft_ctx_free);
-void nft_ctx_free(struct nft_ctx *ctx)
+void __nft_ctx_free(struct nft_ctx *ctx)
 {
 	if (ctx->nf_sock)
 		mnl_socket_close(ctx->nf_sock);
 
 	exit_cookie(&ctx->output.output_cookie);
 	exit_cookie(&ctx->output.error_cookie);
-	iface_cache_release();
 	cache_release(&ctx->cache);
 	nft_ctx_clear_include_paths(ctx);
 	scope_free(ctx->top_scope);
 	xfree(ctx->state);
-	nft_exit(ctx);
+
 	xfree(ctx);
 }
 
+EXPORT_SYMBOL(nft_ctx_free);
+void nft_ctx_free(struct nft_ctx *ctx)
+{
+	nft_exit(ctx);
+	__nft_ctx_free(ctx);
+	iface_cache_release();
+}
+
 EXPORT_SYMBOL(nft_ctx_set_output);
 FILE *nft_ctx_set_output(struct nft_ctx *ctx, FILE *fp)
 {
-- 
2.23.0

[PATCH nft v2 03/11] src: store expr, not dtype to track data in sets

[Florian Westphal]

This will be needed once we add support for the 'typeof' keyword to
handle maps that could e.g. store 'ct helper' "type" values.

Instead of:

set foo {
	type ipv4_addr . mark;

this would allow

set foo {
	typeof(ip saddr) . typeof(ct mark);

(exact syntax TBD).

This would be needed to allow sets that store variable-sized data types
(string, integer and the like) that can't be used at at the moment.

Adding special data types for everything is problematic due to the
large amount of different types needed.

For anonymous sets, e.g. "string" can be used because the needed size can
be inferred from the statement, e.g.  'osf name { "Windows", "Linux }',
but in case of named sets that won't work because 'type string' lacks the
context needed to derive the size information.

With 'typeof(osf name)' the context is there, but at the moment it won't
help because the expression is discarded instantly and only the data
type is retained.

Signed-off-by: Florian Westphal <fw@strlen.de>
---
 include/datatype.h |  1 -
 include/netlink.h  |  1 -
 include/rule.h     |  6 ++---
 src/datatype.c     |  5 ----
 src/evaluate.c     | 58 ++++++++++++++++++++++++++++++++--------------
 src/expression.c   |  2 +-
 src/json.c         |  4 ++--
 src/mnl.c          |  6 ++---
 src/monitor.c      |  2 +-
 src/netlink.c      | 32 ++++++++++++-------------
 src/parser_bison.y |  3 +--
 src/parser_json.c  |  8 +++++--
 src/rule.c         |  8 +++----
 src/segtree.c      |  8 +++++--
 14 files changed, 81 insertions(+), 63 deletions(-)

diff --git a/include/datatype.h b/include/datatype.h
index 49b8f608aa1d..04b4892b29ac 100644
--- a/include/datatype.h
+++ b/include/datatype.h
@@ -293,7 +293,6 @@ concat_subtype_lookup(uint32_t type, unsigned int n)
 
 extern const struct datatype *
 set_datatype_alloc(const struct datatype *orig_dtype, unsigned int byteorder);
-extern void set_datatype_destroy(const struct datatype *dtype);
 
 extern void time_print(uint64_t msec, struct output_ctx *octx);
 extern struct error_record *time_parse(const struct location *loc,
diff --git a/include/netlink.h b/include/netlink.h
index 53a55b61e4de..d02533ec4430 100644
--- a/include/netlink.h
+++ b/include/netlink.h
@@ -189,6 +189,5 @@ int netlink_events_trace_cb(const struct nlmsghdr *nlh, int type,
 			    struct netlink_mon_handler *monh);
 
 enum nft_data_types dtype_map_to_kernel(const struct datatype *dtype);
-const struct datatype *dtype_map_from_kernel(enum nft_data_types type);
 
 #endif /* NFTABLES_NETLINK_H */
diff --git a/include/rule.h b/include/rule.h
index dadeb4b941da..ce1f40db031e 100644
--- a/include/rule.h
+++ b/include/rule.h
@@ -283,8 +283,7 @@ extern struct rule *rule_lookup_by_index(const struct chain *chain,
  * @gc_int:	garbage collection interval
  * @timeout:	default timeout value
  * @key:	key expression (data type, length))
- * @datatype:	mapping data type
- * @datalen:	mapping data len
+ * @data:	mapping data expression
  * @objtype:	mapping object type
  * @init:	initializer
  * @rg_cache:	cached range element (left)
@@ -301,8 +300,7 @@ struct set {
 	uint32_t		gc_int;
 	uint64_t		timeout;
 	struct expr		*key;
-	const struct datatype	*datatype;
-	unsigned int		datalen;
+	struct expr		*data;
 	uint32_t		objtype;
 	struct expr		*init;
 	struct expr		*rg_cache;
diff --git a/src/datatype.c b/src/datatype.c
index b9e167e03765..189e1b482c29 100644
--- a/src/datatype.c
+++ b/src/datatype.c
@@ -1190,11 +1190,6 @@ const struct datatype *set_datatype_alloc(const struct datatype *orig_dtype,
 	return dtype;
 }
 
-void set_datatype_destroy(const struct datatype *dtype)
-{
-	datatype_free(dtype);
-}
-
 static struct error_record *time_unit_parse(const struct location *loc,
 					    const char *str, uint64_t *unit)
 {
diff --git a/src/evaluate.c b/src/evaluate.c
index a865902c0fc7..91d6b254c659 100644
--- a/src/evaluate.c
+++ b/src/evaluate.c
@@ -1367,6 +1367,7 @@ static int expr_evaluate_map(struct eval_ctx *ctx, struct expr **expr)
 {
 	struct expr_ctx ectx = ctx->ectx;
 	struct expr *map = *expr, *mappings;
+	const struct datatype *dtype;
 	struct expr *key;
 
 	expr_set_context(&ctx->ectx, NULL, 0);
@@ -1389,10 +1390,14 @@ static int expr_evaluate_map(struct eval_ctx *ctx, struct expr **expr)
 		mappings = implicit_set_declaration(ctx, "__map%d",
 						    key,
 						    mappings);
-		mappings->set->datatype =
-			datatype_get(set_datatype_alloc(ectx.dtype,
-							ectx.byteorder));
-		mappings->set->datalen  = ectx.len;
+
+		dtype = set_datatype_alloc(ectx.dtype, ectx.byteorder);
+
+		mappings->set->data = constant_expr_alloc(&netlink_location,
+							  dtype, dtype->byteorder,
+							  ectx.len, NULL);
+		if (ectx.len && mappings->set->data->len != ectx.len)
+			BUG("%d vs %d\n", mappings->set->data->len, ectx.len);
 
 		map->mappings = mappings;
 
@@ -1428,7 +1433,7 @@ static int expr_evaluate_map(struct eval_ctx *ctx, struct expr **expr)
 					 map->mappings->set->key->dtype->desc,
 					 map->map->dtype->desc);
 
-	datatype_set(map, map->mappings->set->datatype);
+	datatype_set(map, map->mappings->set->data->dtype);
 	map->flags |= EXPR_F_CONSTANT;
 
 	/* Data for range lookups needs to be in big endian order */
@@ -1458,7 +1463,12 @@ static int expr_evaluate_mapping(struct eval_ctx *ctx, struct expr **expr)
 				  "Key must be a constant");
 	mapping->flags |= mapping->left->flags & EXPR_F_SINGLETON;
 
-	expr_set_context(&ctx->ectx, set->datatype, set->datalen);
+	if (set->data) {
+		expr_set_context(&ctx->ectx, set->data->dtype, set->data->len);
+	} else {
+		assert((set->flags & NFT_SET_MAP) == 0);
+	}
+
 	if (expr_evaluate(ctx, &mapping->right) < 0)
 		return -1;
 	if (!expr_is_constant(mapping->right))
@@ -2103,7 +2113,7 @@ static int stmt_evaluate_arg(struct eval_ctx *ctx, struct stmt *stmt,
 					 (*expr)->len);
 	else if ((*expr)->dtype->type != TYPE_INTEGER &&
 		 !datatype_equal((*expr)->dtype, dtype))
-		return stmt_binary_error(ctx, *expr, stmt,
+		return stmt_binary_error(ctx, *expr, stmt,		/* verdict vs invalid? */
 					 "datatype mismatch: expected %s, "
 					 "expression has type %s",
 					 dtype->desc, (*expr)->dtype->desc);
@@ -3097,9 +3107,9 @@ static int stmt_evaluate_map(struct eval_ctx *ctx, struct stmt *stmt)
 				  "Key expression comments are not supported");
 
 	if (stmt_evaluate_arg(ctx, stmt,
-			      stmt->map.set->set->datatype,
-			      stmt->map.set->set->datalen,
-			      stmt->map.set->set->datatype->byteorder,
+			      stmt->map.set->set->data->dtype,
+			      stmt->map.set->set->data->len,
+			      stmt->map.set->set->data->byteorder,
 			      &stmt->map.data->key) < 0)
 		return -1;
 	if (expr_is_constant(stmt->map.data))
@@ -3145,8 +3155,12 @@ static int stmt_evaluate_objref_map(struct eval_ctx *ctx, struct stmt *stmt)
 
 		mappings = implicit_set_declaration(ctx, "__objmap%d",
 						    key, mappings);
-		mappings->set->datatype = &string_type;
-		mappings->set->datalen  = NFT_OBJ_MAXNAMELEN * BITS_PER_BYTE;
+
+		mappings->set->data = constant_expr_alloc(&netlink_location,
+							  &string_type,
+							  BYTEORDER_HOST_ENDIAN,
+							  NFT_OBJ_MAXNAMELEN * BITS_PER_BYTE,
+							  NULL);
 		mappings->set->objtype  = stmt->objref.type;
 
 		map->mappings = mappings;
@@ -3181,7 +3195,7 @@ static int stmt_evaluate_objref_map(struct eval_ctx *ctx, struct stmt *stmt)
 					 map->mappings->set->key->dtype->desc,
 					 map->map->dtype->desc);
 
-	datatype_set(map, map->mappings->set->datatype);
+	datatype_set(map, map->mappings->set->data->dtype);
 	map->flags |= EXPR_F_CONSTANT;
 
 	/* Data for range lookups needs to be in big endian order */
@@ -3326,17 +3340,25 @@ static int set_evaluate(struct eval_ctx *ctx, struct set *set)
 		return set_error(ctx, set, "concatenated types not supported in interval sets");
 
 	if (set_is_datamap(set->flags)) {
-		if (set->datatype == NULL)
+		if (set->data == NULL)
 			return set_error(ctx, set, "map definition does not "
 					 "specify mapping data type");
 
-		set->datalen = set->datatype->size;
-		if (set->datalen == 0 && set->datatype->type != TYPE_VERDICT)
+		if (set->data->len == 0 && set->data->dtype->type != TYPE_VERDICT)
 			return set_error(ctx, set, "unqualified mapping data "
 					 "type specified in map definition");
 	} else if (set_is_objmap(set->flags)) {
-		set->datatype = &string_type;
-		set->datalen  = NFT_OBJ_MAXNAMELEN * BITS_PER_BYTE;
+		if (set->data) {
+			assert(set->data->etype == EXPR_VALUE);
+			assert(set->data->dtype == &string_type);
+		}
+
+		assert(set->data == NULL);
+		set->data = constant_expr_alloc(&netlink_location, &string_type,
+						BYTEORDER_HOST_ENDIAN,
+						NFT_OBJ_MAXNAMELEN * BITS_PER_BYTE,
+						NULL);
+
 	}
 
 	ctx->set = set;
diff --git a/src/expression.c b/src/expression.c
index 5070b1014392..6fa2f1dd9b12 100644
--- a/src/expression.c
+++ b/src/expression.c
@@ -1010,7 +1010,7 @@ static void map_expr_print(const struct expr *expr, struct output_ctx *octx)
 {
 	expr_print(expr->map, octx);
 	if (expr->mappings->etype == EXPR_SET_REF &&
-	    expr->mappings->set->datatype->type == TYPE_VERDICT)
+	    expr->mappings->set->data->dtype->type == TYPE_VERDICT)
 		nft_print(octx, " vmap ");
 	else
 		nft_print(octx, " map ");
diff --git a/src/json.c b/src/json.c
index 3498e24db363..1906e7db7c19 100644
--- a/src/json.c
+++ b/src/json.c
@@ -82,7 +82,7 @@ static json_t *set_print_json(struct output_ctx *octx, const struct set *set)
 
 	if (set_is_datamap(set->flags)) {
 		type = "map";
-		datatype_ext = set->datatype->name;
+		datatype_ext = set->data->dtype->name;
 	} else if (set_is_objmap(set->flags)) {
 		type = "map";
 		datatype_ext = obj_type_name(set->objtype);
@@ -645,7 +645,7 @@ json_t *map_expr_json(const struct expr *expr, struct output_ctx *octx)
 	const char *type = "map";
 
 	if (expr->mappings->etype == EXPR_SET_REF &&
-	    expr->mappings->set->datatype->type == TYPE_VERDICT)
+	    expr->mappings->set->data->dtype->type == TYPE_VERDICT)
 		type = "vmap";
 
 	return json_pack("{s:{s:o, s:o}}", type,
diff --git a/src/mnl.c b/src/mnl.c
index aa5b0b4652e8..06b790c01acc 100644
--- a/src/mnl.c
+++ b/src/mnl.c
@@ -839,9 +839,9 @@ int mnl_nft_set_add(struct netlink_ctx *ctx, const struct cmd *cmd,
 			  div_round_up(set->key->len, BITS_PER_BYTE));
 	if (set_is_datamap(set->flags)) {
 		nftnl_set_set_u32(nls, NFTNL_SET_DATA_TYPE,
-				  dtype_map_to_kernel(set->datatype));
+				  dtype_map_to_kernel(set->data->dtype));
 		nftnl_set_set_u32(nls, NFTNL_SET_DATA_LEN,
-				  set->datalen / BITS_PER_BYTE);
+				  set->data->len / BITS_PER_BYTE);
 	}
 	if (set_is_objmap(set->flags))
 		nftnl_set_set_u32(nls, NFTNL_SET_OBJ_TYPE, set->objtype);
@@ -873,7 +873,7 @@ int mnl_nft_set_add(struct netlink_ctx *ctx, const struct cmd *cmd,
 
 	if (set_is_datamap(set->flags) &&
 	    !nftnl_udata_put_u32(udbuf, NFTNL_UDATA_SET_DATABYTEORDER,
-				 set->datatype->byteorder))
+				 set->data->byteorder))
 		memory_allocation_error();
 
 	if (set->automerge &&
diff --git a/src/monitor.c b/src/monitor.c
index ea0393cddff4..d586cfa34a97 100644
--- a/src/monitor.c
+++ b/src/monitor.c
@@ -401,7 +401,7 @@ static int netlink_events_setelem_cb(const struct nlmsghdr *nlh, int type,
 	 */
 	dummyset = set_alloc(monh->loc);
 	dummyset->key = expr_clone(set->key);
-	dummyset->datatype = set->datatype;
+	dummyset->data = set->data;
 	dummyset->flags = set->flags;
 	dummyset->init = set_expr_alloc(monh->loc, set);
 
diff --git a/src/netlink.c b/src/netlink.c
index 9fc0b17194a0..46897e43e723 100644
--- a/src/netlink.c
+++ b/src/netlink.c
@@ -534,7 +534,7 @@ enum nft_data_types dtype_map_to_kernel(const struct datatype *dtype)
 	}
 }
 
-const struct datatype *dtype_map_from_kernel(enum nft_data_types type)
+static const struct datatype *dtype_map_from_kernel(enum nft_data_types type)
 {
 	switch (type) {
 	case NFT_DATA_VERDICT:
@@ -581,10 +581,10 @@ struct set *netlink_delinearize_set(struct netlink_ctx *ctx,
 				    const struct nftnl_set *nls)
 {
 	const struct nftnl_udata *ud[NFTNL_UDATA_SET_MAX + 1] = {};
-	uint32_t flags, key, data, data_len, objtype = 0;
 	enum byteorder keybyteorder = BYTEORDER_INVALID;
 	enum byteorder databyteorder = BYTEORDER_INVALID;
-	const struct datatype *keytype, *datatype;
+	const struct datatype *keytype, *datatype = NULL;
+	uint32_t flags, key, objtype = 0;
 	bool automerge = false;
 	const char *udata;
 	struct set *set;
@@ -618,6 +618,8 @@ struct set *netlink_delinearize_set(struct netlink_ctx *ctx,
 
 	flags = nftnl_set_get_u32(nls, NFTNL_SET_FLAGS);
 	if (set_is_datamap(flags)) {
+		uint32_t data;
+
 		data = nftnl_set_get_u32(nls, NFTNL_SET_DATA_TYPE);
 		datatype = dtype_map_from_kernel(data);
 		if (datatype == NULL) {
@@ -626,8 +628,7 @@ struct set *netlink_delinearize_set(struct netlink_ctx *ctx,
 					 data);
 			return NULL;
 		}
-	} else
-		datatype = NULL;
+	}
 
 	if (set_is_objmap(flags)) {
 		objtype = nftnl_set_get_u32(nls, NFTNL_SET_OBJ_TYPE);
@@ -650,16 +651,13 @@ struct set *netlink_delinearize_set(struct netlink_ctx *ctx,
 
 	set->objtype = objtype;
 
+	set->data = NULL;
 	if (datatype)
-		set->datatype = datatype_get(set_datatype_alloc(datatype,
-								databyteorder));
-	else
-		set->datatype = NULL;
-
-	if (nftnl_set_is_set(nls, NFTNL_SET_DATA_LEN)) {
-		data_len = nftnl_set_get_u32(nls, NFTNL_SET_DATA_LEN);
-		set->datalen = data_len * BITS_PER_BYTE;
-	}
+		set->data = constant_expr_alloc(&netlink_location,
+						set_datatype_alloc(datatype, databyteorder),
+						databyteorder,
+						nftnl_set_get_u32(nls, NFTNL_SET_DATA_LEN) * BITS_PER_BYTE,
+						NULL);
 
 	if (nftnl_set_is_set(nls, NFTNL_SET_TIMEOUT))
 		set->timeout = nftnl_set_get_u64(nls, NFTNL_SET_TIMEOUT);
@@ -847,10 +845,10 @@ int netlink_delinearize_setelem(struct nftnl_set_elem *nlse,
 			goto out;
 
 		data = netlink_alloc_data(&netlink_location, &nld,
-					  set->datatype->type == TYPE_VERDICT ?
+					  set->data->dtype->type == TYPE_VERDICT ?
 					  NFT_REG_VERDICT : NFT_REG_1);
-		datatype_set(data, set->datatype);
-		data->byteorder = set->datatype->byteorder;
+		datatype_set(data, set->data->dtype);
+		data->byteorder = set->data->byteorder;
 		if (data->byteorder == BYTEORDER_HOST_ENDIAN)
 			mpz_switch_byteorder(data->value, data->len / BITS_PER_BYTE);
 
diff --git a/src/parser_bison.y b/src/parser_bison.y
index 0fd9b94c5b2f..6d17539fa57e 100644
--- a/src/parser_bison.y
+++ b/src/parser_bison.y
@@ -1749,9 +1749,8 @@ map_block		:	/* empty */	{ $$ = $<set>-1; }
 						stmt_separator
 			{
 				$1->key = $3;
-				$1->datatype = $5->dtype;
+				$1->data = $5;
 
-				expr_free($5);
 				$1->flags |= NFT_SET_MAP;
 				$$ = $1;
 			}
diff --git a/src/parser_json.c b/src/parser_json.c
index 031930e2d708..85082ccee7ef 100644
--- a/src/parser_json.c
+++ b/src/parser_json.c
@@ -2826,11 +2826,15 @@ static struct cmd *json_parse_cmd_add_set(struct json_ctx *ctx, json_t *root,
 	}
 
 	if (!json_unpack(root, "{s:s}", "map", &dtype_ext)) {
+		const struct datatype *dtype;
+
 		set->objtype = string_to_nft_object(dtype_ext);
 		if (set->objtype) {
 			set->flags |= NFT_SET_OBJECT;
-		} else if (datatype_lookup_byname(dtype_ext)) {
-			set->datatype = datatype_lookup_byname(dtype_ext);
+		} else if ((dtype = datatype_lookup_byname(dtype_ext))) {
+			set->data = constant_expr_alloc(&netlink_location,
+							dtype, dtype->byteorder,
+							dtype->size, NULL);
 			set->flags |= NFT_SET_MAP;
 		} else {
 			json_error(ctx, "Invalid map type '%s'.", dtype_ext);
diff --git a/src/rule.c b/src/rule.c
index d985d3a2316e..f8cd4a73054b 100644
--- a/src/rule.c
+++ b/src/rule.c
@@ -332,8 +332,8 @@ struct set *set_clone(const struct set *set)
 	new_set->gc_int		= set->gc_int;
 	new_set->timeout	= set->timeout;
 	new_set->key		= expr_clone(set->key);
-	new_set->datatype	= datatype_get(set->datatype);
-	new_set->datalen	= set->datalen;
+	if (set->data)
+		new_set->data	= expr_clone(set->data);
 	new_set->objtype	= set->objtype;
 	new_set->policy		= set->policy;
 	new_set->automerge	= set->automerge;
@@ -356,7 +356,7 @@ void set_free(struct set *set)
 		expr_free(set->init);
 	handle_free(&set->handle);
 	expr_free(set->key);
-	set_datatype_destroy(set->datatype);
+	expr_free(set->data);
 	xfree(set);
 }
 
@@ -469,7 +469,7 @@ static void set_print_declaration(const struct set *set,
 	nft_print(octx, "%s%stype %s",
 		  opts->tab, opts->tab, set->key->dtype->name);
 	if (set_is_datamap(set->flags))
-		nft_print(octx, " : %s", set->datatype->name);
+		nft_print(octx, " : %s", set->data->dtype->name);
 	else if (set_is_objmap(set->flags))
 		nft_print(octx, " : %s", obj_type_name(set->objtype));
 
diff --git a/src/segtree.c b/src/segtree.c
index d1dbe10c81a7..e8e32412f3a4 100644
--- a/src/segtree.c
+++ b/src/segtree.c
@@ -79,8 +79,12 @@ static void seg_tree_init(struct seg_tree *tree, const struct set *set,
 	tree->root	= RB_ROOT;
 	tree->keytype	= set->key->dtype;
 	tree->keylen	= set->key->len;
-	tree->datatype	= set->datatype;
-	tree->datalen	= set->datalen;
+	tree->datatype	= NULL;
+	tree->datalen	= 0;
+	if (set->data) {
+		tree->datatype	= set->data->dtype;
+		tree->datalen	= set->data->len;
+	}
 	tree->byteorder	= first->byteorder;
 	tree->debug_mask = debug_mask;
 }
-- 
2.23.0

[PATCH nft v2 04/11] src: parser: add syntax to provide size of variable-sized data types

[Florian Westphal]

This allows creation of sets with string and integer types by
providing the datatype width using a comma, e.g.

"type string, 64" or "integer, 32".

This is mainly intended as a fallback for the upcoming "typeof"
keyword -- if we can't make sense of the kernel provided type
(or its missing entirely), we can then fallback to this format.

Signed-off-by: Florian Westphal <fw@strlen.de>
---
 src/parser_bison.y | 18 ++++++++++++++++++
 src/rule.c         | 24 +++++++++++++++++++-----
 2 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/parser_bison.y b/src/parser_bison.y
index 6d17539fa57e..c6cad19f52fb 100644
--- a/src/parser_bison.y
+++ b/src/parser_bison.y
@@ -1891,6 +1891,24 @@ data_type_atom_expr	:	type_identifier
 				$$ = constant_expr_alloc(&@1, &time_type, time_type.byteorder,
 							 time_type.size, NULL);
 			}
+			|	type_identifier	COMMA	NUM
+			{
+				const struct datatype *dtype = datatype_lookup_byname($1);
+				if (dtype == NULL) {
+					erec_queue(error(&@1, "unknown datatype %s", $1),
+						   state->msgs);
+					YYERROR;
+				}
+
+				if (dtype->size) {
+					erec_queue(error(&@1, "Datatype %s has a fixed type", $1),
+						   state->msgs);
+					YYERROR;
+				}
+				$$ = constant_expr_alloc(&@1, dtype, dtype->byteorder,
+							 $3, NULL);
+				xfree($1);
+			}
 			;
 
 data_type_expr		:	data_type_atom_expr
diff --git a/src/rule.c b/src/rule.c
index f8cd4a73054b..a94860865f31 100644
--- a/src/rule.c
+++ b/src/rule.c
@@ -438,6 +438,16 @@ const char *set_policy2str(uint32_t policy)
 	}
 }
 
+static void set_print_key(const struct expr *expr, struct output_ctx *octx)
+{
+	const struct datatype *dtype = expr->dtype;
+
+	if (dtype->size || dtype->type == TYPE_VERDICT)
+		nft_print(octx, "%s", dtype->name);
+	else
+		nft_print(octx, "%s,%d", dtype->name, expr->len);
+}
+
 static void set_print_declaration(const struct set *set,
 				  struct print_fmt_options *opts,
 				  struct output_ctx *octx)
@@ -466,12 +476,16 @@ static void set_print_declaration(const struct set *set,
 	if (nft_output_handle(octx))
 		nft_print(octx, " # handle %" PRIu64, set->handle.handle.id);
 	nft_print(octx, "%s", opts->nl);
-	nft_print(octx, "%s%stype %s",
-		  opts->tab, opts->tab, set->key->dtype->name);
-	if (set_is_datamap(set->flags))
-		nft_print(octx, " : %s", set->data->dtype->name);
-	else if (set_is_objmap(set->flags))
+	nft_print(octx, "%s%stype ",
+		  opts->tab, opts->tab);
+	set_print_key(set->key, octx);
+
+	if (set_is_datamap(set->flags)) {
+		nft_print(octx, " : ");
+		set_print_key(set->data, octx);
+	} else if (set_is_objmap(set->flags)) {
 		nft_print(octx, " : %s", obj_type_name(set->objtype));
+	}
 
 	nft_print(octx, "%s", opts->stmt_separator);
 
-- 
2.23.0

[PATCH nft v2 05/11] parser: add typeof keyword for declarations

[Florian Westphal]

From: Pablo Neira Ayuso <pablo@netfilter.org>

Add a typeof keyword to automatically use the correct type in set and map
declarations.

table filter {
	set blacklist {
		typeof ip saddr
	}

	chain input {
		ip saddr @blacklist counter drop
	}
}

Signed-off-by: Pablo Neira Ayuso <pablo@netfilter.org>
Signed-off-by: Florian Westphal <fw@strlen.de>
---
 src/parser_bison.y | 36 ++++++++++++++++++++++++++++++++++--
 src/scanner.l      |  1 +
 2 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/src/parser_bison.y b/src/parser_bison.y
index c6cad19f52fb..bbb4933c9884 100644
--- a/src/parser_bison.y
+++ b/src/parser_bison.y
@@ -223,6 +223,8 @@ int nft_lex(void *, void *, void *);
 %token WSCALE			"wscale"
 %token SACKPERM			"sack-perm"
 
+%token TYPEOF			"typeof"
+
 %token HOOK			"hook"
 %token DEVICE			"device"
 %token DEVICES			"devices"
@@ -658,8 +660,8 @@ int nft_lex(void *, void *, void *);
 
 %type <expr>			symbol_expr verdict_expr integer_expr variable_expr chain_expr policy_expr
 %destructor { expr_free($$); }	symbol_expr verdict_expr integer_expr variable_expr chain_expr policy_expr
-%type <expr>			primary_expr shift_expr and_expr
-%destructor { expr_free($$); }	primary_expr shift_expr and_expr
+%type <expr>			primary_expr shift_expr and_expr typeof_expr
+%destructor { expr_free($$); }	primary_expr shift_expr and_expr typeof_expr
 %type <expr>			exclusive_or_expr inclusive_or_expr
 %destructor { expr_free($$); }	exclusive_or_expr inclusive_or_expr
 %type <expr>			basic_expr
@@ -1671,6 +1673,19 @@ chain_block		:	/* empty */	{ $$ = $<chain>-1; }
 			}
 			;
 
+typeof_expr		:	primary_expr
+			|	typeof_expr		DOT		primary_expr
+			{
+				struct location rhs[] = {
+					[1]	= @2,
+					[2]	= @3,
+				};
+
+				$$ = handle_concat_expr(&@$, $$, $1, $3, rhs);
+			}
+			;
+
+
 set_block_alloc		:	/* empty */
 			{
 				$$ = set_alloc(NULL);
@@ -1685,6 +1700,12 @@ set_block		:	/* empty */	{ $$ = $<set>-1; }
 				$1->key = $3;
 				$$ = $1;
 			}
+			|	set_block	TYPEOF		typeof_expr	stmt_separator
+			{
+				$1->key = $3;
+				datatype_set($1->key, $3->dtype);
+				$$ = $1;
+			}
 			|	set_block	FLAGS		set_flag_list	stmt_separator
 			{
 				$1->flags = $3;
@@ -1754,6 +1775,17 @@ map_block		:	/* empty */	{ $$ = $<set>-1; }
 				$1->flags |= NFT_SET_MAP;
 				$$ = $1;
 			}
+			|	map_block	TYPEOF
+						typeof_expr	COLON	typeof_expr
+						stmt_separator
+			{
+				$1->key = $3;
+				datatype_set($1->key, $3->dtype);
+				$1->data = $5;
+
+				$1->flags |= NFT_SET_MAP;
+				$$ = $1;
+			}
 			|	map_block	TYPE
 						data_type_expr	COLON	COUNTER
 						stmt_separator
diff --git a/src/scanner.l b/src/scanner.l
index d32adf4897ae..4fbdcf2afa4b 100644
--- a/src/scanner.l
+++ b/src/scanner.l
@@ -385,6 +385,7 @@ addrstring	({macaddr}|{ip4addr}|{ip6addr})
 "saddr"			{ return SADDR; }
 "daddr"			{ return DADDR; }
 "type"			{ return TYPE; }
+"typeof"		{ return TYPEOF; }
 
 "vlan"			{ return VLAN; }
 "id"			{ return ID; }
-- 
2.23.0

[PATCH nft v2 06/11] src: add "typeof" print support

[Florian Westphal]

Use the 'udata' facility to store the original string in the kernel,
then use the existing 'describe' facility to resolve it back to a name.

This isn't strictly required, we can fall back to the 'type, width'
format where needed.  This can happen in any of these cases:

 - no udata description
 - unknown expression/udata has garbage string
 - udata is inconsistent (udata expression maps to a data type with a
   different size than the set key size)

'type, width' allows to list and restore sets even if the original
'typeof' expression description gets lost.

Because the grammar enforces (for maps), to use either

type data_type : data_type
or
typeof expression : expression

we need to check both key and data for a valid descriptive name.
If both check out, flag set->key_typeof_valid as true, then use that
during printing.

Signed-off-by: Florian Westphal <fw@strlen.de>
---
 include/rule.h |   1 +
 src/mnl.c      |  33 ++++++++++
 src/netlink.c  | 163 +++++++++++++++++++++++++++++++++++++++++++++----
 src/rule.c     |  38 ++++++++----
 4 files changed, 211 insertions(+), 24 deletions(-)

diff --git a/include/rule.h b/include/rule.h
index ce1f40db031e..6301fe35b591 100644
--- a/include/rule.h
+++ b/include/rule.h
@@ -307,6 +307,7 @@ struct set {
 	uint32_t		policy;
 	bool			root;
 	bool			automerge;
+	bool			key_typeof_valid;
 	struct {
 		uint32_t	size;
 	} desc;
diff --git a/src/mnl.c b/src/mnl.c
index 06b790c01acc..75113c74c224 100644
--- a/src/mnl.c
+++ b/src/mnl.c
@@ -811,6 +811,35 @@ err:
 	return NULL;
 }
 
+static void set_key_expression(struct netlink_ctx *ctx,
+				struct expr *expr, uint32_t set_flags,
+				struct nftnl_udata_buf *udbuf,
+				unsigned int type)
+{
+	struct output_ctx octx = {};
+	char buf[64];
+
+	if (expr->flags & EXPR_F_CONSTANT ||
+	    set_flags & NFT_SET_ANONYMOUS)
+		return;
+
+	buf[0] = 0;
+	/* Set definition uses typeof to define datatype. */
+	octx.output_fp = fmemopen(buf, sizeof(buf), "w");
+	if (octx.output_fp) {
+		char *end;
+
+		expr_print(expr, &octx);
+		fclose(octx.output_fp);
+		end = strchr(buf, '&');
+		if (end)
+			* end = 0;
+
+		if (!nftnl_udata_put(udbuf, type, strlen(buf) + 1, buf))
+			memory_allocation_error();
+	}
+}
+
 /*
  * Set
  */
@@ -881,6 +910,10 @@ int mnl_nft_set_add(struct netlink_ctx *ctx, const struct cmd *cmd,
 				 set->automerge))
 		memory_allocation_error();
 
+	set_key_expression(ctx, set->key, set->flags, udbuf, NFTNL_UDATA_SET_TYPEOF_KEY);
+	if (set->data)
+		set_key_expression(ctx, set->data, set->flags, udbuf, NFTNL_UDATA_SET_TYPEOF_DATA);
+
 	nftnl_set_set_data(nls, NFTNL_SET_USERDATA, nftnl_udata_buf_data(udbuf),
 			   nftnl_udata_buf_len(udbuf));
 	nftnl_udata_buf_free(udbuf);
diff --git a/src/netlink.c b/src/netlink.c
index 46897e43e723..790c663ccd6d 100644
--- a/src/netlink.c
+++ b/src/netlink.c
@@ -32,6 +32,7 @@
 #include <linux/netfilter.h>
 
 #include <nftables.h>
+#include <parser.h>
 #include <netlink.h>
 #include <mnl.h>
 #include <expression.h>
@@ -570,6 +571,11 @@ static int set_parse_udata_cb(const struct nftnl_udata *attr, void *data)
 		if (len != sizeof(uint32_t))
 			return -1;
 		break;
+	case NFTNL_UDATA_SET_TYPEOF_KEY:
+	case NFTNL_UDATA_SET_TYPEOF_DATA:
+		if (len < 3)
+			return -1;
+		break;
 	default:
 		return 0;
 	}
@@ -577,6 +583,98 @@ static int set_parse_udata_cb(const struct nftnl_udata *attr, void *data)
 	return 0;
 }
 
+static int parse_desc(struct nft_ctx *nft, const char *buf, struct list_head *cmds)
+{
+	static const struct input_descriptor indesc_udata = {
+		.type	= INDESC_BUFFER,
+		.name	= "<setudata>",
+	};
+	struct error_record *erec, *nexte;
+	LIST_HEAD(errors);
+	int ret;
+
+	parser_init(nft, nft->state, &errors, cmds, nft->top_scope);
+	nft->scanner = scanner_init(nft->state);
+	scanner_push_buffer(nft->scanner, &indesc_udata, buf);
+
+	ret = nft_parse(nft, nft->scanner, nft->state);
+
+	list_for_each_entry_safe(erec, nexte, &errors, list) {
+		list_del(&erec->list);
+		erec_destroy(erec);
+	}
+
+	if (nft->scanner) {
+		scanner_destroy(nft);
+		nft->scanner = NULL;
+	}
+
+	if (ret != 0 || nft->state->nerrs > 0)
+		return -1;
+
+	return 0;
+}
+
+static struct expr *set_make_key(const struct nftnl_udata *ud)
+{
+	struct cmd *cmd, *nextc;
+	struct nft_ctx *nft;
+	char cmdline[1024];
+	struct expr *expr;
+	const char *buf;
+	LIST_HEAD(cmds);
+	uint8_t len;
+	int i;
+
+	if (!ud)
+		return NULL;
+
+	len = nftnl_udata_len(ud);
+	if (!len)
+		return NULL;
+
+	buf = nftnl_udata_get(ud);
+	if (!buf)
+		return NULL;
+
+	for (i = 0; i < len; i++) {
+		if (buf[i] > 'z')
+			return NULL;
+		if (buf[i] == ' ' || buf[i] == '6')
+			continue;
+		if (buf[i] < 'a') {
+			if (buf[i] == '\0' && i == len - 1)
+				continue;
+
+			return NULL;
+		}
+	}
+
+	snprintf(cmdline, sizeof(cmdline), "describe %s\n", buf);
+
+	nft = __nft_ctx_new();
+	parse_desc(nft, cmdline, &cmds);
+
+	expr = NULL;
+	list_for_each_entry_safe(cmd, nextc, &cmds, list) {
+		if (cmd->op == CMD_DESCRIBE && !expr)
+			expr = expr_get(cmd->expr);
+		list_del(&cmd->list);
+		cmd_free(cmd);
+	}
+
+	__nft_ctx_free(nft);
+	return expr;
+}
+
+static bool set_udata_key_valid(const struct expr *e, const struct datatype *d, uint32_t len)
+{
+	if (!e)
+		return false;
+
+	return div_round_up(e->len, BITS_PER_BYTE) == len / BITS_PER_BYTE;
+}
+
 struct set *netlink_delinearize_set(struct netlink_ctx *ctx,
 				    const struct nftnl_set *nls)
 {
@@ -584,11 +682,17 @@ struct set *netlink_delinearize_set(struct netlink_ctx *ctx,
 	enum byteorder keybyteorder = BYTEORDER_INVALID;
 	enum byteorder databyteorder = BYTEORDER_INVALID;
 	const struct datatype *keytype, *datatype = NULL;
+	struct expr *typeof_expr_key, *typeof_expr_data;
 	uint32_t flags, key, objtype = 0;
+	const struct datatype *dtype;
 	bool automerge = false;
 	const char *udata;
 	struct set *set;
 	uint32_t ulen;
+	uint32_t klen;
+
+	typeof_expr_key = NULL;
+	typeof_expr_data = NULL;
 
 	if (nftnl_set_is_set(nls, NFTNL_SET_USERDATA)) {
 		udata = nftnl_set_get_data(nls, NFTNL_SET_USERDATA, &ulen);
@@ -606,6 +710,8 @@ struct set *netlink_delinearize_set(struct netlink_ctx *ctx,
 		GET_U32_UDATA(automerge, NFTNL_UDATA_SET_MERGE_ELEMENTS);
 
 #undef GET_U32_UDATA
+		typeof_expr_key = set_make_key(ud[NFTNL_UDATA_SET_TYPEOF_KEY]);
+		typeof_expr_data = set_make_key(ud[NFTNL_UDATA_SET_TYPEOF_DATA]);
 	}
 
 	key = nftnl_set_get_u32(nls, NFTNL_SET_KEY_TYPE);
@@ -626,12 +732,14 @@ struct set *netlink_delinearize_set(struct netlink_ctx *ctx,
 			netlink_io_error(ctx, NULL,
 					 "Unknown data type in set key %u",
 					 data);
+			datatype_free(keytype);
 			return NULL;
 		}
 	}
 
 	if (set_is_objmap(flags)) {
 		objtype = nftnl_set_get_u32(nls, NFTNL_SET_OBJ_TYPE);
+		assert(!datatype);
 		datatype = &string_type;
 	}
 
@@ -641,24 +749,53 @@ struct set *netlink_delinearize_set(struct netlink_ctx *ctx,
 	set->handle.set.name = xstrdup(nftnl_set_get_str(nls, NFTNL_SET_NAME));
 	set->automerge	   = automerge;
 
-	set->key     = constant_expr_alloc(&netlink_location,
-					   set_datatype_alloc(keytype, keybyteorder),
-					   keybyteorder,
-					   nftnl_set_get_u32(nls, NFTNL_SET_KEY_LEN) * BITS_PER_BYTE,
-					   NULL);
+	if (datatype) {
+		dtype = set_datatype_alloc(datatype, databyteorder);
+		klen = nftnl_set_get_u32(nls, NFTNL_SET_DATA_LEN) * BITS_PER_BYTE;
+
+		if (set_udata_key_valid(typeof_expr_data, dtype, klen)) {
+			datatype_free(datatype_get(dtype));
+			set->data = typeof_expr_data;
+		} else {
+			expr_free(typeof_expr_data);
+			set->data = constant_expr_alloc(&netlink_location,
+							dtype,
+							databyteorder, klen,
+							NULL);
+
+			/* Can't use 'typeof' keyword, so discard key too */
+			expr_free(typeof_expr_key);
+			typeof_expr_key = NULL;
+		}
+
+		if (dtype != datatype)
+			datatype_free(datatype);
+	}
+
+	dtype = set_datatype_alloc(keytype, keybyteorder);
+	klen = nftnl_set_get_u32(nls, NFTNL_SET_KEY_LEN) * BITS_PER_BYTE;
+
+	if (set_udata_key_valid(typeof_expr_key, dtype, klen)) {
+		datatype_free(datatype_get(dtype));
+		set->key = typeof_expr_key;
+		set->key_typeof_valid = true;
+	} else {
+		expr_free(typeof_expr_key);
+		set->key = constant_expr_alloc(&netlink_location, dtype,
+					       keybyteorder, klen,
+					       NULL);
+	}
+
+	if (dtype != keytype)
+		datatype_free(keytype);
+
+
+
 	set->flags   = nftnl_set_get_u32(nls, NFTNL_SET_FLAGS);
 	set->handle.handle.id = nftnl_set_get_u64(nls, NFTNL_SET_HANDLE);
 
 	set->objtype = objtype;
 
-	set->data = NULL;
-	if (datatype)
-		set->data = constant_expr_alloc(&netlink_location,
-						set_datatype_alloc(datatype, databyteorder),
-						databyteorder,
-						nftnl_set_get_u32(nls, NFTNL_SET_DATA_LEN) * BITS_PER_BYTE,
-						NULL);
-
 	if (nftnl_set_is_set(nls, NFTNL_SET_TIMEOUT))
 		set->timeout = nftnl_set_get_u64(nls, NFTNL_SET_TIMEOUT);
 	if (nftnl_set_is_set(nls, NFTNL_SET_GC_INTERVAL))
diff --git a/src/rule.c b/src/rule.c
index a94860865f31..04b1f57260fd 100644
--- a/src/rule.c
+++ b/src/rule.c
@@ -444,8 +444,32 @@ static void set_print_key(const struct expr *expr, struct output_ctx *octx)
 
 	if (dtype->size || dtype->type == TYPE_VERDICT)
 		nft_print(octx, "%s", dtype->name);
-	else
+	else if (expr->flags & EXPR_F_CONSTANT)
 		nft_print(octx, "%s,%d", dtype->name, expr->len);
+	else
+		expr_print(expr, octx);
+}
+
+static void set_print_key_and_data(const struct set *set, struct output_ctx *octx)
+{
+	bool use_typeof = set->key_typeof_valid;
+
+	nft_print(octx, "%s ", use_typeof ? "typeof" : "type");
+
+	if (use_typeof)
+		expr_print(set->key, octx);
+	else
+		set_print_key(set->key, octx);
+
+	if (set_is_datamap(set->flags)) {
+		nft_print(octx, " : ");
+		if (use_typeof)
+			expr_print(set->data, octx);
+		else
+			set_print_key(set->data, octx);
+	} else if (set_is_objmap(set->flags)) {
+		nft_print(octx, " : %s", obj_type_name(set->objtype));
+	}
 }
 
 static void set_print_declaration(const struct set *set,
@@ -475,17 +499,9 @@ static void set_print_declaration(const struct set *set,
 
 	if (nft_output_handle(octx))
 		nft_print(octx, " # handle %" PRIu64, set->handle.handle.id);
-	nft_print(octx, "%s", opts->nl);
-	nft_print(octx, "%s%stype ",
-		  opts->tab, opts->tab);
-	set_print_key(set->key, octx);
+	nft_print(octx, "%s%s%s", opts->nl, opts->tab, opts->tab);
 
-	if (set_is_datamap(set->flags)) {
-		nft_print(octx, " : ");
-		set_print_key(set->data, octx);
-	} else if (set_is_objmap(set->flags)) {
-		nft_print(octx, " : %s", obj_type_name(set->objtype));
-	}
+	set_print_key_and_data(set, octx);
 
 	nft_print(octx, "%s", opts->stmt_separator);
 
-- 
2.23.0

[PATCH nft v2 07/11] mnl: round up the map data size too

[Florian Westphal]

Same as key: if the size isn't divisible by BITS_PER_BYTE, we need to
round up, not down.

Without this, you can't store vlan ids in a map, as they are truncated
to 8 bit.

Signed-off-by: Florian Westphal <fw@strlen.de>
---
 src/mnl.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mnl.c b/src/mnl.c
index 75113c74c224..bcf633002f15 100644
--- a/src/mnl.c
+++ b/src/mnl.c
@@ -870,7 +870,7 @@ int mnl_nft_set_add(struct netlink_ctx *ctx, const struct cmd *cmd,
 		nftnl_set_set_u32(nls, NFTNL_SET_DATA_TYPE,
 				  dtype_map_to_kernel(set->data->dtype));
 		nftnl_set_set_u32(nls, NFTNL_SET_DATA_LEN,
-				  set->data->len / BITS_PER_BYTE);
+				  div_round_up(set->data->len, BITS_PER_BYTE));
 	}
 	if (set_is_objmap(set->flags))
 		nftnl_set_set_u32(nls, NFTNL_SET_OBJ_TYPE, set->objtype);
-- 
2.23.0

[PATCH nft v2 08/11] src: netlink: remove assertion

[Florian Westphal]

This assert can trigger as follows:

set s {
	type integer,8
	elemets = { 1 }
};
vlan id @s accept

reason is that 'vlan id' will store a 16 bit value into the dreg,
so set should use 'integer,16'.

The kernel won't detect this, as the lookup expression will only
verify that it can load one byte from the given register.

This removes the assertion, in case we hit this condition we can just
return without doing any further actions.

Signed-off-by: Florian Westphal <fw@strlen.de>
---
 src/netlink_delinearize.c | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/src/netlink_delinearize.c b/src/netlink_delinearize.c
index 154353b8161a..6a09bc2013a4 100644
--- a/src/netlink_delinearize.c
+++ b/src/netlink_delinearize.c
@@ -1800,9 +1800,20 @@ static void binop_adjust_one(const struct expr *binop, struct expr *value,
 {
 	struct expr *left = binop->left;
 
-	assert(value->len >= binop->right->len);
-
 	mpz_rshift_ui(value->value, shift);
+
+	/* This will happen when a set has a key that is
+	 * smaller than the amount of bytes loaded by the
+	 * payload/exthdr expression.
+	 *
+	 * This can't happen with normal nft frontend,
+	 * but it can happen with custom clients or with
+	 * nft sets defined via 'type integer,8' and then
+	 * asking "vlan id @myset".
+	 */
+	if (value->len < binop->right->len)
+		return;
+
 	switch (left->etype) {
 	case EXPR_PAYLOAD:
 	case EXPR_EXTHDR:
-- 
2.23.0

[PATCH nft v2 09/11] evaluate: print a hint about 'type,width' syntax on 0 keylen

[Florian Westphal]

If user says

'type integer; ...' in a set definition, don't just throw an error --
provide a hint that a width can be provided via ', bitsize'.

Signed-off-by: Florian Westphal <fw@strlen.de>
---
 src/evaluate.c | 33 ++++++++++++++++++++++++++++-----
 1 file changed, 28 insertions(+), 5 deletions(-)

diff --git a/src/evaluate.c b/src/evaluate.c
index 91d6b254c659..2e93100349d7 100644
--- a/src/evaluate.c
+++ b/src/evaluate.c
@@ -3307,6 +3307,32 @@ static int setelem_evaluate(struct eval_ctx *ctx, struct expr **expr)
 	return 0;
 }
 
+static int set_key_error(struct eval_ctx *ctx, const struct set *set,
+			 const char *type)
+{
+	if (set->key->dtype == &integer_type ||
+	    set->key->dtype == &string_type)
+		return set_error(ctx, set, "unqualified key type "
+				 "specified in %s definition.  Did you mean \"type %s,%d\"?",
+				 type, set->key->dtype->name, 128);
+
+	return set_error(ctx, set, "unqualified key type %s "
+			 "specified in %s definition",
+			 set->key->dtype->name, type);
+}
+
+static int set_datamap_error(struct eval_ctx *ctx, const struct set *set)
+{
+	if (set->data->dtype == &integer_type ||
+	    set->data->dtype == &string_type)
+		return set_error(ctx, set, "unqualified mapping data type "
+				 "specified in map definition. Did you mean \"type %s,%d\"?",
+				 set->data->dtype->name, 128);
+
+	return set_error(ctx, set, "unqualified mapping data "
+			 "type specified in map definition");
+}
+
 static int set_evaluate(struct eval_ctx *ctx, struct set *set)
 {
 	struct table *table;
@@ -3331,9 +3357,7 @@ static int set_evaluate(struct eval_ctx *ctx, struct set *set)
 			return -1;
 
 		if (set->key->len == 0)
-			return set_error(ctx, set, "unqualified key type %s "
-					 "specified in %s definition",
-					 set->key->dtype->name, type);
+			return set_key_error(ctx, set, type);
 	}
 	if (set->flags & NFT_SET_INTERVAL &&
 	    set->key->etype == EXPR_CONCAT)
@@ -3345,8 +3369,7 @@ static int set_evaluate(struct eval_ctx *ctx, struct set *set)
 					 "specify mapping data type");
 
 		if (set->data->len == 0 && set->data->dtype->type != TYPE_VERDICT)
-			return set_error(ctx, set, "unqualified mapping data "
-					 "type specified in map definition");
+			return set_datamap_error(ctx, set);
 	} else if (set_is_objmap(set->flags)) {
 		if (set->data) {
 			assert(set->data->etype == EXPR_VALUE);
-- 
2.23.0

[PATCH nft v2 10/11] doc: mention 'typeof' as alternative to 'type' keyword

[Florian Westphal]

Signed-off-by: Florian Westphal <fw@strlen.de>
---
 doc/nft.txt | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/doc/nft.txt b/doc/nft.txt
index abb9260d3f2f..a5586b74f370 100644
--- a/doc/nft.txt
+++ b/doc/nft.txt
@@ -492,7 +492,7 @@ The sets allowed_hosts and allowed_ports need to be created first. The next
 section describes nft set syntax in more detail.
 
 [verse]
-*add set* ['family'] 'table' 'set' *{ type* 'type' *;* [*flags* 'flags' *;*] [*timeout* 'timeout' *;*] [*gc-interval* 'gc-interval' *;*] [*elements = {* 'element'[*,* ...] *} ;*] [*size* 'size' *;*] [*policy* 'policy' *;*] [*auto-merge ;*] *}*
+*add set* ['family'] 'table' 'set' *{ type* 'type' | *typeof* 'expression' *;* [*flags* 'flags' *;*] [*timeout* 'timeout' *;*] [*gc-interval* 'gc-interval' *;*] [*elements = {* 'element'[*,* ...] *} ;*] [*size* 'size' *;*] [*policy* 'policy' *;*] [*auto-merge ;*] *}*
 {*delete* | *list* | *flush*} *set* ['family'] 'table' 'set'
 *list sets* ['family']
 *delete set* ['family'] 'table' *handle* 'handle'
@@ -516,7 +516,10 @@ be tuned with the flags that can be specified at set creation time.
 |Keyword | Description | Type
 |type |
 data type of set elements |
-string: ipv4_addr, ipv6_addr, ether_addr, inet_proto, inet_service, mark
+string: ipv4_addr, ipv6_addr, ether_addr, inet_proto, inet_service, mark, string, integer
+|typeof |
+data type of set element |
+expression to derive the data type from
 |flags |
 set flags |
 string: constant, dynamic, interval, timeout
@@ -544,7 +547,7 @@ automatic merge of adjacent/overlapping set elements (only for interval sets) |
 MAPS
 -----
 [verse]
-*add map* ['family'] 'table' 'map' *{ type* 'type' [*flags* 'flags' *;*] [*elements = {* 'element'[*,* ...] *} ;*] [*size* 'size' *;*] [*policy* 'policy' *;*] *}*
+*add map* ['family'] 'table' 'map' *{ type* 'type' | *typeof* 'expression' [*flags* 'flags' *;*] [*elements = {* 'element'[*,* ...] *} ;*] [*size* 'size' *;*] [*policy* 'policy' *;*] *}*
 {*delete* | *list* | *flush*} *map* ['family'] 'table' 'map'
 *list maps* ['family']
 {*add* | *delete*} *element* ['family'] 'table' 'map' *{ elements = {* 'element'[*,* ...] *} ; }*
@@ -565,7 +568,10 @@ Maps store data based on some specific key used as input. They are uniquely iden
 |Keyword | Description | Type
 |type |
 data type of map elements |
-string `:' string: ipv4_addr, ipv6_addr, ether_addr, inet_proto, inet_service, mark, counter, quota. Counter and quota can't be used as keys
+string: ipv4_addr, ipv6_addr, ether_addr, inet_proto, inet_service, mark, string, integer, counter, quota. Counter and quota can't be used as keys
+|typeof |
+data type of set element |
+expression to derive the data type from
 |flags |
 map flags |
 string: constant, interval
-- 
2.23.0

[PATCH nft v2 11/11] tests: add typeof test cases

[Florian Westphal]

Signed-off-by: Florian Westphal <fw@strlen.de>
---
 .../testcases/maps/dumps/typeof_maps_0.nft    | 16 ++++++++
 tests/shell/testcases/maps/typeof_maps_0      | 26 ++++++++++++
 .../testcases/sets/dumps/typeof_sets_0.nft    | 31 ++++++++++++++
 tests/shell/testcases/sets/typeof_sets_0      | 40 +++++++++++++++++++
 4 files changed, 113 insertions(+)
 create mode 100644 tests/shell/testcases/maps/dumps/typeof_maps_0.nft
 create mode 100755 tests/shell/testcases/maps/typeof_maps_0
 create mode 100644 tests/shell/testcases/sets/dumps/typeof_sets_0.nft
 create mode 100755 tests/shell/testcases/sets/typeof_sets_0

diff --git a/tests/shell/testcases/maps/dumps/typeof_maps_0.nft b/tests/shell/testcases/maps/dumps/typeof_maps_0.nft
new file mode 100644
index 000000000000..5c6a4d1f8662
--- /dev/null
+++ b/tests/shell/testcases/maps/dumps/typeof_maps_0.nft
@@ -0,0 +1,16 @@
+table inet t {
+	map m1 {
+		typeof osf name : ct mark
+		elements = { "Linux" : 0x00000001 }
+	}
+
+	map m2 {
+		type string,128 : mark
+		elements = { "Linux" : 0x00000001 }
+	}
+
+	chain c {
+		ct mark set osf name map @m1
+		ct mark set osf name map @m2
+	}
+}
diff --git a/tests/shell/testcases/maps/typeof_maps_0 b/tests/shell/testcases/maps/typeof_maps_0
new file mode 100755
index 000000000000..16b54b7adeb3
--- /dev/null
+++ b/tests/shell/testcases/maps/typeof_maps_0
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# support for strings/typeof in named maps.
+# m1 and m2 are identical, they just use different
+# ways for declaration.
+
+EXPECTED="table inet t {
+	map m1 {
+		typeof osf name : ct mark
+		elements = { \"Linux\" : 0x1 }
+	}
+
+	map m2 {
+		type string, 128 : mark
+		elements = { \"Linux\" : 0x1 }
+	}
+
+	chain c {
+		ct mark set osf name map @m1
+		ct mark set osf name map @m2
+	}
+}"
+
+set -e
+$NFT -f - <<< $EXPECTED
+
diff --git a/tests/shell/testcases/sets/dumps/typeof_sets_0.nft b/tests/shell/testcases/sets/dumps/typeof_sets_0.nft
new file mode 100644
index 000000000000..eeeeb844e0e1
--- /dev/null
+++ b/tests/shell/testcases/sets/dumps/typeof_sets_0.nft
@@ -0,0 +1,31 @@
+table inet t {
+	set s1 {
+		typeof osf name
+		elements = { "Linux" }
+	}
+
+	set s2 {
+		type string,128
+		elements = { "Linux" }
+	}
+
+	set s3 {
+		typeof vlan id
+		elements = { 2, 3, 103 }
+	}
+
+	set s4 {
+		type integer,16
+		elements = { 2, 3, 103, 2003 }
+	}
+
+	chain c1 {
+		osf name @s1 accept
+		osf name @s2 accept
+	}
+
+	chain c2 {
+		vlan id @s3 accept
+		vlan id @s4 accept
+	}
+}
diff --git a/tests/shell/testcases/sets/typeof_sets_0 b/tests/shell/testcases/sets/typeof_sets_0
new file mode 100755
index 000000000000..bbdeb4743ee7
--- /dev/null
+++ b/tests/shell/testcases/sets/typeof_sets_0
@@ -0,0 +1,40 @@
+#!/bin/bash
+
+# support for strings/typeof in named sets.
+# s1 and s2 are identical, they just use different
+# ways for declaration.
+
+EXPECTED="table inet t {
+	set s1 {
+		typeof osf name
+		elements = { \"Linux\" }
+	}
+
+	set s2 {
+		type string, 128
+		elements = { \"Linux\" }
+	}
+
+	set s3 {
+		typeof vlan id
+		elements = { 2, 3, 103 }
+	}
+
+	set s4 {
+		type integer,16
+		elements = { 2, 3, 103, 2003 }
+	}
+	chain c1 {
+		osf name @s1 accept
+		osf name @s2 accept
+	}
+
+	chain c2 {
+		ether type vlan vlan id @s3 accept
+		ether type vlan vlan id @s4 accept
+	}
+}"
+
+set -e
+$NFT -f - <<< $EXPECTED
+
-- 
2.23.0