[PATCH net-next 0/3] net: Allow non-fatal messages to be passed in extack

[PATCH net-next 0/3] net: Allow non-fatal messages to be passed in extack

[David Ahern]

There are many cases where networking subsystems throw non-fatal warning
messages that end up in dmesg / kernel log to which a user making the
change is completely oblivious. This set makes the extack facility
usable for returning such messages. The case in point here is spectrum
and adding FIB rules which causes an offload abort. Make the use case
more user friendly by letting the user know that offload is no longer
happening.

David Ahern (3):
  netlink: Allow ext_ack to carry non-error messages
  net: Add extack to fib_notifier_info
  mlxsw: spectrum_router: Return extack message on abort due to fib
    rules

 .../net/ethernet/mellanox/mlxsw/spectrum_router.c  | 97 ++++++++++++----------
 include/net/fib_notifier.h                         |  1 +
 net/core/fib_rules.c                               |  9 +-
 net/ipv4/fib_trie.c                                | 13 +--
 net/ipv6/ip6_fib.c                                 | 15 ++--
 net/netlink/af_netlink.c                           | 18 ++--
 6 files changed, 85 insertions(+), 68 deletions(-)

-- 
2.1.4

[PATCH net-next 1/3] netlink: Allow ext_ack to carry non-error messages

[David Ahern]

The NLMSGERR API already carries data (eg, a cookie) on the success path.
Allow a message string to be returned as well.

Signed-off-by: David Ahern <dsahern@gmail.com>
---
 net/netlink/af_netlink.c | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/net/netlink/af_netlink.c b/net/netlink/af_netlink.c
index 767c84e10e20..26ded4239611 100644
--- a/net/netlink/af_netlink.c
+++ b/net/netlink/af_netlink.c
@@ -2313,17 +2313,16 @@ void netlink_ack(struct sk_buff *in_skb, struct nlmsghdr *nlh, int err,
 	 * requests to cap the error message, and get extra error data if
 	 * requested.
 	 */
+	if (nlk_has_extack && extack && extack->_msg)
+		tlvlen += nla_total_size(strlen(extack->_msg) + 1);
+
 	if (err) {
 		if (!(nlk->flags & NETLINK_F_CAP_ACK))
 			payload += nlmsg_len(nlh);
 		else
 			flags |= NLM_F_CAPPED;
-		if (nlk_has_extack && extack) {
-			if (extack->_msg)
-				tlvlen += nla_total_size(strlen(extack->_msg) + 1);
-			if (extack->bad_attr)
-				tlvlen += nla_total_size(sizeof(u32));
-		}
+		if (nlk_has_extack && extack && extack->bad_attr)
+			tlvlen += nla_total_size(sizeof(u32));
 	} else {
 		flags |= NLM_F_CAPPED;
 
@@ -2348,10 +2347,11 @@ void netlink_ack(struct sk_buff *in_skb, struct nlmsghdr *nlh, int err,
 	memcpy(&errmsg->msg, nlh, payload > sizeof(*errmsg) ? nlh->nlmsg_len : sizeof(*nlh));
 
 	if (nlk_has_extack && extack) {
+		if (extack->_msg) {
+			WARN_ON(nla_put_string(skb, NLMSGERR_ATTR_MSG,
+					       extack->_msg));
+		}
 		if (err) {
-			if (extack->_msg)
-				WARN_ON(nla_put_string(skb, NLMSGERR_ATTR_MSG,
-						       extack->_msg));
 			if (extack->bad_attr &&
 			    !WARN_ON((u8 *)extack->bad_attr < in_skb->data ||
 				     (u8 *)extack->bad_attr >= in_skb->data +
-- 
2.1.4

[PATCH net-next 2/3] net: Add extack to fib_notifier_info

[David Ahern]

Add extack to fib_notifier_info and plumb through stack to
call_fib_rule_notifiers, call_fib_entry_notifiers and
call_fib6_entry_notifiers. This allows notifer handlers to
return messages to user.

Signed-off-by: David Ahern <dsahern@gmail.com>
---
 include/net/fib_notifier.h |  1 +
 net/core/fib_rules.c       |  9 ++++++---
 net/ipv4/fib_trie.c        | 13 ++++++++-----
 net/ipv6/ip6_fib.c         | 15 +++++++++------
 4 files changed, 24 insertions(+), 14 deletions(-)

diff --git a/include/net/fib_notifier.h b/include/net/fib_notifier.h
index 54cd6b839d2f..c91ec732afd6 100644
--- a/include/net/fib_notifier.h
+++ b/include/net/fib_notifier.h
@@ -9,6 +9,7 @@
 struct fib_notifier_info {
 	struct net *net;
 	int family;
+	struct netlink_ext_ack  *extack;
 };
 
 enum fib_event_type {
diff --git a/net/core/fib_rules.c b/net/core/fib_rules.c
index 9a6d97c1d810..fafd0a41e3f7 100644
--- a/net/core/fib_rules.c
+++ b/net/core/fib_rules.c
@@ -314,10 +314,12 @@ static int call_fib_rule_notifier(struct notifier_block *nb, struct net *net,
 static int call_fib_rule_notifiers(struct net *net,
 				   enum fib_event_type event_type,
 				   struct fib_rule *rule,
-				   struct fib_rules_ops *ops)
+				   struct fib_rules_ops *ops,
+				   struct netlink_ext_ack *extack)
 {
 	struct fib_rule_notifier_info info = {
 		.info.family = ops->family,
+		.info.extack = extack,
 		.rule = rule,
 	};
 
@@ -609,7 +611,7 @@ int fib_nl_newrule(struct sk_buff *skb, struct nlmsghdr *nlh,
 	if (rule->tun_id)
 		ip_tunnel_need_metadata();
 
-	call_fib_rule_notifiers(net, FIB_EVENT_RULE_ADD, rule, ops);
+	call_fib_rule_notifiers(net, FIB_EVENT_RULE_ADD, rule, ops, extack);
 	notify_rule_change(RTM_NEWRULE, rule, ops, nlh, NETLINK_CB(skb).portid);
 	flush_route_cache(ops);
 	rules_ops_put(ops);
@@ -749,7 +751,8 @@ int fib_nl_delrule(struct sk_buff *skb, struct nlmsghdr *nlh,
 			}
 		}
 
-		call_fib_rule_notifiers(net, FIB_EVENT_RULE_DEL, rule, ops);
+		call_fib_rule_notifiers(net, FIB_EVENT_RULE_DEL, rule, ops,
+					NULL);
 		notify_rule_change(RTM_DELRULE, rule, ops, nlh,
 				   NETLINK_CB(skb).portid);
 		fib_rule_put(rule);
diff --git a/net/ipv4/fib_trie.c b/net/ipv4/fib_trie.c
index aaa1ba09afaa..5ddc4aefff12 100644
--- a/net/ipv4/fib_trie.c
+++ b/net/ipv4/fib_trie.c
@@ -102,9 +102,11 @@ static int call_fib_entry_notifier(struct notifier_block *nb, struct net *net,
 
 static int call_fib_entry_notifiers(struct net *net,
 				    enum fib_event_type event_type, u32 dst,
-				    int dst_len, struct fib_alias *fa)
+				    int dst_len, struct fib_alias *fa,
+				    struct netlink_ext_ack *extack)
 {
 	struct fib_entry_notifier_info info = {
+		.info.extack = extack,
 		.dst = dst,
 		.dst_len = dst_len,
 		.fi = fa->fa_info,
@@ -1214,7 +1216,7 @@ int fib_table_insert(struct net *net, struct fib_table *tb,
 			new_fa->fa_default = -1;
 
 			call_fib_entry_notifiers(net, FIB_EVENT_ENTRY_REPLACE,
-						 key, plen, new_fa);
+						 key, plen, new_fa, extack);
 			rtmsg_fib(RTM_NEWROUTE, htonl(key), new_fa, plen,
 				  tb->tb_id, &cfg->fc_nlinfo, nlflags);
 
@@ -1269,7 +1271,7 @@ int fib_table_insert(struct net *net, struct fib_table *tb,
 		tb->tb_num_default++;
 
 	rt_cache_flush(cfg->fc_nlinfo.nl_net);
-	call_fib_entry_notifiers(net, event, key, plen, new_fa);
+	call_fib_entry_notifiers(net, event, key, plen, new_fa, extack);
 	rtmsg_fib(RTM_NEWROUTE, htonl(key), new_fa, plen, new_fa->tb_id,
 		  &cfg->fc_nlinfo, nlflags);
 succeeded:
@@ -1569,7 +1571,7 @@ int fib_table_delete(struct net *net, struct fib_table *tb,
 		return -ESRCH;
 
 	call_fib_entry_notifiers(net, FIB_EVENT_ENTRY_DEL, key, plen,
-				 fa_to_delete);
+				 fa_to_delete, extack);
 	rtmsg_fib(RTM_DELROUTE, htonl(key), fa_to_delete, plen, tb->tb_id,
 		  &cfg->fc_nlinfo, 0);
 
@@ -1886,7 +1888,8 @@ int fib_table_flush(struct net *net, struct fib_table *tb)
 
 			call_fib_entry_notifiers(net, FIB_EVENT_ENTRY_DEL,
 						 n->key,
-						 KEYLENGTH - fa->fa_slen, fa);
+						 KEYLENGTH - fa->fa_slen, fa,
+						 NULL);
 			hlist_del_rcu(&fa->fa_list);
 			fib_release_info(fa->fa_info);
 			alias_free_mem_rcu(fa);
diff --git a/net/ipv6/ip6_fib.c b/net/ipv6/ip6_fib.c
index 1ada9672d198..2e2804f5823e 100644
--- a/net/ipv6/ip6_fib.c
+++ b/net/ipv6/ip6_fib.c
@@ -353,9 +353,11 @@ static int call_fib6_entry_notifier(struct notifier_block *nb, struct net *net,
 
 static int call_fib6_entry_notifiers(struct net *net,
 				     enum fib_event_type event_type,
-				     struct rt6_info *rt)
+				     struct rt6_info *rt,
+				     struct netlink_ext_ack *extack)
 {
 	struct fib6_entry_notifier_info info = {
+		.info.extack = extack,
 		.rt = rt,
 	};
 
@@ -868,7 +870,8 @@ static void fib6_purge_rt(struct rt6_info *rt, struct fib6_node *fn,
  */
 
 static int fib6_add_rt2node(struct fib6_node *fn, struct rt6_info *rt,
-			    struct nl_info *info, struct mx6_config *mxc)
+			    struct nl_info *info, struct mx6_config *mxc,
+			    struct netlink_ext_ack *extack)
 {
 	struct rt6_info *leaf = rcu_dereference_protected(fn->leaf,
 				    lockdep_is_held(&rt->rt6i_table->tb6_lock));
@@ -1011,7 +1014,7 @@ static int fib6_add_rt2node(struct fib6_node *fn, struct rt6_info *rt,
 		rcu_assign_pointer(rt->rt6i_node, fn);
 		rcu_assign_pointer(*ins, rt);
 		call_fib6_entry_notifiers(info->nl_net, FIB_EVENT_ENTRY_ADD,
-					  rt);
+					  rt, extack);
 		if (!info->skip_notify)
 			inet6_rt_notify(RTM_NEWROUTE, rt, info, nlflags);
 		info->nl_net->ipv6.rt6_stats->fib_rt_entries++;
@@ -1040,7 +1043,7 @@ static int fib6_add_rt2node(struct fib6_node *fn, struct rt6_info *rt,
 		rt->dst.rt6_next = iter->dst.rt6_next;
 		rcu_assign_pointer(*ins, rt);
 		call_fib6_entry_notifiers(info->nl_net, FIB_EVENT_ENTRY_REPLACE,
-					  rt);
+					  rt, extack);
 		if (!info->skip_notify)
 			inet6_rt_notify(RTM_NEWROUTE, rt, info, NLM_F_REPLACE);
 		if (!(fn->fn_flags & RTN_RTINFO)) {
@@ -1225,7 +1228,7 @@ int fib6_add(struct fib6_node *root, struct rt6_info *rt,
 	}
 #endif
 
-	err = fib6_add_rt2node(fn, rt, info, mxc);
+	err = fib6_add_rt2node(fn, rt, info, mxc, extack);
 	if (!err) {
 		fib6_update_sernum_upto_root(rt, sernum);
 		fib6_start_gc(info->nl_net, rt);
@@ -1686,7 +1689,7 @@ static void fib6_del_route(struct fib6_table *table, struct fib6_node *fn,
 
 	fib6_purge_rt(rt, fn, net);
 
-	call_fib6_entry_notifiers(net, FIB_EVENT_ENTRY_DEL, rt);
+	call_fib6_entry_notifiers(net, FIB_EVENT_ENTRY_DEL, rt, NULL);
 	if (!info->skip_notify)
 		inet6_rt_notify(RTM_DELROUTE, rt, info, 0);
 	rt6_release(rt);
-- 
2.1.4

[PATCH net-next 3/3] mlxsw: spectrum_router: Return extack message on abort due to fib rules

[David Ahern]

Adding a FIB rule on a spectrum platform silently aborts FIB offload:
    $ ip ru add pref 99 from all to 192.168.1.1 table 10
    $ dmesg -c
    [  623.144736] mlxsw_spectrum 0000:03:00.0: FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.

This patch reworks FIB rule handling to return a message to the user:
    $ ip ru add pref 99 from all to 8.8.8.8 table 11
    Error: spectrum: FIB rules not supported. Aborting offload.

spectrum currently only checks whether the fib rule is a default rule or
an l3mdev rule, both of which it knows how to handle. Any other it aborts
FIB offload. Since the processing is fairly quick, move the code to inline
with the user request rather than a work queue to allow a message to be
returned if the offload is aborted. Change the delete handling to just return
since it does nothing at the moment.

Signed-off-by: David Ahern <dsahern@gmail.com>
---
 .../net/ethernet/mellanox/mlxsw/spectrum_router.c  | 97 ++++++++++++----------
 1 file changed, 52 insertions(+), 45 deletions(-)

diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
index 27b632cac991..d2e3360b7283 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
@@ -5234,7 +5234,6 @@ static void mlxsw_sp_router_fib4_event_work(struct work_struct *work)
 	struct mlxsw_sp_fib_event_work *fib_work =
 		container_of(work, struct mlxsw_sp_fib_event_work, work);
 	struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
-	struct fib_rule *rule;
 	bool replace, append;
 	int err;
 
@@ -5256,13 +5255,6 @@ static void mlxsw_sp_router_fib4_event_work(struct work_struct *work)
 		mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info);
 		fib_info_put(fib_work->fen_info.fi);
 		break;
-	case FIB_EVENT_RULE_ADD: /* fall through */
-	case FIB_EVENT_RULE_DEL:
-		rule = fib_work->fr_info.rule;
-		if (!fib4_rule_default(rule) && !rule->l3mdev)
-			mlxsw_sp_router_fib_abort(mlxsw_sp);
-		fib_rule_put(rule);
-		break;
 	case FIB_EVENT_NH_ADD: /* fall through */
 	case FIB_EVENT_NH_DEL:
 		mlxsw_sp_nexthop4_event(mlxsw_sp, fib_work->event,
@@ -5279,7 +5271,6 @@ static void mlxsw_sp_router_fib6_event_work(struct work_struct *work)
 	struct mlxsw_sp_fib_event_work *fib_work =
 		container_of(work, struct mlxsw_sp_fib_event_work, work);
 	struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
-	struct fib_rule *rule;
 	bool replace;
 	int err;
 
@@ -5298,13 +5289,6 @@ static void mlxsw_sp_router_fib6_event_work(struct work_struct *work)
 		mlxsw_sp_router_fib6_del(mlxsw_sp, fib_work->fen6_info.rt);
 		mlxsw_sp_rt6_release(fib_work->fen6_info.rt);
 		break;
-	case FIB_EVENT_RULE_ADD: /* fall through */
-	case FIB_EVENT_RULE_DEL:
-		rule = fib_work->fr_info.rule;
-		if (!fib6_rule_default(rule) && !rule->l3mdev)
-			mlxsw_sp_router_fib_abort(mlxsw_sp);
-		fib_rule_put(rule);
-		break;
 	}
 	rtnl_unlock();
 	kfree(fib_work);
@@ -5315,7 +5299,6 @@ static void mlxsw_sp_router_fibmr_event_work(struct work_struct *work)
 	struct mlxsw_sp_fib_event_work *fib_work =
 		container_of(work, struct mlxsw_sp_fib_event_work, work);
 	struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
-	struct fib_rule *rule;
 	bool replace;
 	int err;
 
@@ -5347,13 +5330,6 @@ static void mlxsw_sp_router_fibmr_event_work(struct work_struct *work)
 					      &fib_work->ven_info);
 		dev_put(fib_work->ven_info.dev);
 		break;
-	case FIB_EVENT_RULE_ADD: /* fall through */
-	case FIB_EVENT_RULE_DEL:
-		rule = fib_work->fr_info.rule;
-		if (!ipmr_rule_default(rule) && !rule->l3mdev)
-			mlxsw_sp_router_fib_abort(mlxsw_sp);
-		fib_rule_put(rule);
-		break;
 	}
 	rtnl_unlock();
 	kfree(fib_work);
@@ -5363,7 +5339,6 @@ static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event_work *fib_work,
 				       struct fib_notifier_info *info)
 {
 	struct fib_entry_notifier_info *fen_info;
-	struct fib_rule_notifier_info *fr_info;
 	struct fib_nh_notifier_info *fnh_info;
 
 	switch (fib_work->event) {
@@ -5379,13 +5354,6 @@ static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event_work *fib_work,
 		 */
 		fib_info_hold(fib_work->fen_info.fi);
 		break;
-	case FIB_EVENT_RULE_ADD: /* fall through */
-	case FIB_EVENT_RULE_DEL:
-		fr_info = container_of(info, struct fib_rule_notifier_info,
-				       info);
-		fib_work->fr_info = *fr_info;
-		fib_rule_get(fib_work->fr_info.rule);
-		break;
 	case FIB_EVENT_NH_ADD: /* fall through */
 	case FIB_EVENT_NH_DEL:
 		fnh_info = container_of(info, struct fib_nh_notifier_info,
@@ -5400,7 +5368,6 @@ static void mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event_work *fib_work,
 				       struct fib_notifier_info *info)
 {
 	struct fib6_entry_notifier_info *fen6_info;
-	struct fib_rule_notifier_info *fr_info;
 
 	switch (fib_work->event) {
 	case FIB_EVENT_ENTRY_REPLACE: /* fall through */
@@ -5411,13 +5378,6 @@ static void mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event_work *fib_work,
 		fib_work->fen6_info = *fen6_info;
 		rt6_hold(fib_work->fen6_info.rt);
 		break;
-	case FIB_EVENT_RULE_ADD: /* fall through */
-	case FIB_EVENT_RULE_DEL:
-		fr_info = container_of(info, struct fib_rule_notifier_info,
-				       info);
-		fib_work->fr_info = *fr_info;
-		fib_rule_get(fib_work->fr_info.rule);
-		break;
 	}
 }
 
@@ -5437,12 +5397,51 @@ mlxsw_sp_router_fibmr_event(struct mlxsw_sp_fib_event_work *fib_work,
 		memcpy(&fib_work->ven_info, info, sizeof(fib_work->ven_info));
 		dev_hold(fib_work->ven_info.dev);
 		break;
-	case FIB_EVENT_RULE_ADD: /* fall through */
-	case FIB_EVENT_RULE_DEL:
-		memcpy(&fib_work->fr_info, info, sizeof(fib_work->fr_info));
-		fib_rule_get(fib_work->fr_info.rule);
+	}
+}
+
+static int mlxsw_sp_router_fib_rule_event(unsigned long event,
+					  struct fib_notifier_info *info,
+					  struct mlxsw_sp *mlxsw_sp)
+{
+	struct netlink_ext_ack *extack = info->extack;
+	struct fib_rule_notifier_info *fr_info;
+	struct fib_rule *rule;
+	bool add_unsupported_msg = false;
+
+	/* nothing to do at the moment */
+	if (event == FIB_EVENT_RULE_DEL)
+		goto out;
+
+	fr_info = container_of(info, struct fib_rule_notifier_info, info);
+	rule = fr_info->rule;
+
+	switch (info->family) {
+	case AF_INET:
+		if (!fib4_rule_default(rule) && !rule->l3mdev) {
+			add_unsupported_msg = true;
+			mlxsw_sp_router_fib_abort(mlxsw_sp);
+		}
+		break;
+	case AF_INET6:
+		if (!fib6_rule_default(rule) && !rule->l3mdev) {
+			add_unsupported_msg = true;
+			mlxsw_sp_router_fib_abort(mlxsw_sp);
+		}
+		break;
+	case RTNL_FAMILY_IPMR:
+		if (!ipmr_rule_default(rule) && !rule->l3mdev) {
+			add_unsupported_msg = true;
+			mlxsw_sp_router_fib_abort(mlxsw_sp);
+		}
 		break;
 	}
+
+	if (add_unsupported_msg)
+		NL_SET_ERR_MSG(extack, "spectrum: FIB rules not supported. Aborting offload");
+
+out:
+	return NOTIFY_DONE;
 }
 
 /* Called with rcu_read_lock() */
@@ -5458,11 +5457,19 @@ static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
 	     info->family != RTNL_FAMILY_IPMR))
 		return NOTIFY_DONE;
 
+	router = container_of(nb, struct mlxsw_sp_router, fib_nb);
+
+	switch (event) {
+	case FIB_EVENT_RULE_ADD: /* fall through */
+	case FIB_EVENT_RULE_DEL:
+		return mlxsw_sp_router_fib_rule_event(event, info,
+						      router->mlxsw_sp);
+	}
+
 	fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC);
 	if (WARN_ON(!fib_work))
 		return NOTIFY_BAD;
 
-	router = container_of(nb, struct mlxsw_sp_router, fib_nb);
 	fib_work->mlxsw_sp = router->mlxsw_sp;
 	fib_work->event = event;
 
-- 
2.1.4

Re: [PATCH net-next 3/3] mlxsw: spectrum_router: Return extack message on abort due to fib rules

[Jiri Pirko]

Thu, Oct 26, 2017 at 07:08:05AM CEST, dsahern@gmail.com wrote:
>Adding a FIB rule on a spectrum platform silently aborts FIB offload:
>    $ ip ru add pref 99 from all to 192.168.1.1 table 10
>    $ dmesg -c
>    [  623.144736] mlxsw_spectrum 0000:03:00.0: FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.
>
>This patch reworks FIB rule handling to return a message to the user:
>    $ ip ru add pref 99 from all to 8.8.8.8 table 11
>    Error: spectrum: FIB rules not supported. Aborting offload.
>
>spectrum currently only checks whether the fib rule is a default rule or
>an l3mdev rule, both of which it knows how to handle. Any other it aborts
>FIB offload. Since the processing is fairly quick, move the code to inline
>with the user request rather than a work queue to allow a message to be
>returned if the offload is aborted. Change the delete handling to just return
>since it does nothing at the moment.
>
>Signed-off-by: David Ahern <dsahern@gmail.com>
>---

[...]

>+static int mlxsw_sp_router_fib_rule_event(unsigned long event,
>+					  struct fib_notifier_info *info,
>+					  struct mlxsw_sp *mlxsw_sp)
>+{
>+	struct netlink_ext_ack *extack = info->extack;
>+	struct fib_rule_notifier_info *fr_info;
>+	struct fib_rule *rule;
>+	bool add_unsupported_msg = false;
>+
>+	/* nothing to do at the moment */
>+	if (event == FIB_EVENT_RULE_DEL)
>+		goto out;
>+
>+	fr_info = container_of(info, struct fib_rule_notifier_info, info);
>+	rule = fr_info->rule;
>+
>+	switch (info->family) {
>+	case AF_INET:
>+		if (!fib4_rule_default(rule) && !rule->l3mdev) {

Why don't we abort on removal of default rule?



>+			add_unsupported_msg = true;
>+			mlxsw_sp_router_fib_abort(mlxsw_sp);
>+		}
>+		break;
>+	case AF_INET6:
>+		if (!fib6_rule_default(rule) && !rule->l3mdev) {
>+			add_unsupported_msg = true;
>+			mlxsw_sp_router_fib_abort(mlxsw_sp);
>+		}
>+		break;
>+	case RTNL_FAMILY_IPMR:
>+		if (!ipmr_rule_default(rule) && !rule->l3mdev) {
>+			add_unsupported_msg = true;
>+			mlxsw_sp_router_fib_abort(mlxsw_sp);
>+		}
> 		break;

Re: [PATCH net-next 3/3] mlxsw: spectrum_router: Return extack message on abort due to fib rules

[Ido Schimmel]

Hi David,

On Wed, Oct 25, 2017 at 10:08:05PM -0700, David Ahern wrote:
> Adding a FIB rule on a spectrum platform silently aborts FIB offload:
>     $ ip ru add pref 99 from all to 192.168.1.1 table 10
>     $ dmesg -c
>     [  623.144736] mlxsw_spectrum 0000:03:00.0: FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.
> 
> This patch reworks FIB rule handling to return a message to the user:
>     $ ip ru add pref 99 from all to 8.8.8.8 table 11
>     Error: spectrum: FIB rules not supported. Aborting offload.
> 
> spectrum currently only checks whether the fib rule is a default rule or
> an l3mdev rule, both of which it knows how to handle. Any other it aborts
> FIB offload. Since the processing is fairly quick, move the code to inline
> with the user request rather than a work queue to allow a message to be
> returned if the offload is aborted. Change the delete handling to just return
> since it does nothing at the moment.

Nice idea, but one problem is that the FIB notifier is atomic and thus
when you trigger abort you end up sleeping in an atomic context:

[  178.933902] mlxsw_spectrum 0000:01:00.0: FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.
[  178.946983] BUG: sleeping function called from invalid context at mm/slab.h:420
[  178.955199] in_atomic(): 0, irqs_disabled(): 0, pid: 3215, name: ip
[  178.962244] INFO: lockdep is turned off.
[  178.966666] Preemption disabled at:
[  178.966685] [<ffffffff811bb603>] wake_up_klogd+0x13/0xc0
[  178.976594] CPU: 0 PID: 3215 Comm: ip Not tainted 4.14.0-rc5-rules-extack-custom #688
[  178.985353] Hardware name: Mellanox Technologies Ltd. MSN2100-CB2FO/SA001017, BIOS 5.6.5 06/07/2016
[  178.995469] Call Trace:
[  178.998214]  dump_stack+0xb1/0x10c
[  179.002026]  ? _atomic_dec_and_lock+0x124/0x124
[  179.007102]  ? debug_show_held_locks+0x14/0x40
[  179.012083]  ___might_sleep+0x32d/0x420
[  179.016383]  __might_sleep+0x4c/0x1a0
[  179.020508]  ? mlxsw_core_reg_access_emad+0xcf/0x1490 [mlxsw_core]
[  179.027449]  ? mlxsw_core_reg_access_emad+0xcf/0x1490 [mlxsw_core]
[  179.034369]  kmem_cache_alloc_trace+0x50/0x4d0
[  179.039344]  ? _raw_spin_unlock_irqrestore+0x3d/0x60
[  179.044931]  mlxsw_core_reg_access_emad+0xcf/0x1490 [mlxsw_core]
[  179.051654]  ? up+0x14/0x80
[  179.054807]  ? mlxsw_core_schedule_dw+0x20/0x20 [mlxsw_core]
[  179.061171]  ? mlxsw_emad_pack_op_tlv+0x1080/0x1080 [mlxsw_core]
[  179.067897]  ? preempt_count_sub+0x13/0xd0
[  179.072476]  ? irq_work_queue+0x141/0x1d0
[  179.076990]  mlxsw_core_reg_access+0xf0/0x8f0 [mlxsw_core]
[  179.083135]  ? console_unlock+0x697/0x7d0
[  179.087650]  ? mlxsw_reg_trans_write+0x20/0x20 [mlxsw_core]
[  179.093890]  ? vprintk_emit+0x208/0x380
[  179.098252]  ? __mlxsw_item_set32.constprop.48+0xf3/0x240 [mlxsw_spectrum]
[  179.105973]  mlxsw_reg_write+0xe/0x10 [mlxsw_core]
[  179.111400]  __mlxsw_sp_fib_entry_op+0x1ad/0x540 [mlxsw_spectrum]
[  179.118288]  ? mlxsw_sp_fib_entry_ralue_pack+0x2b0/0x2b0 [mlxsw_spectrum]
[  179.125893]  ? note_gp_changes+0x210/0x210
[  179.130482]  ? fib_trie_get_first+0x170/0x170
[  179.135420]  mlxsw_sp_fib_node_entry_del+0x97/0xe0 [mlxsw_spectrum]
[  179.142501]  mlxsw_sp_fib4_node_entry_unlink+0x6a/0x230 [mlxsw_spectrum]
[  179.150066]  ? mlxsw_sp_fib_node_entry_del+0xe0/0xe0 [mlxsw_spectrum]
[  179.157277]  ? __lock_acquire+0xf0/0x2700
[  179.161840]  mlxsw_sp_vr_fib_flush+0x30e/0x4e0 [mlxsw_spectrum]
[  179.168537]  ? mlxsw_sp_fib6_entry_destroy+0x4c0/0x4c0 [mlxsw_spectrum]
[  179.176026]  mlxsw_sp_router_fib_flush+0x9e/0x1c0 [mlxsw_spectrum]
[  179.183026]  mlxsw_sp_router_fib_abort+0x9b/0x1b0 [mlxsw_spectrum]
[  179.190015]  mlxsw_sp_router_fib_event+0x25c/0x790 [mlxsw_spectrum]
...

How about we keep the checking and error reporting inline, but do the
actual abort in a workqueue?

Thanks

Re: [PATCH net-next 3/3] mlxsw: spectrum_router: Return extack message on abort due to fib rules

[Ido Schimmel]

On Thu, Oct 26, 2017 at 08:26:16AM +0200, Jiri Pirko wrote:
> Thu, Oct 26, 2017 at 07:08:05AM CEST, dsahern@gmail.com wrote:
> >Adding a FIB rule on a spectrum platform silently aborts FIB offload:
> >    $ ip ru add pref 99 from all to 192.168.1.1 table 10
> >    $ dmesg -c
> >    [  623.144736] mlxsw_spectrum 0000:03:00.0: FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.
> >
> >This patch reworks FIB rule handling to return a message to the user:
> >    $ ip ru add pref 99 from all to 8.8.8.8 table 11
> >    Error: spectrum: FIB rules not supported. Aborting offload.
> >
> >spectrum currently only checks whether the fib rule is a default rule or
> >an l3mdev rule, both of which it knows how to handle. Any other it aborts
> >FIB offload. Since the processing is fairly quick, move the code to inline
> >with the user request rather than a work queue to allow a message to be
> >returned if the offload is aborted. Change the delete handling to just return
> >since it does nothing at the moment.
> >
> >Signed-off-by: David Ahern <dsahern@gmail.com>
> >---
> 
> [...]
> 
> >+static int mlxsw_sp_router_fib_rule_event(unsigned long event,
> >+					  struct fib_notifier_info *info,
> >+					  struct mlxsw_sp *mlxsw_sp)
> >+{
> >+	struct netlink_ext_ack *extack = info->extack;
> >+	struct fib_rule_notifier_info *fr_info;
> >+	struct fib_rule *rule;
> >+	bool add_unsupported_msg = false;
> >+
> >+	/* nothing to do at the moment */
> >+	if (event == FIB_EVENT_RULE_DEL)
> >+		goto out;
> >+
> >+	fr_info = container_of(info, struct fib_rule_notifier_info, info);
> >+	rule = fr_info->rule;
> >+
> >+	switch (info->family) {
> >+	case AF_INET:
> >+		if (!fib4_rule_default(rule) && !rule->l3mdev) {
> 
> Why don't we abort on removal of default rule?

This was discussed during initial VRF offload submission. Users need to
be able to re-order the rule for the local table after the l3mdev rule.

Re: [PATCH net-next 3/3] mlxsw: spectrum_router: Return extack message on abort due to fib rules

[David Ahern]

On 10/26/17 12:38 AM, Ido Schimmel wrote:
> 
> Nice idea, but one problem is that the FIB notifier is atomic and thus
> when you trigger abort you end up sleeping in an atomic context:
> 
...

> 
> How about we keep the checking and error reporting inline, but do the
> actual abort in a workqueue?

yep. Will spin a v2.