From mboxrd@z Thu Jan 1 00:00:00 1970 From: Eran Ben Elisha Subject: [PATCH RFC net-next 03/19] devlink: Add health report functionality Date: Mon, 31 Dec 2018 16:31:57 +0200 Message-ID: <1546266733-9512-4-git-send-email-eranbe@mellanox.com> References: <1546266733-9512-1-git-send-email-eranbe@mellanox.com> Cc: Moshe Shemesh , Aya Levin , Eran Ben Elisha , Tal Alon , Ariel Almog To: netdev@vger.kernel.org, "David S. Miller" , Jiri Pirko Return-path: Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:52201 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1726151AbeLaOc1 (ORCPT ); Mon, 31 Dec 2018 09:32:27 -0500 In-Reply-To: <1546266733-9512-1-git-send-email-eranbe@mellanox.com> Sender: netdev-owner@vger.kernel.org List-ID: Upon error discover, every driver can report it to the devlink health mechanism via devlink_health_report function, using the appropriate reporter registered to it. Driver can pass error specific context which will be delivered to it as part of the objdump / recovery callbacks. Once an error is reported, devlink health will do the following actions: * A log is being send to the kernel trace events buffer * Health status and statistics are being updated for the reporter instance * Object dump is being taken and stored at the reporter instance (as long as there is no other Objdump which is already stored) * Auto recovery attempt is being done. depends on: - Auto Recovery configuration - Grace period vs. time since last recover Signed-off-by: Eran Ben Elisha --- include/net/devlink.h | 8 +++ include/trace/events/devlink.h | 62 +++++++++++++++++++++++ net/core/devlink.c | 93 ++++++++++++++++++++++++++++++++++ 3 files changed, 163 insertions(+) diff --git a/include/net/devlink.h b/include/net/devlink.h index 6884a2571348..40f988f4c940 100644 --- a/include/net/devlink.h +++ b/include/net/devlink.h @@ -641,6 +641,8 @@ devlink_health_reporter_destroy(struct devlink_health_reporter *reporter); void * devlink_health_reporter_priv(struct devlink_health_reporter *reporter); +int devlink_health_report(struct devlink_health_reporter *reporter, + const char *msg, void *priv_ctx); #else static inline struct devlink *devlink_alloc(const struct devlink_ops *ops, @@ -977,6 +979,12 @@ static inline void * devlink_health_reporter_priv(struct devlink_health_reporter *reporter) { return NULL; + +static inline int +devlink_health_report(struct devlink_health_reporter *reporter, + const char *msg, void *priv_ctx) +{ + return 0; } #endif diff --git a/include/trace/events/devlink.h b/include/trace/events/devlink.h index 44acfbca1266..7e39d2fc7c75 100644 --- a/include/trace/events/devlink.h +++ b/include/trace/events/devlink.h @@ -46,6 +46,65 @@ TRACE_EVENT(devlink_hwmsg, (int) __entry->len, __get_dynamic_array(buf), __entry->len) ); +TRACE_EVENT(devlink_health_report, + TP_PROTO(const struct devlink *devlink, const char *reporter_name, + const char *msg), + + TP_ARGS(devlink, reporter_name, msg), + + TP_STRUCT__entry( + __string(bus_name, devlink->dev->bus->name) + __string(dev_name, dev_name(devlink->dev)) + __string(driver_name, devlink->dev->driver->name) + __string(reporter_name, msg) + __string(msg, msg) + ), + + TP_fast_assign( + __assign_str(bus_name, devlink->dev->bus->name); + __assign_str(dev_name, dev_name(devlink->dev)); + __assign_str(driver_name, devlink->dev->driver->name); + __assign_str(reporter_name, reporter_name); + __assign_str(msg, msg); + ), + + TP_printk("bus_name=%s dev_name=%s driver_name=%s reporter_name=%s: %s", + __get_str(bus_name), __get_str(dev_name), + __get_str(driver_name), __get_str(reporter_name), + __get_str(msg)) +); + +TRACE_EVENT(devlink_health_recover_aborted, + TP_PROTO(const struct devlink *devlink, const char *reporter_name, + bool health_state, u64 time_since_last_recover), + + TP_ARGS(devlink, reporter_name, health_state, time_since_last_recover), + + TP_STRUCT__entry( + __string(bus_name, devlink->dev->bus->name) + __string(dev_name, dev_name(devlink->dev)) + __string(driver_name, devlink->dev->driver->name) + __string(reporter_name, reporter_name) + __field(bool, health_state) + __field(u64, time_since_last_recover) + ), + + TP_fast_assign( + __assign_str(bus_name, devlink->dev->bus->name); + __assign_str(dev_name, dev_name(devlink->dev)); + __assign_str(driver_name, devlink->dev->driver->name); + __assign_str(reporter_name, reporter_name); + __entry->health_state = health_state; + __entry->time_since_last_recover = time_since_last_recover; + ), + + TP_printk("bus_name=%s dev_name=%s driver_name=%s reporter_name=%s: health_state=%d time_since_last_recover = %llu recover aborted", + __get_str(bus_name), __get_str(dev_name), + __get_str(driver_name), __get_str(reporter_name), + __entry->health_state, + __entry->time_since_last_recover) +); + #endif /* _TRACE_DEVLINK_H */ /* This part must be outside protection */ @@ -64,6 +123,9 @@ static inline void trace_devlink_hwmsg(const struct devlink *devlink, { } +static inline void trace_devlink_health(const char *msg) +{ +} #endif /* _TRACE_DEVLINK_H */ #endif diff --git a/net/core/devlink.c b/net/core/devlink.c index a69d4679211f..5150d127da5f 100644 --- a/net/core/devlink.c +++ b/net/core/devlink.c @@ -4110,6 +4110,16 @@ struct devlink_health_reporter { u64 graceful_period; bool auto_recover; u8 health_state; + u8 objdump_avail; + u64 objdump_ts; + u64 error_count; + u64 recovery_count; + u64 last_recovery_ts; +}; + +enum devlink_health_reporter_state { + DEVLINK_HEALTH_REPORTER_STATE_HEALTHY, + DEVLINK_HEALTH_REPORTER_STATE_ERROR, }; void * @@ -4209,6 +4219,89 @@ devlink_health_reporter_destroy(struct devlink_health_reporter *reporter) } EXPORT_SYMBOL_GPL(devlink_health_reporter_destroy); +static int +devlink_health_reporter_recover(struct devlink_health_reporter *reporter, + void *priv_ctx) +{ + int err; + + if (!reporter->ops->recover) + return -EOPNOTSUPP; + + err = reporter->ops->recover(reporter, priv_ctx); + if (err) + return err; + + reporter->recovery_count++; + reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_HEALTHY; + reporter->last_recovery_ts = jiffies; + + return 0; +} + +static int devlink_health_do_objdump(struct devlink_health_reporter *reporter, + void *priv_ctx) +{ + int err; + + if (!reporter->ops->objdump) + return 0; + + if (reporter->objdump_avail) + return 0; + + devlink_health_buffers_reset(reporter->objdump_buffers_array, + DEVLINK_HEALTH_SIZE_TO_BUFFERS(reporter->ops->objdump_size)); + err = reporter->ops->objdump(reporter, reporter->objdump_buffers_array, + DEVLINK_HEALTH_BUFFER_SIZE, + DEVLINK_HEALTH_SIZE_TO_BUFFERS(reporter->ops->objdump_size), + priv_ctx); + if (!err) { + reporter->objdump_avail = true; + reporter->objdump_ts = jiffies; + } + + return err; +} + +int devlink_health_report(struct devlink_health_reporter *reporter, + const char *msg, void *priv_ctx) +{ + struct devlink *devlink = reporter->devlink; + int err = 0; + + /* write a log message of the current error */ + WARN_ON(!msg); + trace_devlink_health_report(devlink, reporter->ops->name, msg); + reporter->error_count++; + + /* abort if the previous error wasn't recovered */ + if (reporter->auto_recover && + (reporter->health_state != DEVLINK_HEALTH_REPORTER_STATE_HEALTHY || + jiffies - reporter->last_recovery_ts < + msecs_to_jiffies(reporter->graceful_period))) { + trace_devlink_health_recover_aborted(devlink, + reporter->ops->name, + reporter->health_state, + jiffies - + reporter->last_recovery_ts); + return -ECANCELED; + } + + reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_ERROR; + + mutex_lock(&reporter->objdump_lock); + /* store current objdump of current error, for later analysis */ + devlink_health_do_objdump(reporter, priv_ctx); + mutex_unlock(&reporter->objdump_lock); + + if (reporter->auto_recover) + err = devlink_health_reporter_recover(reporter, priv_ctx); + + return err; +} +EXPORT_SYMBOL_GPL(devlink_health_report); + static const struct nla_policy devlink_nl_policy[DEVLINK_ATTR_MAX + 1] = { [DEVLINK_ATTR_BUS_NAME] = { .type = NLA_NUL_STRING }, [DEVLINK_ATTR_DEV_NAME] = { .type = NLA_NUL_STRING }, -- 2.17.1