[PATCH v3] opensm: support routing engine update

[PATCH v3] opensm: support routing engine update

[Slava Strebkov]

setup routing engine when in use and delete when failed.
setup routing engine before use.
delete resources when routing algorithm fails.
this will save allocation for routing algorithms that are not used.

Signed-off-by: Slava Strebkov <slavas-smomgflXvOZWk0Htik3J/w@public.gmane.org>
---
 opensm/include/opensm/osm_opensm.h |    5 +++
 opensm/opensm/osm_opensm.c         |   57 +++++++++++++++++++++++++++++++-----
 opensm/opensm/osm_subnet.c         |    7 ++++-
 opensm/opensm/osm_ucast_mgr.c      |   28 +++++++++++++++++
 4 files changed, 88 insertions(+), 9 deletions(-)

diff --git a/opensm/include/opensm/osm_opensm.h b/opensm/include/opensm/osm_opensm.h
index c121be4..ca0fddb 100644
--- a/opensm/include/opensm/osm_opensm.h
+++ b/opensm/include/opensm/osm_opensm.h
@@ -109,6 +109,7 @@ typedef enum _osm_routing_engine_type {
 } osm_routing_engine_type_t;
 /***********/
 
+struct osm_opensm;
 /****s* OpenSM: OpenSM/osm_routing_engine
 * NAME
 *	struct osm_routing_engine
@@ -122,6 +123,8 @@ typedef enum _osm_routing_engine_type {
 struct osm_routing_engine {
 	const char *name;
 	void *context;
+	int initialized;
+	int (*setup) (struct osm_routing_engine *re, struct osm_opensm *p_osm);
 	int (*build_lid_matrices) (void *context);
 	int (*ucast_build_fwd_tables) (void *context);
 	void (*ucast_dump_tables) (void *context);
@@ -183,6 +186,7 @@ typedef struct osm_opensm {
 	cl_dispatcher_t disp;
 	cl_plock_t lock;
 	struct osm_routing_engine *routing_engine_list;
+	struct osm_routing_engine *last_routing_engine;
 	osm_routing_engine_type_t routing_engine_used;
 	osm_stats_t stats;
 	osm_console_t console;
@@ -522,6 +526,7 @@ extern volatile unsigned int osm_exit_flag;
 * DESCRIPTION
 *  Set to one to cause all threads to leave
 *********/
+void osm_update_routing_engines(osm_opensm_t *osm, const char *engine_names);
 
 END_C_DECLS
 #endif				/* _OSM_OPENSM_H_ */
diff --git a/opensm/opensm/osm_opensm.c b/opensm/opensm/osm_opensm.c
index 50d1349..f90584d 100644
--- a/opensm/opensm/osm_opensm.c
+++ b/opensm/opensm/osm_opensm.c
@@ -169,14 +169,7 @@ static void setup_routing_engine(osm_opensm_t *osm, const char *name)
 			memset(re, 0, sizeof(struct osm_routing_engine));
 
 			re->name = m->name;
-			if (m->setup(re, osm)) {
-				OSM_LOG(&osm->log, OSM_LOG_VERBOSE,
-					"setup of routing"
-					" engine \'%s\' failed\n", name);
-				return;
-			}
-			OSM_LOG(&osm->log, OSM_LOG_DEBUG,
-				"\'%s\' routing engine set up\n", re->name);
+			re->setup = m->setup;
 			append_routing_engine(osm, re);
 			return;
 		}
@@ -236,6 +229,54 @@ static void destroy_routing_engines(osm_opensm_t *osm)
 			r->delete(r->context);
 		free(r);
 	}
+    osm->routing_engine_list = NULL;
+}
+
+static void update_routing_engine(
+       struct osm_routing_engine *cur,
+       struct osm_routing_engine *last)
+{
+	struct osm_routing_engine *next = cur->next;
+	if (!last)
+		return; /* no last routing engine */
+	memcpy(cur, last, sizeof(*cur));
+	/* restore next */
+	cur->next = next;
+}
+
+void osm_update_routing_engines(osm_opensm_t *osm, const char *engine_names)
+{
+	struct osm_routing_engine *r, *l;
+	/* find used routing engine and save as last */
+	l = r = osm->routing_engine_list;
+	if (r && osm->routing_engine_used == osm_routing_engine_type(r->name)) {
+		osm->last_routing_engine = r;
+		osm->routing_engine_list = r->next;
+	}
+	else while ((r = r->next)) {
+		if (osm->routing_engine_used ==
+			osm_routing_engine_type(r->name)) {
+				osm->last_routing_engine = r;
+				l->next = r->next;
+				break;
+		}
+		l = r;
+	}
+	/* cleanup prev routing engine list and replace with current list */
+	destroy_routing_engines(osm);
+	setup_routing_engines(osm, engine_names);
+	/* check if last routing engine exist in new list and update callbacks */
+	r = osm->routing_engine_list;
+	while (r) {
+		if (osm->routing_engine_used ==
+			osm_routing_engine_type(r->name)) {
+				update_routing_engine(r, osm->last_routing_engine);
+				free(osm->last_routing_engine);
+				osm->last_routing_engine = NULL;
+				break;
+		}
+	r = r->next;
+	}
 }
 
 /**********************************************************************
diff --git a/opensm/opensm/osm_subnet.c b/opensm/opensm/osm_subnet.c
index 8d63a75..742ae64 100644
--- a/opensm/opensm/osm_subnet.c
+++ b/opensm/opensm/osm_subnet.c
@@ -152,6 +152,11 @@ static void opts_setup_sm_priority(osm_subn_t *p_subn, void *p_val)
 	osm_set_sm_priority(p_sm, sm_priority);
 }
 
+static void opts_setup_routing_engine(osm_subn_t *p_subn, void *p_val)
+{
+	osm_update_routing_engines(p_subn->p_osm, p_val);
+}
+
 static void opts_parse_net64(IN osm_subn_t *p_subn, IN char *p_key,
 			     IN char *p_val_str, void *p_v1, void *p_v2,
 			     void (*pfn)(osm_subn_t *, void *))
@@ -324,7 +329,7 @@ static const opt_rec_t opt_tbl[] = {
 	{ "hop_weights_file", OPT_OFFSET(hop_weights_file), opts_parse_charp, NULL, 0 },
 	{ "port_profile_switch_nodes", OPT_OFFSET(port_profile_switch_nodes), opts_parse_boolean, NULL, 1 },
 	{ "sweep_on_trap", OPT_OFFSET(sweep_on_trap), opts_parse_boolean, NULL, 1 },
-	{ "routing_engine", OPT_OFFSET(routing_engine_names), opts_parse_charp, NULL, 0 },
+	{ "routing_engine", OPT_OFFSET(routing_engine_names), opts_parse_charp, opts_setup_routing_engine, 1 },
 	{ "connect_roots", OPT_OFFSET(connect_roots), opts_parse_boolean, NULL, 1 },
 	{ "use_ucast_cache", OPT_OFFSET(use_ucast_cache), opts_parse_boolean, NULL, 1 },
 	{ "log_file", OPT_OFFSET(log_file), opts_parse_charp, NULL, 0 },
diff --git a/opensm/opensm/osm_ucast_mgr.c b/opensm/opensm/osm_ucast_mgr.c
index 39d825c..d6294ac 100644
--- a/opensm/opensm/osm_ucast_mgr.c
+++ b/opensm/opensm/osm_ucast_mgr.c
@@ -998,8 +998,23 @@ int osm_ucast_mgr_process(IN osm_ucast_mgr_t * p_mgr)
 
 	p_osm->routing_engine_used = OSM_ROUTING_ENGINE_TYPE_NONE;
 	while (p_routing_eng) {
+		if (!p_routing_eng->initialized &&
+			p_routing_eng->setup(p_routing_eng, p_osm)) {
+			OSM_LOG(p_mgr->p_log, OSM_LOG_ERROR,
+				"ERR 3A0F: setup of routing engine \'%s\' failed\n",
+					p_routing_eng->name);
+					p_routing_eng = p_routing_eng->next;
+					continue;
+		}
+		OSM_LOG(p_mgr->p_log, OSM_LOG_INFO,
+			"\'%s\' routing engine set up\n", p_routing_eng->name);
+		p_routing_eng->initialized = 1;
 		if (!ucast_mgr_route(p_routing_eng, p_osm))
 			break;
+		/* delete unused routing engine */
+		if (p_routing_eng->delete)
+			p_routing_eng->delete(p_routing_eng->context);
+		p_routing_eng->initialized = 0;
 		p_routing_eng = p_routing_eng->next;
 	}
 
@@ -1011,6 +1026,19 @@ int osm_ucast_mgr_process(IN osm_ucast_mgr_t * p_mgr)
 		p_osm->routing_engine_used = OSM_ROUTING_ENGINE_TYPE_MINHOP;
 	}
 
+	/* if for some reason different routing engine is used */
+	/* cleanup last unused routing engine */
+	p_routing_eng = p_osm->last_routing_engine;
+	if (p_routing_eng) {
+			if (p_routing_eng->initialized &&
+					p_routing_eng->delete &&
+					p_osm->routing_engine_used !=
+						osm_routing_engine_type(p_routing_eng->name))
+					p_routing_eng->delete(p_routing_eng->context);
+			free(p_routing_eng);
+			p_osm->last_routing_engine = NULL;
+	}
+
 	OSM_LOG(p_mgr->p_log, OSM_LOG_INFO,
 		"%s tables configured on all switches\n",
 		osm_routing_engine_type_str(p_osm->routing_engine_used));
-- 
1.6.3.3

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v3] opensm: support routing engine update

[Yevgeny Kliteynik]

Slava,

Slava Strebkov wrote:
> setup routing engine when in use and delete when failed.
> setup routing engine before use.
> delete resources when routing algorithm fails.
> this will save allocation for routing algorithms that are not used.

Suppose a user runs SM with ftree & updn routings (in that order),
and SM manages to route the fabric with ftree. At some point 
some switch reboots and causes ftree to fail and SM routes the
fabric with updn.
Does this mean that ftree will be removed from the list, and
when the switch comes back, SM won't try ftree any more?

-- Yevgeny
 
> Signed-off-by: Slava Strebkov <slavas-smomgflXvOZWk0Htik3J/w@public.gmane.org>
> ---
>  opensm/include/opensm/osm_opensm.h |    5 +++
>  opensm/opensm/osm_opensm.c         |   57 +++++++++++++++++++++++++++++++-----
>  opensm/opensm/osm_subnet.c         |    7 ++++-
>  opensm/opensm/osm_ucast_mgr.c      |   28 +++++++++++++++++
>  4 files changed, 88 insertions(+), 9 deletions(-)
> 
> diff --git a/opensm/include/opensm/osm_opensm.h b/opensm/include/opensm/osm_opensm.h
> index c121be4..ca0fddb 100644
> --- a/opensm/include/opensm/osm_opensm.h
> +++ b/opensm/include/opensm/osm_opensm.h
> @@ -109,6 +109,7 @@ typedef enum _osm_routing_engine_type {
>  } osm_routing_engine_type_t;
>  /***********/
>  
> +struct osm_opensm;
>  /****s* OpenSM: OpenSM/osm_routing_engine
>  * NAME
>  *	struct osm_routing_engine
> @@ -122,6 +123,8 @@ typedef enum _osm_routing_engine_type {
>  struct osm_routing_engine {
>  	const char *name;
>  	void *context;
> +	int initialized;
> +	int (*setup) (struct osm_routing_engine *re, struct osm_opensm *p_osm);
>  	int (*build_lid_matrices) (void *context);
>  	int (*ucast_build_fwd_tables) (void *context);
>  	void (*ucast_dump_tables) (void *context);
> @@ -183,6 +186,7 @@ typedef struct osm_opensm {
>  	cl_dispatcher_t disp;
>  	cl_plock_t lock;
>  	struct osm_routing_engine *routing_engine_list;
> +	struct osm_routing_engine *last_routing_engine;
>  	osm_routing_engine_type_t routing_engine_used;
>  	osm_stats_t stats;
>  	osm_console_t console;
> @@ -522,6 +526,7 @@ extern volatile unsigned int osm_exit_flag;
>  * DESCRIPTION
>  *  Set to one to cause all threads to leave
>  *********/
> +void osm_update_routing_engines(osm_opensm_t *osm, const char *engine_names);
>  
>  END_C_DECLS
>  #endif				/* _OSM_OPENSM_H_ */
> diff --git a/opensm/opensm/osm_opensm.c b/opensm/opensm/osm_opensm.c
> index 50d1349..f90584d 100644
> --- a/opensm/opensm/osm_opensm.c
> +++ b/opensm/opensm/osm_opensm.c
> @@ -169,14 +169,7 @@ static void setup_routing_engine(osm_opensm_t *osm, const char *name)
>  			memset(re, 0, sizeof(struct osm_routing_engine));
>  
>  			re->name = m->name;
> -			if (m->setup(re, osm)) {
> -				OSM_LOG(&osm->log, OSM_LOG_VERBOSE,
> -					"setup of routing"
> -					" engine \'%s\' failed\n", name);
> -				return;
> -			}
> -			OSM_LOG(&osm->log, OSM_LOG_DEBUG,
> -				"\'%s\' routing engine set up\n", re->name);
> +			re->setup = m->setup;
>  			append_routing_engine(osm, re);
>  			return;
>  		}
> @@ -236,6 +229,54 @@ static void destroy_routing_engines(osm_opensm_t *osm)
>  			r->delete(r->context);
>  		free(r);
>  	}
> +    osm->routing_engine_list = NULL;
> +}
> +
> +static void update_routing_engine(
> +       struct osm_routing_engine *cur,
> +       struct osm_routing_engine *last)
> +{
> +	struct osm_routing_engine *next = cur->next;
> +	if (!last)
> +		return; /* no last routing engine */
> +	memcpy(cur, last, sizeof(*cur));
> +	/* restore next */
> +	cur->next = next;
> +}
> +
> +void osm_update_routing_engines(osm_opensm_t *osm, const char *engine_names)
> +{
> +	struct osm_routing_engine *r, *l;
> +	/* find used routing engine and save as last */
> +	l = r = osm->routing_engine_list;
> +	if (r && osm->routing_engine_used == osm_routing_engine_type(r->name)) {
> +		osm->last_routing_engine = r;
> +		osm->routing_engine_list = r->next;
> +	}
> +	else while ((r = r->next)) {
> +		if (osm->routing_engine_used ==
> +			osm_routing_engine_type(r->name)) {
> +				osm->last_routing_engine = r;
> +				l->next = r->next;
> +				break;
> +		}
> +		l = r;
> +	}
> +	/* cleanup prev routing engine list and replace with current list */
> +	destroy_routing_engines(osm);
> +	setup_routing_engines(osm, engine_names);
> +	/* check if last routing engine exist in new list and update callbacks */
> +	r = osm->routing_engine_list;
> +	while (r) {
> +		if (osm->routing_engine_used ==
> +			osm_routing_engine_type(r->name)) {
> +				update_routing_engine(r, osm->last_routing_engine);
> +				free(osm->last_routing_engine);
> +				osm->last_routing_engine = NULL;
> +				break;
> +		}
> +	r = r->next;
> +	}
>  }
>  
>  /**********************************************************************
> diff --git a/opensm/opensm/osm_subnet.c b/opensm/opensm/osm_subnet.c
> index 8d63a75..742ae64 100644
> --- a/opensm/opensm/osm_subnet.c
> +++ b/opensm/opensm/osm_subnet.c
> @@ -152,6 +152,11 @@ static void opts_setup_sm_priority(osm_subn_t *p_subn, void *p_val)
>  	osm_set_sm_priority(p_sm, sm_priority);
>  }
>  
> +static void opts_setup_routing_engine(osm_subn_t *p_subn, void *p_val)
> +{
> +	osm_update_routing_engines(p_subn->p_osm, p_val);
> +}
> +
>  static void opts_parse_net64(IN osm_subn_t *p_subn, IN char *p_key,
>  			     IN char *p_val_str, void *p_v1, void *p_v2,
>  			     void (*pfn)(osm_subn_t *, void *))
> @@ -324,7 +329,7 @@ static const opt_rec_t opt_tbl[] = {
>  	{ "hop_weights_file", OPT_OFFSET(hop_weights_file), opts_parse_charp, NULL, 0 },
>  	{ "port_profile_switch_nodes", OPT_OFFSET(port_profile_switch_nodes), opts_parse_boolean, NULL, 1 },
>  	{ "sweep_on_trap", OPT_OFFSET(sweep_on_trap), opts_parse_boolean, NULL, 1 },
> -	{ "routing_engine", OPT_OFFSET(routing_engine_names), opts_parse_charp, NULL, 0 },
> +	{ "routing_engine", OPT_OFFSET(routing_engine_names), opts_parse_charp, opts_setup_routing_engine, 1 },
>  	{ "connect_roots", OPT_OFFSET(connect_roots), opts_parse_boolean, NULL, 1 },
>  	{ "use_ucast_cache", OPT_OFFSET(use_ucast_cache), opts_parse_boolean, NULL, 1 },
>  	{ "log_file", OPT_OFFSET(log_file), opts_parse_charp, NULL, 0 },
> diff --git a/opensm/opensm/osm_ucast_mgr.c b/opensm/opensm/osm_ucast_mgr.c
> index 39d825c..d6294ac 100644
> --- a/opensm/opensm/osm_ucast_mgr.c
> +++ b/opensm/opensm/osm_ucast_mgr.c
> @@ -998,8 +998,23 @@ int osm_ucast_mgr_process(IN osm_ucast_mgr_t * p_mgr)
>  
>  	p_osm->routing_engine_used = OSM_ROUTING_ENGINE_TYPE_NONE;
>  	while (p_routing_eng) {
> +		if (!p_routing_eng->initialized &&
> +			p_routing_eng->setup(p_routing_eng, p_osm)) {
> +			OSM_LOG(p_mgr->p_log, OSM_LOG_ERROR,
> +				"ERR 3A0F: setup of routing engine \'%s\' failed\n",
> +					p_routing_eng->name);
> +					p_routing_eng = p_routing_eng->next;
> +					continue;
> +		}
> +		OSM_LOG(p_mgr->p_log, OSM_LOG_INFO,
> +			"\'%s\' routing engine set up\n", p_routing_eng->name);
> +		p_routing_eng->initialized = 1;
>  		if (!ucast_mgr_route(p_routing_eng, p_osm))
>  			break;
> +		/* delete unused routing engine */
> +		if (p_routing_eng->delete)
> +			p_routing_eng->delete(p_routing_eng->context);
> +		p_routing_eng->initialized = 0;
>  		p_routing_eng = p_routing_eng->next;
>  	}
>  
> @@ -1011,6 +1026,19 @@ int osm_ucast_mgr_process(IN osm_ucast_mgr_t * p_mgr)
>  		p_osm->routing_engine_used = OSM_ROUTING_ENGINE_TYPE_MINHOP;
>  	}
>  
> +	/* if for some reason different routing engine is used */
> +	/* cleanup last unused routing engine */
> +	p_routing_eng = p_osm->last_routing_engine;
> +	if (p_routing_eng) {
> +			if (p_routing_eng->initialized &&
> +					p_routing_eng->delete &&
> +					p_osm->routing_engine_used !=
> +						osm_routing_engine_type(p_routing_eng->name))
> +					p_routing_eng->delete(p_routing_eng->context);
> +			free(p_routing_eng);
> +			p_osm->last_routing_engine = NULL;
> +	}
> +
>  	OSM_LOG(p_mgr->p_log, OSM_LOG_INFO,
>  		"%s tables configured on all switches\n",
>  		osm_routing_engine_type_str(p_osm->routing_engine_used));

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

RE: [PATCH v3] opensm: support routing engine update

[Slava Strebkov]

Hi Yevgeny,
In that case SM will use updn and will not come back to ftree
automatically.

Slava

-----Original Message-----
From: linux-rdma-owner-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
[mailto:linux-rdma-owner-u79uwXL29TY76Z2rM5mHXA@public.gmane.org] On Behalf Of Yevgeny Kliteynik
Sent: Sunday, December 06, 2009 6:03 PM
To: Slava Strebkov
Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
Subject: Re: [PATCH v3] opensm: support routing engine update

Slava,

Slava Strebkov wrote:
> setup routing engine when in use and delete when failed.
> setup routing engine before use.
> delete resources when routing algorithm fails.
> this will save allocation for routing algorithms that are not used.

Suppose a user runs SM with ftree & updn routings (in that order),
and SM manages to route the fabric with ftree. At some point 
some switch reboots and causes ftree to fail and SM routes the
fabric with updn.
Does this mean that ftree will be removed from the list, and
when the switch comes back, SM won't try ftree any more?

-- Yevgeny
 
> Signed-off-by: Slava Strebkov <slavas-smomgflXvOZWk0Htik3J/w@public.gmane.org>
> ---
>  opensm/include/opensm/osm_opensm.h |    5 +++
>  opensm/opensm/osm_opensm.c         |   57
+++++++++++++++++++++++++++++++-----
>  opensm/opensm/osm_subnet.c         |    7 ++++-
>  opensm/opensm/osm_ucast_mgr.c      |   28 +++++++++++++++++
>  4 files changed, 88 insertions(+), 9 deletions(-)
> 
> diff --git a/opensm/include/opensm/osm_opensm.h
b/opensm/include/opensm/osm_opensm.h
> index c121be4..ca0fddb 100644
> --- a/opensm/include/opensm/osm_opensm.h
> +++ b/opensm/include/opensm/osm_opensm.h
> @@ -109,6 +109,7 @@ typedef enum _osm_routing_engine_type {
>  } osm_routing_engine_type_t;
>  /***********/
>  
> +struct osm_opensm;
>  /****s* OpenSM: OpenSM/osm_routing_engine
>  * NAME
>  *	struct osm_routing_engine
> @@ -122,6 +123,8 @@ typedef enum _osm_routing_engine_type {
>  struct osm_routing_engine {
>  	const char *name;
>  	void *context;
> +	int initialized;
> +	int (*setup) (struct osm_routing_engine *re, struct osm_opensm
*p_osm);
>  	int (*build_lid_matrices) (void *context);
>  	int (*ucast_build_fwd_tables) (void *context);
>  	void (*ucast_dump_tables) (void *context);
> @@ -183,6 +186,7 @@ typedef struct osm_opensm {
>  	cl_dispatcher_t disp;
>  	cl_plock_t lock;
>  	struct osm_routing_engine *routing_engine_list;
> +	struct osm_routing_engine *last_routing_engine;
>  	osm_routing_engine_type_t routing_engine_used;
>  	osm_stats_t stats;
>  	osm_console_t console;
> @@ -522,6 +526,7 @@ extern volatile unsigned int osm_exit_flag;
>  * DESCRIPTION
>  *  Set to one to cause all threads to leave
>  *********/
> +void osm_update_routing_engines(osm_opensm_t *osm, const char
*engine_names);
>  
>  END_C_DECLS
>  #endif				/* _OSM_OPENSM_H_ */
> diff --git a/opensm/opensm/osm_opensm.c b/opensm/opensm/osm_opensm.c
> index 50d1349..f90584d 100644
> --- a/opensm/opensm/osm_opensm.c
> +++ b/opensm/opensm/osm_opensm.c
> @@ -169,14 +169,7 @@ static void setup_routing_engine(osm_opensm_t
*osm, const char *name)
>  			memset(re, 0, sizeof(struct
osm_routing_engine));
>  
>  			re->name = m->name;
> -			if (m->setup(re, osm)) {
> -				OSM_LOG(&osm->log, OSM_LOG_VERBOSE,
> -					"setup of routing"
> -					" engine \'%s\' failed\n",
name);
> -				return;
> -			}
> -			OSM_LOG(&osm->log, OSM_LOG_DEBUG,
> -				"\'%s\' routing engine set up\n",
re->name);
> +			re->setup = m->setup;
>  			append_routing_engine(osm, re);
>  			return;
>  		}
> @@ -236,6 +229,54 @@ static void destroy_routing_engines(osm_opensm_t
*osm)
>  			r->delete(r->context);
>  		free(r);
>  	}
> +    osm->routing_engine_list = NULL;
> +}
> +
> +static void update_routing_engine(
> +       struct osm_routing_engine *cur,
> +       struct osm_routing_engine *last)
> +{
> +	struct osm_routing_engine *next = cur->next;
> +	if (!last)
> +		return; /* no last routing engine */
> +	memcpy(cur, last, sizeof(*cur));
> +	/* restore next */
> +	cur->next = next;
> +}
> +
> +void osm_update_routing_engines(osm_opensm_t *osm, const char
*engine_names)
> +{
> +	struct osm_routing_engine *r, *l;
> +	/* find used routing engine and save as last */
> +	l = r = osm->routing_engine_list;
> +	if (r && osm->routing_engine_used ==
osm_routing_engine_type(r->name)) {
> +		osm->last_routing_engine = r;
> +		osm->routing_engine_list = r->next;
> +	}
> +	else while ((r = r->next)) {
> +		if (osm->routing_engine_used ==
> +			osm_routing_engine_type(r->name)) {
> +				osm->last_routing_engine = r;
> +				l->next = r->next;
> +				break;
> +		}
> +		l = r;
> +	}
> +	/* cleanup prev routing engine list and replace with current
list */
> +	destroy_routing_engines(osm);
> +	setup_routing_engines(osm, engine_names);
> +	/* check if last routing engine exist in new list and update
callbacks */
> +	r = osm->routing_engine_list;
> +	while (r) {
> +		if (osm->routing_engine_used ==
> +			osm_routing_engine_type(r->name)) {
> +				update_routing_engine(r,
osm->last_routing_engine);
> +				free(osm->last_routing_engine);
> +				osm->last_routing_engine = NULL;
> +				break;
> +		}
> +	r = r->next;
> +	}
>  }
>  
>
/**********************************************************************
> diff --git a/opensm/opensm/osm_subnet.c b/opensm/opensm/osm_subnet.c
> index 8d63a75..742ae64 100644
> --- a/opensm/opensm/osm_subnet.c
> +++ b/opensm/opensm/osm_subnet.c
> @@ -152,6 +152,11 @@ static void opts_setup_sm_priority(osm_subn_t
*p_subn, void *p_val)
>  	osm_set_sm_priority(p_sm, sm_priority);
>  }
>  
> +static void opts_setup_routing_engine(osm_subn_t *p_subn, void
*p_val)
> +{
> +	osm_update_routing_engines(p_subn->p_osm, p_val);
> +}
> +
>  static void opts_parse_net64(IN osm_subn_t *p_subn, IN char *p_key,
>  			     IN char *p_val_str, void *p_v1, void *p_v2,
>  			     void (*pfn)(osm_subn_t *, void *))
> @@ -324,7 +329,7 @@ static const opt_rec_t opt_tbl[] = {
>  	{ "hop_weights_file", OPT_OFFSET(hop_weights_file),
opts_parse_charp, NULL, 0 },
>  	{ "port_profile_switch_nodes",
OPT_OFFSET(port_profile_switch_nodes), opts_parse_boolean, NULL, 1 },
>  	{ "sweep_on_trap", OPT_OFFSET(sweep_on_trap),
opts_parse_boolean, NULL, 1 },
> -	{ "routing_engine", OPT_OFFSET(routing_engine_names),
opts_parse_charp, NULL, 0 },
> +	{ "routing_engine", OPT_OFFSET(routing_engine_names),
opts_parse_charp, opts_setup_routing_engine, 1 },
>  	{ "connect_roots", OPT_OFFSET(connect_roots),
opts_parse_boolean, NULL, 1 },
>  	{ "use_ucast_cache", OPT_OFFSET(use_ucast_cache),
opts_parse_boolean, NULL, 1 },
>  	{ "log_file", OPT_OFFSET(log_file), opts_parse_charp, NULL, 0 },
> diff --git a/opensm/opensm/osm_ucast_mgr.c
b/opensm/opensm/osm_ucast_mgr.c
> index 39d825c..d6294ac 100644
> --- a/opensm/opensm/osm_ucast_mgr.c
> +++ b/opensm/opensm/osm_ucast_mgr.c
> @@ -998,8 +998,23 @@ int osm_ucast_mgr_process(IN osm_ucast_mgr_t *
p_mgr)
>  
>  	p_osm->routing_engine_used = OSM_ROUTING_ENGINE_TYPE_NONE;
>  	while (p_routing_eng) {
> +		if (!p_routing_eng->initialized &&
> +			p_routing_eng->setup(p_routing_eng, p_osm)) {
> +			OSM_LOG(p_mgr->p_log, OSM_LOG_ERROR,
> +				"ERR 3A0F: setup of routing engine
\'%s\' failed\n",
> +					p_routing_eng->name);
> +					p_routing_eng =
p_routing_eng->next;
> +					continue;
> +		}
> +		OSM_LOG(p_mgr->p_log, OSM_LOG_INFO,
> +			"\'%s\' routing engine set up\n",
p_routing_eng->name);
> +		p_routing_eng->initialized = 1;
>  		if (!ucast_mgr_route(p_routing_eng, p_osm))
>  			break;
> +		/* delete unused routing engine */
> +		if (p_routing_eng->delete)
> +			p_routing_eng->delete(p_routing_eng->context);
> +		p_routing_eng->initialized = 0;
>  		p_routing_eng = p_routing_eng->next;
>  	}
>  
> @@ -1011,6 +1026,19 @@ int osm_ucast_mgr_process(IN osm_ucast_mgr_t *
p_mgr)
>  		p_osm->routing_engine_used =
OSM_ROUTING_ENGINE_TYPE_MINHOP;
>  	}
>  
> +	/* if for some reason different routing engine is used */
> +	/* cleanup last unused routing engine */
> +	p_routing_eng = p_osm->last_routing_engine;
> +	if (p_routing_eng) {
> +			if (p_routing_eng->initialized &&
> +					p_routing_eng->delete &&
> +					p_osm->routing_engine_used !=
> +
osm_routing_engine_type(p_routing_eng->name))
> +
p_routing_eng->delete(p_routing_eng->context);
> +			free(p_routing_eng);
> +			p_osm->last_routing_engine = NULL;
> +	}
> +
>  	OSM_LOG(p_mgr->p_log, OSM_LOG_INFO,
>  		"%s tables configured on all switches\n",
>
osm_routing_engine_type_str(p_osm->routing_engine_used));

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v3] opensm: support routing engine update

[Yevgeny Kliteynik]

Slava,

Slava Strebkov wrote:
> Hi Yevgeny,
> In that case SM will use updn and will not come back to ftree
> automatically.

I think that this is a bad thing.

I wouldn't want *temporary* change of fabric to cause
*permanent* change of SM mode of operation. Such changes
do happen, and I'd prefer SM to continue functioning
in accordance to the user's configuration once the
fabric is settled again.

I do see the cases were the change that you propose is
beneficial - if fabric topology doesn't fits the chosen
routing, SM will waste time on retrying the wrong routing
at every heavy sweep, but this happens due to suboptimal
SM configuration and not as a result of some event that
user has no control of.

-- Yevgeny


 
> Slava
> 
> -----Original Message-----
> From: linux-rdma-owner-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
> [mailto:linux-rdma-owner-u79uwXL29TY76Z2rM5mHXA@public.gmane.org] On Behalf Of Yevgeny Kliteynik
> Sent: Sunday, December 06, 2009 6:03 PM
> To: Slava Strebkov
> Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
> Subject: Re: [PATCH v3] opensm: support routing engine update
> 
> Slava,
> 
> Slava Strebkov wrote:
>> setup routing engine when in use and delete when failed.
>> setup routing engine before use.
>> delete resources when routing algorithm fails.
>> this will save allocation for routing algorithms that are not used.
> 
> Suppose a user runs SM with ftree & updn routings (in that order),
> and SM manages to route the fabric with ftree. At some point 
> some switch reboots and causes ftree to fail and SM routes the
> fabric with updn.
> Does this mean that ftree will be removed from the list, and
> when the switch comes back, SM won't try ftree any more?
> 
> -- Yevgeny
>  
>> Signed-off-by: Slava Strebkov <slavas-smomgflXvOZWk0Htik3J/w@public.gmane.org>
>> ---
>>  opensm/include/opensm/osm_opensm.h |    5 +++
>>  opensm/opensm/osm_opensm.c         |   57
> +++++++++++++++++++++++++++++++-----
>>  opensm/opensm/osm_subnet.c         |    7 ++++-
>>  opensm/opensm/osm_ucast_mgr.c      |   28 +++++++++++++++++
>>  4 files changed, 88 insertions(+), 9 deletions(-)
>>
>> diff --git a/opensm/include/opensm/osm_opensm.h
> b/opensm/include/opensm/osm_opensm.h
>> index c121be4..ca0fddb 100644
>> --- a/opensm/include/opensm/osm_opensm.h
>> +++ b/opensm/include/opensm/osm_opensm.h
>> @@ -109,6 +109,7 @@ typedef enum _osm_routing_engine_type {
>>  } osm_routing_engine_type_t;
>>  /***********/
>>  
>> +struct osm_opensm;
>>  /****s* OpenSM: OpenSM/osm_routing_engine
>>  * NAME
>>  *	struct osm_routing_engine
>> @@ -122,6 +123,8 @@ typedef enum _osm_routing_engine_type {
>>  struct osm_routing_engine {
>>  	const char *name;
>>  	void *context;
>> +	int initialized;
>> +	int (*setup) (struct osm_routing_engine *re, struct osm_opensm
> *p_osm);
>>  	int (*build_lid_matrices) (void *context);
>>  	int (*ucast_build_fwd_tables) (void *context);
>>  	void (*ucast_dump_tables) (void *context);
>> @@ -183,6 +186,7 @@ typedef struct osm_opensm {
>>  	cl_dispatcher_t disp;
>>  	cl_plock_t lock;
>>  	struct osm_routing_engine *routing_engine_list;
>> +	struct osm_routing_engine *last_routing_engine;
>>  	osm_routing_engine_type_t routing_engine_used;
>>  	osm_stats_t stats;
>>  	osm_console_t console;
>> @@ -522,6 +526,7 @@ extern volatile unsigned int osm_exit_flag;
>>  * DESCRIPTION
>>  *  Set to one to cause all threads to leave
>>  *********/
>> +void osm_update_routing_engines(osm_opensm_t *osm, const char
> *engine_names);
>>  
>>  END_C_DECLS
>>  #endif				/* _OSM_OPENSM_H_ */
>> diff --git a/opensm/opensm/osm_opensm.c b/opensm/opensm/osm_opensm.c
>> index 50d1349..f90584d 100644
>> --- a/opensm/opensm/osm_opensm.c
>> +++ b/opensm/opensm/osm_opensm.c
>> @@ -169,14 +169,7 @@ static void setup_routing_engine(osm_opensm_t
> *osm, const char *name)
>>  			memset(re, 0, sizeof(struct
> osm_routing_engine));
>>  
>>  			re->name = m->name;
>> -			if (m->setup(re, osm)) {
>> -				OSM_LOG(&osm->log, OSM_LOG_VERBOSE,
>> -					"setup of routing"
>> -					" engine \'%s\' failed\n",
> name);
>> -				return;
>> -			}
>> -			OSM_LOG(&osm->log, OSM_LOG_DEBUG,
>> -				"\'%s\' routing engine set up\n",
> re->name);
>> +			re->setup = m->setup;
>>  			append_routing_engine(osm, re);
>>  			return;
>>  		}
>> @@ -236,6 +229,54 @@ static void destroy_routing_engines(osm_opensm_t
> *osm)
>>  			r->delete(r->context);
>>  		free(r);
>>  	}
>> +    osm->routing_engine_list = NULL;
>> +}
>> +
>> +static void update_routing_engine(
>> +       struct osm_routing_engine *cur,
>> +       struct osm_routing_engine *last)
>> +{
>> +	struct osm_routing_engine *next = cur->next;
>> +	if (!last)
>> +		return; /* no last routing engine */
>> +	memcpy(cur, last, sizeof(*cur));
>> +	/* restore next */
>> +	cur->next = next;
>> +}
>> +
>> +void osm_update_routing_engines(osm_opensm_t *osm, const char
> *engine_names)
>> +{
>> +	struct osm_routing_engine *r, *l;
>> +	/* find used routing engine and save as last */
>> +	l = r = osm->routing_engine_list;
>> +	if (r && osm->routing_engine_used ==
> osm_routing_engine_type(r->name)) {
>> +		osm->last_routing_engine = r;
>> +		osm->routing_engine_list = r->next;
>> +	}
>> +	else while ((r = r->next)) {
>> +		if (osm->routing_engine_used ==
>> +			osm_routing_engine_type(r->name)) {
>> +				osm->last_routing_engine = r;
>> +				l->next = r->next;
>> +				break;
>> +		}
>> +		l = r;
>> +	}
>> +	/* cleanup prev routing engine list and replace with current
> list */
>> +	destroy_routing_engines(osm);
>> +	setup_routing_engines(osm, engine_names);
>> +	/* check if last routing engine exist in new list and update
> callbacks */
>> +	r = osm->routing_engine_list;
>> +	while (r) {
>> +		if (osm->routing_engine_used ==
>> +			osm_routing_engine_type(r->name)) {
>> +				update_routing_engine(r,
> osm->last_routing_engine);
>> +				free(osm->last_routing_engine);
>> +				osm->last_routing_engine = NULL;
>> +				break;
>> +		}
>> +	r = r->next;
>> +	}
>>  }
>>  
>>
> /**********************************************************************
>> diff --git a/opensm/opensm/osm_subnet.c b/opensm/opensm/osm_subnet.c
>> index 8d63a75..742ae64 100644
>> --- a/opensm/opensm/osm_subnet.c
>> +++ b/opensm/opensm/osm_subnet.c
>> @@ -152,6 +152,11 @@ static void opts_setup_sm_priority(osm_subn_t
> *p_subn, void *p_val)
>>  	osm_set_sm_priority(p_sm, sm_priority);
>>  }
>>  
>> +static void opts_setup_routing_engine(osm_subn_t *p_subn, void
> *p_val)
>> +{
>> +	osm_update_routing_engines(p_subn->p_osm, p_val);
>> +}
>> +
>>  static void opts_parse_net64(IN osm_subn_t *p_subn, IN char *p_key,
>>  			     IN char *p_val_str, void *p_v1, void *p_v2,
>>  			     void (*pfn)(osm_subn_t *, void *))
>> @@ -324,7 +329,7 @@ static const opt_rec_t opt_tbl[] = {
>>  	{ "hop_weights_file", OPT_OFFSET(hop_weights_file),
> opts_parse_charp, NULL, 0 },
>>  	{ "port_profile_switch_nodes",
> OPT_OFFSET(port_profile_switch_nodes), opts_parse_boolean, NULL, 1 },
>>  	{ "sweep_on_trap", OPT_OFFSET(sweep_on_trap),
> opts_parse_boolean, NULL, 1 },
>> -	{ "routing_engine", OPT_OFFSET(routing_engine_names),
> opts_parse_charp, NULL, 0 },
>> +	{ "routing_engine", OPT_OFFSET(routing_engine_names),
> opts_parse_charp, opts_setup_routing_engine, 1 },
>>  	{ "connect_roots", OPT_OFFSET(connect_roots),
> opts_parse_boolean, NULL, 1 },
>>  	{ "use_ucast_cache", OPT_OFFSET(use_ucast_cache),
> opts_parse_boolean, NULL, 1 },
>>  	{ "log_file", OPT_OFFSET(log_file), opts_parse_charp, NULL, 0 },
>> diff --git a/opensm/opensm/osm_ucast_mgr.c
> b/opensm/opensm/osm_ucast_mgr.c
>> index 39d825c..d6294ac 100644
>> --- a/opensm/opensm/osm_ucast_mgr.c
>> +++ b/opensm/opensm/osm_ucast_mgr.c
>> @@ -998,8 +998,23 @@ int osm_ucast_mgr_process(IN osm_ucast_mgr_t *
> p_mgr)
>>  
>>  	p_osm->routing_engine_used = OSM_ROUTING_ENGINE_TYPE_NONE;
>>  	while (p_routing_eng) {
>> +		if (!p_routing_eng->initialized &&
>> +			p_routing_eng->setup(p_routing_eng, p_osm)) {
>> +			OSM_LOG(p_mgr->p_log, OSM_LOG_ERROR,
>> +				"ERR 3A0F: setup of routing engine
> \'%s\' failed\n",
>> +					p_routing_eng->name);
>> +					p_routing_eng =
> p_routing_eng->next;
>> +					continue;
>> +		}
>> +		OSM_LOG(p_mgr->p_log, OSM_LOG_INFO,
>> +			"\'%s\' routing engine set up\n",
> p_routing_eng->name);
>> +		p_routing_eng->initialized = 1;
>>  		if (!ucast_mgr_route(p_routing_eng, p_osm))
>>  			break;
>> +		/* delete unused routing engine */
>> +		if (p_routing_eng->delete)
>> +			p_routing_eng->delete(p_routing_eng->context);
>> +		p_routing_eng->initialized = 0;
>>  		p_routing_eng = p_routing_eng->next;
>>  	}
>>  
>> @@ -1011,6 +1026,19 @@ int osm_ucast_mgr_process(IN osm_ucast_mgr_t *
> p_mgr)
>>  		p_osm->routing_engine_used =
> OSM_ROUTING_ENGINE_TYPE_MINHOP;
>>  	}
>>  
>> +	/* if for some reason different routing engine is used */
>> +	/* cleanup last unused routing engine */
>> +	p_routing_eng = p_osm->last_routing_engine;
>> +	if (p_routing_eng) {
>> +			if (p_routing_eng->initialized &&
>> +					p_routing_eng->delete &&
>> +					p_osm->routing_engine_used !=
>> +
> osm_routing_engine_type(p_routing_eng->name))
>> +
> p_routing_eng->delete(p_routing_eng->context);
>> +			free(p_routing_eng);
>> +			p_osm->last_routing_engine = NULL;
>> +	}
>> +
>>  	OSM_LOG(p_mgr->p_log, OSM_LOG_INFO,
>>  		"%s tables configured on all switches\n",
>>
> osm_routing_engine_type_str(p_osm->routing_engine_used));
> 
> --
> To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
> the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> 

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

RE: [PATCH v3] opensm: support routing engine update

[Slava Strebkov]

HI!
That was my misunderstanding - upon heavy sweep SM will try to load
routing engines as defined in the SM.conf file (ftree & updn - in that
order).
So ftree will be loaded when switch comes back from reboot.

Slava

-----Original Message-----
From: linux-rdma-owner-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
[mailto:linux-rdma-owner-u79uwXL29TY76Z2rM5mHXA@public.gmane.org] On Behalf Of Yevgeny Kliteynik
Sent: Monday, December 07, 2009 10:22 AM
To: Slava Strebkov
Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
Subject: Re: [PATCH v3] opensm: support routing engine update

Slava,

Slava Strebkov wrote:
> Hi Yevgeny,
> In that case SM will use updn and will not come back to ftree
> automatically.

I think that this is a bad thing.

I wouldn't want *temporary* change of fabric to cause
*permanent* change of SM mode of operation. Such changes
do happen, and I'd prefer SM to continue functioning
in accordance to the user's configuration once the
fabric is settled again.

I do see the cases were the change that you propose is
beneficial - if fabric topology doesn't fits the chosen
routing, SM will waste time on retrying the wrong routing
at every heavy sweep, but this happens due to suboptimal
SM configuration and not as a result of some event that
user has no control of.

-- Yevgeny


 
> Slava
> 
> -----Original Message-----
> From: linux-rdma-owner-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
> [mailto:linux-rdma-owner-u79uwXL29TY76Z2rM5mHXA@public.gmane.org] On Behalf Of Yevgeny
Kliteynik
> Sent: Sunday, December 06, 2009 6:03 PM
> To: Slava Strebkov
> Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
> Subject: Re: [PATCH v3] opensm: support routing engine update
> 
> Slava,
> 
> Slava Strebkov wrote:
>> setup routing engine when in use and delete when failed.
>> setup routing engine before use.
>> delete resources when routing algorithm fails.
>> this will save allocation for routing algorithms that are not used.
> 
> Suppose a user runs SM with ftree & updn routings (in that order),
> and SM manages to route the fabric with ftree. At some point 
> some switch reboots and causes ftree to fail and SM routes the
> fabric with updn.
> Does this mean that ftree will be removed from the list, and
> when the switch comes back, SM won't try ftree any more?
> 
> -- Yevgeny
>  
>> Signed-off-by: Slava Strebkov <slavas-smomgflXvOZWk0Htik3J/w@public.gmane.org>
>> ---
>>  opensm/include/opensm/osm_opensm.h |    5 +++
>>  opensm/opensm/osm_opensm.c         |   57
> +++++++++++++++++++++++++++++++-----
>>  opensm/opensm/osm_subnet.c         |    7 ++++-
>>  opensm/opensm/osm_ucast_mgr.c      |   28 +++++++++++++++++
>>  4 files changed, 88 insertions(+), 9 deletions(-)
>>
>> diff --git a/opensm/include/opensm/osm_opensm.h
> b/opensm/include/opensm/osm_opensm.h
>> index c121be4..ca0fddb 100644
>> --- a/opensm/include/opensm/osm_opensm.h
>> +++ b/opensm/include/opensm/osm_opensm.h
>> @@ -109,6 +109,7 @@ typedef enum _osm_routing_engine_type {
>>  } osm_routing_engine_type_t;
>>  /***********/
>>  
>> +struct osm_opensm;
>>  /****s* OpenSM: OpenSM/osm_routing_engine
>>  * NAME
>>  *	struct osm_routing_engine
>> @@ -122,6 +123,8 @@ typedef enum _osm_routing_engine_type {
>>  struct osm_routing_engine {
>>  	const char *name;
>>  	void *context;
>> +	int initialized;
>> +	int (*setup) (struct osm_routing_engine *re, struct osm_opensm
> *p_osm);
>>  	int (*build_lid_matrices) (void *context);
>>  	int (*ucast_build_fwd_tables) (void *context);
>>  	void (*ucast_dump_tables) (void *context);
>> @@ -183,6 +186,7 @@ typedef struct osm_opensm {
>>  	cl_dispatcher_t disp;
>>  	cl_plock_t lock;
>>  	struct osm_routing_engine *routing_engine_list;
>> +	struct osm_routing_engine *last_routing_engine;
>>  	osm_routing_engine_type_t routing_engine_used;
>>  	osm_stats_t stats;
>>  	osm_console_t console;
>> @@ -522,6 +526,7 @@ extern volatile unsigned int osm_exit_flag;
>>  * DESCRIPTION
>>  *  Set to one to cause all threads to leave
>>  *********/
>> +void osm_update_routing_engines(osm_opensm_t *osm, const char
> *engine_names);
>>  
>>  END_C_DECLS
>>  #endif				/* _OSM_OPENSM_H_ */
>> diff --git a/opensm/opensm/osm_opensm.c b/opensm/opensm/osm_opensm.c
>> index 50d1349..f90584d 100644
>> --- a/opensm/opensm/osm_opensm.c
>> +++ b/opensm/opensm/osm_opensm.c
>> @@ -169,14 +169,7 @@ static void setup_routing_engine(osm_opensm_t
> *osm, const char *name)
>>  			memset(re, 0, sizeof(struct
> osm_routing_engine));
>>  
>>  			re->name = m->name;
>> -			if (m->setup(re, osm)) {
>> -				OSM_LOG(&osm->log, OSM_LOG_VERBOSE,
>> -					"setup of routing"
>> -					" engine \'%s\' failed\n",
> name);
>> -				return;
>> -			}
>> -			OSM_LOG(&osm->log, OSM_LOG_DEBUG,
>> -				"\'%s\' routing engine set up\n",
> re->name);
>> +			re->setup = m->setup;
>>  			append_routing_engine(osm, re);
>>  			return;
>>  		}
>> @@ -236,6 +229,54 @@ static void destroy_routing_engines(osm_opensm_t
> *osm)
>>  			r->delete(r->context);
>>  		free(r);
>>  	}
>> +    osm->routing_engine_list = NULL;
>> +}
>> +
>> +static void update_routing_engine(
>> +       struct osm_routing_engine *cur,
>> +       struct osm_routing_engine *last)
>> +{
>> +	struct osm_routing_engine *next = cur->next;
>> +	if (!last)
>> +		return; /* no last routing engine */
>> +	memcpy(cur, last, sizeof(*cur));
>> +	/* restore next */
>> +	cur->next = next;
>> +}
>> +
>> +void osm_update_routing_engines(osm_opensm_t *osm, const char
> *engine_names)
>> +{
>> +	struct osm_routing_engine *r, *l;
>> +	/* find used routing engine and save as last */
>> +	l = r = osm->routing_engine_list;
>> +	if (r && osm->routing_engine_used ==
> osm_routing_engine_type(r->name)) {
>> +		osm->last_routing_engine = r;
>> +		osm->routing_engine_list = r->next;
>> +	}
>> +	else while ((r = r->next)) {
>> +		if (osm->routing_engine_used ==
>> +			osm_routing_engine_type(r->name)) {
>> +				osm->last_routing_engine = r;
>> +				l->next = r->next;
>> +				break;
>> +		}
>> +		l = r;
>> +	}
>> +	/* cleanup prev routing engine list and replace with current
> list */
>> +	destroy_routing_engines(osm);
>> +	setup_routing_engines(osm, engine_names);
>> +	/* check if last routing engine exist in new list and update
> callbacks */
>> +	r = osm->routing_engine_list;
>> +	while (r) {
>> +		if (osm->routing_engine_used ==
>> +			osm_routing_engine_type(r->name)) {
>> +				update_routing_engine(r,
> osm->last_routing_engine);
>> +				free(osm->last_routing_engine);
>> +				osm->last_routing_engine = NULL;
>> +				break;
>> +		}
>> +	r = r->next;
>> +	}
>>  }
>>  
>>
>
/**********************************************************************
>> diff --git a/opensm/opensm/osm_subnet.c b/opensm/opensm/osm_subnet.c
>> index 8d63a75..742ae64 100644
>> --- a/opensm/opensm/osm_subnet.c
>> +++ b/opensm/opensm/osm_subnet.c
>> @@ -152,6 +152,11 @@ static void opts_setup_sm_priority(osm_subn_t
> *p_subn, void *p_val)
>>  	osm_set_sm_priority(p_sm, sm_priority);
>>  }
>>  
>> +static void opts_setup_routing_engine(osm_subn_t *p_subn, void
> *p_val)
>> +{
>> +	osm_update_routing_engines(p_subn->p_osm, p_val);
>> +}
>> +
>>  static void opts_parse_net64(IN osm_subn_t *p_subn, IN char *p_key,
>>  			     IN char *p_val_str, void *p_v1, void *p_v2,
>>  			     void (*pfn)(osm_subn_t *, void *))
>> @@ -324,7 +329,7 @@ static const opt_rec_t opt_tbl[] = {
>>  	{ "hop_weights_file", OPT_OFFSET(hop_weights_file),
> opts_parse_charp, NULL, 0 },
>>  	{ "port_profile_switch_nodes",
> OPT_OFFSET(port_profile_switch_nodes), opts_parse_boolean, NULL, 1 },
>>  	{ "sweep_on_trap", OPT_OFFSET(sweep_on_trap),
> opts_parse_boolean, NULL, 1 },
>> -	{ "routing_engine", OPT_OFFSET(routing_engine_names),
> opts_parse_charp, NULL, 0 },
>> +	{ "routing_engine", OPT_OFFSET(routing_engine_names),
> opts_parse_charp, opts_setup_routing_engine, 1 },
>>  	{ "connect_roots", OPT_OFFSET(connect_roots),
> opts_parse_boolean, NULL, 1 },
>>  	{ "use_ucast_cache", OPT_OFFSET(use_ucast_cache),
> opts_parse_boolean, NULL, 1 },
>>  	{ "log_file", OPT_OFFSET(log_file), opts_parse_charp, NULL, 0 },
>> diff --git a/opensm/opensm/osm_ucast_mgr.c
> b/opensm/opensm/osm_ucast_mgr.c
>> index 39d825c..d6294ac 100644
>> --- a/opensm/opensm/osm_ucast_mgr.c
>> +++ b/opensm/opensm/osm_ucast_mgr.c
>> @@ -998,8 +998,23 @@ int osm_ucast_mgr_process(IN osm_ucast_mgr_t *
> p_mgr)
>>  
>>  	p_osm->routing_engine_used = OSM_ROUTING_ENGINE_TYPE_NONE;
>>  	while (p_routing_eng) {
>> +		if (!p_routing_eng->initialized &&
>> +			p_routing_eng->setup(p_routing_eng, p_osm)) {
>> +			OSM_LOG(p_mgr->p_log, OSM_LOG_ERROR,
>> +				"ERR 3A0F: setup of routing engine
> \'%s\' failed\n",
>> +					p_routing_eng->name);
>> +					p_routing_eng =
> p_routing_eng->next;
>> +					continue;
>> +		}
>> +		OSM_LOG(p_mgr->p_log, OSM_LOG_INFO,
>> +			"\'%s\' routing engine set up\n",
> p_routing_eng->name);
>> +		p_routing_eng->initialized = 1;
>>  		if (!ucast_mgr_route(p_routing_eng, p_osm))
>>  			break;
>> +		/* delete unused routing engine */
>> +		if (p_routing_eng->delete)
>> +			p_routing_eng->delete(p_routing_eng->context);
>> +		p_routing_eng->initialized = 0;
>>  		p_routing_eng = p_routing_eng->next;
>>  	}
>>  
>> @@ -1011,6 +1026,19 @@ int osm_ucast_mgr_process(IN osm_ucast_mgr_t *
> p_mgr)
>>  		p_osm->routing_engine_used =
> OSM_ROUTING_ENGINE_TYPE_MINHOP;
>>  	}
>>  
>> +	/* if for some reason different routing engine is used */
>> +	/* cleanup last unused routing engine */
>> +	p_routing_eng = p_osm->last_routing_engine;
>> +	if (p_routing_eng) {
>> +			if (p_routing_eng->initialized &&
>> +					p_routing_eng->delete &&
>> +					p_osm->routing_engine_used !=
>> +
> osm_routing_engine_type(p_routing_eng->name))
>> +
> p_routing_eng->delete(p_routing_eng->context);
>> +			free(p_routing_eng);
>> +			p_osm->last_routing_engine = NULL;
>> +	}
>> +
>>  	OSM_LOG(p_mgr->p_log, OSM_LOG_INFO,
>>  		"%s tables configured on all switches\n",
>>
> osm_routing_engine_type_str(p_osm->routing_engine_used));
> 
> --
> To unsubscribe from this list: send the line "unsubscribe linux-rdma"
in
> the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> 

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v3] opensm: support routing engine update

[Eli Dorfman (Voltaire)]

Slava Strebkov wrote:
> HI!
> That was my misunderstanding - upon heavy sweep SM will try to load
> routing engines as defined in the SM.conf file (ftree & updn - in that
> order).
> So ftree will be loaded when switch comes back from reboot.
> 
> Slava
> 
> -----Original Message-----
> From: linux-rdma-owner-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
> [mailto:linux-rdma-owner-u79uwXL29TY76Z2rM5mHXA@public.gmane.org] On Behalf Of Yevgeny Kliteynik
> Sent: Monday, December 07, 2009 10:22 AM
> To: Slava Strebkov
> Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
> Subject: Re: [PATCH v3] opensm: support routing engine update
> 
> Slava,
> 
> Slava Strebkov wrote:
>> Hi Yevgeny,
>> In that case SM will use updn and will not come back to ftree
>> automatically.
> 
> I think that this is a bad thing.
> 
> I wouldn't want *temporary* change of fabric to cause
> *permanent* change of SM mode of operation. Such changes
> do happen, and I'd prefer SM to continue functioning
> in accordance to the user's configuration once the
> fabric is settled again.
> 
> I do see the cases were the change that you propose is
> beneficial - if fabric topology doesn't fits the chosen
> routing, SM will waste time on retrying the wrong routing
> at every heavy sweep, but this happens due to suboptimal
> SM configuration and not as a result of some event that
> user has no control of.

For every heavy sweep the SM will try to configure the routing engines as specified in the conf file.
So when a switch goes up and ftree configuration is valid, the SM will configure ftree instead of updn.

Eli

> 
> -- Yevgeny
> 
> 
>  
>> Slava
>>
>> -----Original Message-----
>> From: linux-rdma-owner-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
>> [mailto:linux-rdma-owner-u79uwXL29TY76Z2rM5mHXA@public.gmane.org] On Behalf Of Yevgeny
> Kliteynik
>> Sent: Sunday, December 06, 2009 6:03 PM
>> To: Slava Strebkov
>> Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
>> Subject: Re: [PATCH v3] opensm: support routing engine update
>>
>> Slava,
>>
>> Slava Strebkov wrote:
>>> setup routing engine when in use and delete when failed.
>>> setup routing engine before use.
>>> delete resources when routing algorithm fails.
>>> this will save allocation for routing algorithms that are not used.
>> Suppose a user runs SM with ftree & updn routings (in that order),
>> and SM manages to route the fabric with ftree. At some point 
>> some switch reboots and causes ftree to fail and SM routes the
>> fabric with updn.
>> Does this mean that ftree will be removed from the list, and
>> when the switch comes back, SM won't try ftree any more?
>>
>> -- Yevgeny
>>  
>>> Signed-off-by: Slava Strebkov <slavas-smomgflXvOZWk0Htik3J/w@public.gmane.org>
>>> ---
>>>  opensm/include/opensm/osm_opensm.h |    5 +++
>>>  opensm/opensm/osm_opensm.c         |   57
>> +++++++++++++++++++++++++++++++-----
>>>  opensm/opensm/osm_subnet.c         |    7 ++++-
>>>  opensm/opensm/osm_ucast_mgr.c      |   28 +++++++++++++++++
>>>  4 files changed, 88 insertions(+), 9 deletions(-)
>>>
>>> diff --git a/opensm/include/opensm/osm_opensm.h
>> b/opensm/include/opensm/osm_opensm.h
>>> index c121be4..ca0fddb 100644
>>> --- a/opensm/include/opensm/osm_opensm.h
>>> +++ b/opensm/include/opensm/osm_opensm.h
>>> @@ -109,6 +109,7 @@ typedef enum _osm_routing_engine_type {
>>>  } osm_routing_engine_type_t;
>>>  /***********/
>>>  
>>> +struct osm_opensm;
>>>  /****s* OpenSM: OpenSM/osm_routing_engine
>>>  * NAME
>>>  *	struct osm_routing_engine
>>> @@ -122,6 +123,8 @@ typedef enum _osm_routing_engine_type {
>>>  struct osm_routing_engine {
>>>  	const char *name;
>>>  	void *context;
>>> +	int initialized;
>>> +	int (*setup) (struct osm_routing_engine *re, struct osm_opensm
>> *p_osm);
>>>  	int (*build_lid_matrices) (void *context);
>>>  	int (*ucast_build_fwd_tables) (void *context);
>>>  	void (*ucast_dump_tables) (void *context);
>>> @@ -183,6 +186,7 @@ typedef struct osm_opensm {
>>>  	cl_dispatcher_t disp;
>>>  	cl_plock_t lock;
>>>  	struct osm_routing_engine *routing_engine_list;
>>> +	struct osm_routing_engine *last_routing_engine;
>>>  	osm_routing_engine_type_t routing_engine_used;
>>>  	osm_stats_t stats;
>>>  	osm_console_t console;
>>> @@ -522,6 +526,7 @@ extern volatile unsigned int osm_exit_flag;
>>>  * DESCRIPTION
>>>  *  Set to one to cause all threads to leave
>>>  *********/
>>> +void osm_update_routing_engines(osm_opensm_t *osm, const char
>> *engine_names);
>>>  
>>>  END_C_DECLS
>>>  #endif				/* _OSM_OPENSM_H_ */
>>> diff --git a/opensm/opensm/osm_opensm.c b/opensm/opensm/osm_opensm.c
>>> index 50d1349..f90584d 100644
>>> --- a/opensm/opensm/osm_opensm.c
>>> +++ b/opensm/opensm/osm_opensm.c
>>> @@ -169,14 +169,7 @@ static void setup_routing_engine(osm_opensm_t
>> *osm, const char *name)
>>>  			memset(re, 0, sizeof(struct
>> osm_routing_engine));
>>>  
>>>  			re->name = m->name;
>>> -			if (m->setup(re, osm)) {
>>> -				OSM_LOG(&osm->log, OSM_LOG_VERBOSE,
>>> -					"setup of routing"
>>> -					" engine \'%s\' failed\n",
>> name);
>>> -				return;
>>> -			}
>>> -			OSM_LOG(&osm->log, OSM_LOG_DEBUG,
>>> -				"\'%s\' routing engine set up\n",
>> re->name);
>>> +			re->setup = m->setup;
>>>  			append_routing_engine(osm, re);
>>>  			return;
>>>  		}
>>> @@ -236,6 +229,54 @@ static void destroy_routing_engines(osm_opensm_t
>> *osm)
>>>  			r->delete(r->context);
>>>  		free(r);
>>>  	}
>>> +    osm->routing_engine_list = NULL;
>>> +}
>>> +
>>> +static void update_routing_engine(
>>> +       struct osm_routing_engine *cur,
>>> +       struct osm_routing_engine *last)
>>> +{
>>> +	struct osm_routing_engine *next = cur->next;
>>> +	if (!last)
>>> +		return; /* no last routing engine */
>>> +	memcpy(cur, last, sizeof(*cur));
>>> +	/* restore next */
>>> +	cur->next = next;
>>> +}
>>> +
>>> +void osm_update_routing_engines(osm_opensm_t *osm, const char
>> *engine_names)
>>> +{
>>> +	struct osm_routing_engine *r, *l;
>>> +	/* find used routing engine and save as last */
>>> +	l = r = osm->routing_engine_list;
>>> +	if (r && osm->routing_engine_used ==
>> osm_routing_engine_type(r->name)) {
>>> +		osm->last_routing_engine = r;
>>> +		osm->routing_engine_list = r->next;
>>> +	}
>>> +	else while ((r = r->next)) {
>>> +		if (osm->routing_engine_used ==
>>> +			osm_routing_engine_type(r->name)) {
>>> +				osm->last_routing_engine = r;
>>> +				l->next = r->next;
>>> +				break;
>>> +		}
>>> +		l = r;
>>> +	}
>>> +	/* cleanup prev routing engine list and replace with current
>> list */
>>> +	destroy_routing_engines(osm);
>>> +	setup_routing_engines(osm, engine_names);
>>> +	/* check if last routing engine exist in new list and update
>> callbacks */
>>> +	r = osm->routing_engine_list;
>>> +	while (r) {
>>> +		if (osm->routing_engine_used ==
>>> +			osm_routing_engine_type(r->name)) {
>>> +				update_routing_engine(r,
>> osm->last_routing_engine);
>>> +				free(osm->last_routing_engine);
>>> +				osm->last_routing_engine = NULL;
>>> +				break;
>>> +		}
>>> +	r = r->next;
>>> +	}
>>>  }
>>>  
>>>
> /**********************************************************************
>>> diff --git a/opensm/opensm/osm_subnet.c b/opensm/opensm/osm_subnet.c
>>> index 8d63a75..742ae64 100644
>>> --- a/opensm/opensm/osm_subnet.c
>>> +++ b/opensm/opensm/osm_subnet.c
>>> @@ -152,6 +152,11 @@ static void opts_setup_sm_priority(osm_subn_t
>> *p_subn, void *p_val)
>>>  	osm_set_sm_priority(p_sm, sm_priority);
>>>  }
>>>  
>>> +static void opts_setup_routing_engine(osm_subn_t *p_subn, void
>> *p_val)
>>> +{
>>> +	osm_update_routing_engines(p_subn->p_osm, p_val);
>>> +}
>>> +
>>>  static void opts_parse_net64(IN osm_subn_t *p_subn, IN char *p_key,
>>>  			     IN char *p_val_str, void *p_v1, void *p_v2,
>>>  			     void (*pfn)(osm_subn_t *, void *))
>>> @@ -324,7 +329,7 @@ static const opt_rec_t opt_tbl[] = {
>>>  	{ "hop_weights_file", OPT_OFFSET(hop_weights_file),
>> opts_parse_charp, NULL, 0 },
>>>  	{ "port_profile_switch_nodes",
>> OPT_OFFSET(port_profile_switch_nodes), opts_parse_boolean, NULL, 1 },
>>>  	{ "sweep_on_trap", OPT_OFFSET(sweep_on_trap),
>> opts_parse_boolean, NULL, 1 },
>>> -	{ "routing_engine", OPT_OFFSET(routing_engine_names),
>> opts_parse_charp, NULL, 0 },
>>> +	{ "routing_engine", OPT_OFFSET(routing_engine_names),
>> opts_parse_charp, opts_setup_routing_engine, 1 },
>>>  	{ "connect_roots", OPT_OFFSET(connect_roots),
>> opts_parse_boolean, NULL, 1 },
>>>  	{ "use_ucast_cache", OPT_OFFSET(use_ucast_cache),
>> opts_parse_boolean, NULL, 1 },
>>>  	{ "log_file", OPT_OFFSET(log_file), opts_parse_charp, NULL, 0 },
>>> diff --git a/opensm/opensm/osm_ucast_mgr.c
>> b/opensm/opensm/osm_ucast_mgr.c
>>> index 39d825c..d6294ac 100644
>>> --- a/opensm/opensm/osm_ucast_mgr.c
>>> +++ b/opensm/opensm/osm_ucast_mgr.c
>>> @@ -998,8 +998,23 @@ int osm_ucast_mgr_process(IN osm_ucast_mgr_t *
>> p_mgr)
>>>  
>>>  	p_osm->routing_engine_used = OSM_ROUTING_ENGINE_TYPE_NONE;
>>>  	while (p_routing_eng) {
>>> +		if (!p_routing_eng->initialized &&
>>> +			p_routing_eng->setup(p_routing_eng, p_osm)) {
>>> +			OSM_LOG(p_mgr->p_log, OSM_LOG_ERROR,
>>> +				"ERR 3A0F: setup of routing engine
>> \'%s\' failed\n",
>>> +					p_routing_eng->name);
>>> +					p_routing_eng =
>> p_routing_eng->next;
>>> +					continue;
>>> +		}
>>> +		OSM_LOG(p_mgr->p_log, OSM_LOG_INFO,
>>> +			"\'%s\' routing engine set up\n",
>> p_routing_eng->name);
>>> +		p_routing_eng->initialized = 1;
>>>  		if (!ucast_mgr_route(p_routing_eng, p_osm))
>>>  			break;
>>> +		/* delete unused routing engine */
>>> +		if (p_routing_eng->delete)
>>> +			p_routing_eng->delete(p_routing_eng->context);
>>> +		p_routing_eng->initialized = 0;
>>>  		p_routing_eng = p_routing_eng->next;
>>>  	}
>>>  
>>> @@ -1011,6 +1026,19 @@ int osm_ucast_mgr_process(IN osm_ucast_mgr_t *
>> p_mgr)
>>>  		p_osm->routing_engine_used =
>> OSM_ROUTING_ENGINE_TYPE_MINHOP;
>>>  	}
>>>  
>>> +	/* if for some reason different routing engine is used */
>>> +	/* cleanup last unused routing engine */
>>> +	p_routing_eng = p_osm->last_routing_engine;
>>> +	if (p_routing_eng) {
>>> +			if (p_routing_eng->initialized &&
>>> +					p_routing_eng->delete &&
>>> +					p_osm->routing_engine_used !=
>>> +
>> osm_routing_engine_type(p_routing_eng->name))
>>> +
>> p_routing_eng->delete(p_routing_eng->context);
>>> +			free(p_routing_eng);
>>> +			p_osm->last_routing_engine = NULL;
>>> +	}
>>> +
>>>  	OSM_LOG(p_mgr->p_log, OSM_LOG_INFO,
>>>  		"%s tables configured on all switches\n",
>>>
>> osm_routing_engine_type_str(p_osm->routing_engine_used));
>>
>> --
>> To unsubscribe from this list: send the line "unsubscribe linux-rdma"
> in
>> the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
>> More majordomo info at  http://vger.kernel.org/majordomo-info.html
>>
> 
> --
> To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
> the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> --
> To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
> the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v3] opensm: support routing engine update

[Yevgeny Kliteynik]

On 14/Feb/10 12:18, Eli Dorfman (Voltaire) wrote:
> Slava Strebkov wrote:
>> HI!
>> That was my misunderstanding - upon heavy sweep SM will try to load
>> routing engines as defined in the SM.conf file (ftree&  updn - in that
>> order).
>> So ftree will be loaded when switch comes back from reboot.
>>
>> Slava
>>
>> -----Original Message-----
>> From: linux-rdma-owner-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
>> [mailto:linux-rdma-owner-u79uwXL29TY76Z2rM5mHXA@public.gmane.org] On Behalf Of Yevgeny Kliteynik
>> Sent: Monday, December 07, 2009 10:22 AM
>> To: Slava Strebkov
>> Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
>> Subject: Re: [PATCH v3] opensm: support routing engine update
>>
>> Slava,
>>
>> Slava Strebkov wrote:
>>> Hi Yevgeny,
>>> In that case SM will use updn and will not come back to ftree
>>> automatically.
>>
>> I think that this is a bad thing.
>>
>> I wouldn't want *temporary* change of fabric to cause
>> *permanent* change of SM mode of operation. Such changes
>> do happen, and I'd prefer SM to continue functioning
>> in accordance to the user's configuration once the
>> fabric is settled again.
>>
>> I do see the cases were the change that you propose is
>> beneficial - if fabric topology doesn't fits the chosen
>> routing, SM will waste time on retrying the wrong routing
>> at every heavy sweep, but this happens due to suboptimal
>> SM configuration and not as a result of some event that
>> user has no control of.
>
> For every heavy sweep the SM will try to configure the routing engines as specified in the conf file.
> So when a switch goes up and ftree configuration is valid, the SM will configure ftree instead of updn.

OK then, sounds good.

-- Yevgeny
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html