From: Albert L. Chu <chu11-i2BcT+NCU+M@public.gmane.org>
Date: Mon, 7 Feb 2011 16:53:39 -0800
Subject: [PATCH] Support remote guid sorting
Message-Id: <1297379237.18394.291.camel-akkeaxHeDKRliZ7u+bvwcg@public.gmane.org>
Mime-Version: 1.0
Content-Transfer-Encoding: 7bit


Signed-off-by: Albert L. Chu <chu11-i2BcT+NCU+M@public.gmane.org>
---
 include/opensm/osm_subnet.h |    4 ++++
 include/opensm/osm_switch.h |    6 +++++-
 man/opensm.8.in             |    6 ++++++
 opensm/main.c               |    8 ++++++++
 opensm/osm_dump.c           |    3 ++-
 opensm/osm_subnet.c         |    7 +++++++
 opensm/osm_switch.c         |   26 +++++++++++++++++++++++++-
 opensm/osm_ucast_mgr.c      |    3 ++-
 8 files changed, 59 insertions(+), 4 deletions(-)

diff --git a/include/opensm/osm_subnet.h b/include/opensm/osm_subnet.h
index 59f877e..589e96c 100644
--- a/include/opensm/osm_subnet.h
+++ b/include/opensm/osm_subnet.h
@@ -200,6 +200,7 @@ typedef struct osm_subn_opt {
 	char *cn_guid_file;
 	char *io_guid_file;
 	boolean_t port_shifting;
+	boolean_t remote_guid_sorting;
 	uint16_t max_reverse_hops;
 	char *ids_guid_file;
 	char *guid_routing_order_file;
@@ -422,6 +423,9 @@ typedef struct osm_subn_opt {
 *	port_shifting
 *		This option will turn on port_shifting in routing.
 *
+*	remote_guid_sorting
+*		This option will turn on remote_guid_sorting in routing.
+*
 *	ids_guid_file
 *		Name of the file that contains list of ids which should be
 *		used by Up/Down algorithm instead of node GUIDs
diff --git a/include/opensm/osm_switch.h b/include/opensm/osm_switch.h
index 8eae119..aef45cb 100644
--- a/include/opensm/osm_switch.h
+++ b/include/opensm/osm_switch.h
@@ -920,7 +920,8 @@ uint8_t osm_switch_recommend_path(IN const osm_switch_t * p_sw,
 				  IN boolean_t ignore_existing,
 				  IN boolean_t routing_for_lmc,
 				  IN boolean_t dor,
-				  IN boolean_t port_shifting);
+				  IN boolean_t port_shifting,
+				  IN boolean_t remote_guid_sorting);
 /*
 * PARAMETERS
 *	p_sw
@@ -959,6 +960,9 @@ uint8_t osm_switch_recommend_path(IN const osm_switch_t * p_sw,
 *	port_shifting
 *		[in] If TRUE, port_shifting will be done.
 *
+*	remote_guid_sorting
+*		[in] If TRUE, remote_guid_sorting will be done.
+*
 * RETURN VALUE
 *	Returns the recommended port on which to route this LID.
 *
diff --git a/man/opensm.8.in b/man/opensm.8.in
index db48d52..decaee7 100644
--- a/man/opensm.8.in
+++ b/man/opensm.8.in
@@ -216,6 +216,12 @@ congest with other routes due to algorithmically unchanging traffic
 patterns.  This routing option will "shift" routing around in an
 attempt to alleviate this problem.
 .TP
+\fB\-\-remote\-guid\-sorting\fR
+This option enables a feature called \fBremote guid sorting\fR.  In some
+fabrics, switches may be cabled in an inconsistent fashion.  This option
+may alleviate those issues by sorting remote guids before routing,
+making remote destinations appear to be ordered consistently.
+.TP
 \fB\-H\fR, \fB\-\-max_reverse_hops\fR <file name>
 Set the maximum number of reverse hops an I/O node is allowed
 to make. A reverse hop is the use of a switch the wrong way around.
diff --git a/opensm/main.c b/opensm/main.c
index abb32ec..91ae940 100644
--- a/opensm/main.c
+++ b/opensm/main.c
@@ -226,6 +226,9 @@ static void show_usage(void)
 	printf("--port-shifting\n"
 	       "          Attempt to shift port routes around to remove alignment problems\n"
 	       "          in routing tables\n\n");
+	printf("--remote-guid-sorting\n"
+	       "          Sort ports by remote port guid before routing to alleviate\n"
+	       "          problems with inconsistent cabling across a fabric\n\n");
 	printf("--max_reverse_hops, -H <hop_count>\n"
 	       "          Set the max number of hops the wrong way around\n"
 	       "          an I/O node is allowed to do (connectivity for I/O nodes on top swithces)\n\n");
@@ -605,6 +608,7 @@ int main(int argc, char *argv[])
 		{"cn_guid_file", 1, NULL, 'u'},
 		{"io_guid_file", 1, NULL, 'G'},
 		{"port-shifting", 0, NULL, 11},
+		{"remote-guid-sorting", 0, NULL, 13},
 		{"max_reverse_hops", 1, NULL, 'H'},
 		{"ids_guid_file", 1, NULL, 'm'},
 		{"guid_routing_order_file", 1, NULL, 'X'},
@@ -945,6 +949,10 @@ int main(int argc, char *argv[])
 			opt.port_shifting = TRUE;
 			printf(" Port Shifting is on\n");
 			break;
+		case 13:
+			opt.remote_guid_sorting = TRUE;
+			printf(" Remote Guid Sorting is on\n");
+			break;
 		case 'H':
 			opt.max_reverse_hops = atoi(optarg);
 			printf(" Max Reverse Hops: %d\n", opt.max_reverse_hops);
diff --git a/opensm/osm_dump.c b/opensm/osm_dump.c
index a1ff168..bfe63c3 100644
--- a/opensm/osm_dump.c
+++ b/opensm/osm_dump.c
@@ -221,7 +221,8 @@ static void dump_ucast_routes(cl_map_item_t * item, FILE * file, void *cxt)
 			/* No LMC Optimization */
 			best_port = osm_switch_recommend_path(p_sw, p_port,
 							      lid_ho, 1, TRUE,
-							      FALSE, dor, FALSE);
+							      FALSE, dor, FALSE,
+							      FALSE);
 			fprintf(file, "No %u hop path possible via port %u!",
 				best_hops, best_port);
 		}
diff --git a/opensm/osm_subnet.c b/opensm/osm_subnet.c
index c62192c..b2b219f 100644
--- a/opensm/osm_subnet.c
+++ b/opensm/osm_subnet.c
@@ -348,6 +348,7 @@ static const opt_rec_t opt_tbl[] = {
 	{ "cn_guid_file", OPT_OFFSET(cn_guid_file), opts_parse_charp, NULL, 0 },
 	{ "io_guid_file", OPT_OFFSET(io_guid_file), opts_parse_charp, NULL, 0 },
 	{ "port_shifting", OPT_OFFSET(port_shifting), opts_parse_boolean, NULL, 1 },
+	{ "remote_guid_sorting", OPT_OFFSET(remote_guid_sorting), opts_parse_boolean, NULL, 1 },
 	{ "max_reverse_hops", OPT_OFFSET(max_reverse_hops), opts_parse_uint16, NULL, 0 },
 	{ "ids_guid_file", OPT_OFFSET(ids_guid_file), opts_parse_charp, NULL, 0 },
 	{ "guid_routing_order_file", OPT_OFFSET(guid_routing_order_file), opts_parse_charp, NULL, 0 },
@@ -742,6 +743,7 @@ void osm_subn_set_default_opt(IN osm_subn_opt_t * p_opt)
 	p_opt->cn_guid_file = NULL;
 	p_opt->io_guid_file = NULL;
 	p_opt->port_shifting = FALSE;
+	p_opt->remote_guid_sorting = FALSE;
 	p_opt->max_reverse_hops = 0;
 	p_opt->ids_guid_file = NULL;
 	p_opt->guid_routing_order_file = NULL;
@@ -1447,6 +1449,11 @@ int osm_subn_output_conf(FILE *out, IN osm_subn_opt_t * p_opts)
 		p_opts->port_shifting ? "TRUE" : "FALSE");
 
 	fprintf(out,
+		"# Remote Guid Sorting (use FALSE if unsure)\n"
+		"remote_guid_sorting %s\n\n",
+		p_opts->remote_guid_sorting ? "TRUE" : "FALSE");
+
+	fprintf(out,
 		"# SA database file name\nsa_db_file %s\n\n",
 		p_opts->sa_db_file ? p_opts->sa_db_file : null_str);
 
diff --git a/opensm/osm_switch.c b/opensm/osm_switch.c
index f24d9ea..0aa0137 100644
--- a/opensm/osm_switch.c
+++ b/opensm/osm_switch.c
@@ -57,6 +57,7 @@ struct switch_port_path {
 	int found_sys_guid;
 	int found_node_guid;
 	uint32_t forwarded_to;
+	uint64_t remote_node_guid;
 };
 
 cl_status_t osm_switch_set_hops(IN osm_switch_t * p_sw, IN uint16_t lid_ho,
@@ -169,6 +170,19 @@ boolean_t osm_switch_get_lft_block(IN const osm_switch_t * p_sw,
 	return TRUE;
 }
 
+static int
+port_path_guid_cmp(IN const void *x, IN const void *y)
+{
+	struct switch_port_path *a = (struct switch_port_path *)x;
+	struct switch_port_path *b = (struct switch_port_path *)y;
+
+	if (a->remote_node_guid < b->remote_node_guid)
+		return -1;
+	if (a->remote_node_guid > b->remote_node_guid)
+		return 1;
+	return 0;
+}
+
 static struct osm_remote_node *
 switch_find_guid_common(IN const osm_switch_t * p_sw,
 			IN struct osm_remote_guids_count *r,
@@ -226,7 +240,8 @@ uint8_t osm_switch_recommend_path(IN const osm_switch_t * p_sw,
 				  IN boolean_t ignore_existing,
 				  IN boolean_t routing_for_lmc,
 				  IN boolean_t dor,
-				  IN boolean_t port_shifting)
+				  IN boolean_t port_shifting,
+				  IN boolean_t remote_guid_sorting)
 {
 	/*
 	   We support an enhanced LMC aware routing mode:
@@ -428,6 +443,7 @@ uint8_t osm_switch_recommend_path(IN const osm_switch_t * p_sw,
 					least_forwarded_to = 0;
 				}
 				found_sys_guid = 0;
+				found_node_guid = 0;
 			} else {	/* same sys found - try node */
 
 
@@ -463,6 +479,9 @@ uint8_t osm_switch_recommend_path(IN const osm_switch_t * p_sw,
 			port_paths[port_paths_count].forwarded_to = p_remote_guid->forwarded_to;
 		else
 			port_paths[port_paths_count].forwarded_to = 0;
+		p_rem_physp = osm_physp_get_remote(p_physp);
+		p_rem_node = osm_physp_get_node_ptr(p_rem_physp);
+		port_paths[port_paths_count].remote_node_guid = p_rem_node->node_info.node_guid;
 		port_paths_total_paths += check_count;
 		port_paths_count++;
 
@@ -490,6 +509,11 @@ uint8_t osm_switch_recommend_path(IN const osm_switch_t * p_sw,
 	if (port_found == FALSE)
 		return OSM_NO_PATH;
 
+	if (remote_guid_sorting && port_paths_count) {
+		qsort(port_paths, port_paths_count, sizeof(struct switch_port_path),
+		      port_path_guid_cmp);
+	}
+
 	if (port_shifting && port_paths_count) {
 		/* In the port_paths[] array, we now have all the ports that we
 		 * can route out of.  Using some shifting math below, possibly
diff --git a/opensm/osm_ucast_mgr.c b/opensm/osm_ucast_mgr.c
index d32eb60..a8982df 100644
--- a/opensm/osm_ucast_mgr.c
+++ b/opensm/osm_ucast_mgr.c
@@ -256,7 +256,8 @@ static void ucast_mgr_process_port(IN osm_ucast_mgr_t * p_mgr,
 					 p_mgr->p_subn->ignore_existing_lfts,
 					 p_mgr->p_subn->opt.lmc,
 					 p_mgr->is_dor,
-					 p_mgr->p_subn->opt.port_shifting);
+					 p_mgr->p_subn->opt.port_shifting,
+					 p_mgr->p_subn->opt.remote_guid_sorting);
 
 	if (port == OSM_NO_PATH) {
 		/* do not try to overwrite the ppro of non existing port ... */
-- 
1.5.4.5

