[PATCH v3 0/4] powerpc/pseries/vas: VAS/NXGZIP support with LPM

[PATCH v3 0/4] powerpc/pseries/vas: VAS/NXGZIP support with LPM

[Haren Myneni]

Virtual Accelerator Switchboard (VAS) is an engine stays on the
chip. So all windows opened on a specific engine belongs to VAS
the chip. The hypervisor expects the partition to close all
active windows on the sources system and reopen them after
migration on the destination machine.

This patch series adds VAS support with the partition migration.
When the migration initiates, the VAS migration handler will be
invoked before pseries_suspend() to close all active windows and
mark them in-active with VAS_WIN_MIGRATE_CLOSE status. Whereas
this migration handler is called after migration to reopen all
windows which has VAS_WIN_MIGRATE_CLOSE status and make them
active again. The user space gets paste instruction failure
when it sends requests on these in-active windows.

These patches depend on VAS/DLPAR support patch series

Changes in v2:
- Added new patch "Define global hv_cop_caps struct" to eliminate
  memory allocation failure during migration (suggestion by
  Nathan Lynch)

Changes in v3:
- Rebase on 5.17-rc4
- Naming changes for VAS capability struct elemets based on the V4 DLPAR
  support patch series.

Haren Myneni (4):
  powerpc/pseries/vas: Define global hv_cop_caps struct
  powerpc/pseries/vas: Modify reconfig open/close functions for
    migration
  powerpc/pseries/vas: Add VAS migration handler
  powerpc/pseries/vas: Disable window open during migration

 arch/powerpc/include/asm/vas.h            |   2 +
 arch/powerpc/platforms/pseries/mobility.c |   5 +
 arch/powerpc/platforms/pseries/vas.c      | 234 +++++++++++++++++-----
 arch/powerpc/platforms/pseries/vas.h      |   6 +
 4 files changed, 201 insertions(+), 46 deletions(-)

-- 
2.27.0

[PATCH v3 1/4] powerpc/pseries/vas: Define global hv_cop_caps struct

[Haren Myneni]

The coprocessor capabilities struct is used to get default and
QoS capabilities from the hypervisor during init, DLPAR event and
migration. So instead of allocating this struct for each event,
define global struct and reuse it which allows the migration code
to avoid adding an error path.

Also disable copy/paste feature flag if any capabilities HCALL
is failed.

Signed-off-by: Haren Myneni <haren@linux.ibm.com>
Acked-by: Nathan Lynch <nathanl@linux.ibm.com>
---
 arch/powerpc/platforms/pseries/vas.c | 47 ++++++++++++----------------
 1 file changed, 20 insertions(+), 27 deletions(-)

diff --git a/arch/powerpc/platforms/pseries/vas.c b/arch/powerpc/platforms/pseries/vas.c
index 591c7597db5a..3bb219f54806 100644
--- a/arch/powerpc/platforms/pseries/vas.c
+++ b/arch/powerpc/platforms/pseries/vas.c
@@ -26,6 +26,7 @@
 
 static struct vas_all_caps caps_all;
 static bool copypaste_feat;
+static struct hv_vas_cop_feat_caps hv_cop_caps;
 
 static struct vas_caps vascaps[VAS_MAX_FEAT_TYPE];
 static DEFINE_MUTEX(vas_pseries_mutex);
@@ -724,7 +725,6 @@ static int reconfig_close_windows(struct vas_caps *vcap, int excess_creds)
  */
 int vas_reconfig_capabilties(u8 type)
 {
-	struct hv_vas_cop_feat_caps *hv_caps;
 	struct vas_cop_feat_caps *caps;
 	int old_nr_creds, new_nr_creds;
 	struct vas_caps *vcaps;
@@ -738,17 +738,13 @@ int vas_reconfig_capabilties(u8 type)
 	vcaps = &vascaps[type];
 	caps = &vcaps->caps;
 
-	hv_caps = kmalloc(sizeof(*hv_caps), GFP_KERNEL);
-	if (!hv_caps)
-		return -ENOMEM;
-
 	mutex_lock(&vas_pseries_mutex);
 	rc = h_query_vas_capabilities(H_QUERY_VAS_CAPABILITIES, vcaps->feat,
-				      (u64)virt_to_phys(hv_caps));
+				      (u64)virt_to_phys(&hv_cop_caps));
 	if (rc)
 		goto out;
 
-	new_nr_creds = be16_to_cpu(hv_caps->target_lpar_creds);
+	new_nr_creds = be16_to_cpu(hv_cop_caps.target_lpar_creds);
 
 	old_nr_creds = atomic_read(&caps->nr_total_credits);
 
@@ -780,7 +776,6 @@ int vas_reconfig_capabilties(u8 type)
 
 out:
 	mutex_unlock(&vas_pseries_mutex);
-	kfree(hv_caps);
 	return rc;
 }
 /*
@@ -822,9 +817,8 @@ static struct notifier_block pseries_vas_nb = {
 
 static int __init pseries_vas_init(void)
 {
-	struct hv_vas_cop_feat_caps *hv_cop_caps;
 	struct hv_vas_all_caps *hv_caps;
-	int rc;
+	int rc = 0;
 
 	/*
 	 * Linux supports user space COPY/PASTE only with Radix
@@ -850,38 +844,37 @@ static int __init pseries_vas_init(void)
 
 	sysfs_pseries_vas_init(&caps_all);
 
-	hv_cop_caps = kmalloc(sizeof(*hv_cop_caps), GFP_KERNEL);
-	if (!hv_cop_caps) {
-		rc = -ENOMEM;
-		goto out;
-	}
 	/*
 	 * QOS capabilities available
 	 */
 	if (caps_all.feat_type & VAS_GZIP_QOS_FEAT_BIT) {
 		rc = get_vas_capabilities(VAS_GZIP_QOS_FEAT,
-					  VAS_GZIP_QOS_FEAT_TYPE, hv_cop_caps);
+					  VAS_GZIP_QOS_FEAT_TYPE, &hv_cop_caps);
 
 		if (rc)
-			goto out_cop;
+			goto out;
 	}
 	/*
 	 * Default capabilities available
 	 */
-	if (caps_all.feat_type & VAS_GZIP_DEF_FEAT_BIT) {
+	if (caps_all.feat_type & VAS_GZIP_DEF_FEAT_BIT)
 		rc = get_vas_capabilities(VAS_GZIP_DEF_FEAT,
-					  VAS_GZIP_DEF_FEAT_TYPE, hv_cop_caps);
-		if (rc)
-			goto out_cop;
-	}
+					  VAS_GZIP_DEF_FEAT_TYPE, &hv_cop_caps);
 
-	if (copypaste_feat && firmware_has_feature(FW_FEATURE_LPAR))
-		of_reconfig_notifier_register(&pseries_vas_nb);
+	if (!rc && copypaste_feat) {
+		if (firmware_has_feature(FW_FEATURE_LPAR))
+			of_reconfig_notifier_register(&pseries_vas_nb);
 
-	pr_info("GZIP feature is available\n");
+		pr_info("GZIP feature is available\n");
+	} else {
+		/*
+		 * Should not happen, but only when get default
+		 * capabilities HCALL failed. So disable copy paste
+		 * feature.
+		 */
+		copypaste_feat = false;
+	}
 
-out_cop:
-	kfree(hv_cop_caps);
 out:
 	kfree(hv_caps);
 	return rc;
-- 
2.27.0

[PATCH v3 2/4] powerpc/pseries/vas: Modify reconfig open/close functions for migration

[Haren Myneni]

VAS is a hardware engine stays on the chip. So when the partition
migrates, all VAS windows on the source system have to be closed
and reopen them on the destination after migration.

This patch make changes to the current reconfig_open/close_windows
functions to support migration:
- Set VAS_WIN_MIGRATE_CLOSE to the window status when closes and
  reopen windows with the same status during resume.
- Continue to close all windows even if deallocate HCALL failed
  (should not happen) since no way to stop migration with the
  current LPM implementation.
- If the DLPAR CPU event happens while migration is in progress,
  set VAS_WIN_NO_CRED_CLOSE to the window status. Close window
  happens with the first event (migration or DLPAR) and Reopen
  window happens only with the last event (migration or DLPAR).

Signed-off-by: Haren Myneni <haren@linux.ibm.com>
---
 arch/powerpc/include/asm/vas.h       |  2 +
 arch/powerpc/platforms/pseries/vas.c | 88 ++++++++++++++++++++++------
 2 files changed, 73 insertions(+), 17 deletions(-)

diff --git a/arch/powerpc/include/asm/vas.h b/arch/powerpc/include/asm/vas.h
index 6baf7b9ffed4..83afcb6c194b 100644
--- a/arch/powerpc/include/asm/vas.h
+++ b/arch/powerpc/include/asm/vas.h
@@ -36,6 +36,8 @@
 					/* vas mmap() */
 /* Window is closed in the hypervisor due to lost credit */
 #define VAS_WIN_NO_CRED_CLOSE	0x00000001
+/* Window is closed due to migration */
+#define VAS_WIN_MIGRATE_CLOSE	0x00000002
 
 /*
  * Get/Set bit fields
diff --git a/arch/powerpc/platforms/pseries/vas.c b/arch/powerpc/platforms/pseries/vas.c
index 3bb219f54806..fbcf311da0ec 100644
--- a/arch/powerpc/platforms/pseries/vas.c
+++ b/arch/powerpc/platforms/pseries/vas.c
@@ -457,11 +457,12 @@ static int vas_deallocate_window(struct vas_window *vwin)
 	mutex_lock(&vas_pseries_mutex);
 	/*
 	 * VAS window is already closed in the hypervisor when
-	 * lost the credit. So just remove the entry from
-	 * the list, remove task references and free vas_window
+	 * lost the credit or with migration. So just remove the entry
+	 * from the list, remove task references and free vas_window
 	 * struct.
 	 */
-	if (win->vas_win.status & VAS_WIN_NO_CRED_CLOSE) {
+	if (!(win->vas_win.status & VAS_WIN_NO_CRED_CLOSE) &&
+		!(win->vas_win.status & VAS_WIN_MIGRATE_CLOSE)) {
 		rc = deallocate_free_window(win);
 		if (rc) {
 			mutex_unlock(&vas_pseries_mutex);
@@ -578,12 +579,14 @@ static int __init get_vas_capabilities(u8 feat, enum vas_cop_feat_type type,
  * by setting the remapping to new paste address if the window is
  * active.
  */
-static int reconfig_open_windows(struct vas_caps *vcaps, int creds)
+static int reconfig_open_windows(struct vas_caps *vcaps, int creds,
+				 bool migrate)
 {
 	long domain[PLPAR_HCALL9_BUFSIZE] = {VAS_DEFAULT_DOMAIN_ID};
 	struct vas_cop_feat_caps *caps = &vcaps->caps;
 	struct pseries_vas_window *win = NULL, *tmp;
 	int rc, mv_ents = 0;
+	int flag;
 
 	/*
 	 * Nothing to do if there are no closed windows.
@@ -602,8 +605,10 @@ static int reconfig_open_windows(struct vas_caps *vcaps, int creds)
 	 * (dedicated). If 1 core is added, this LPAR can have 20 more
 	 * credits. It means the kernel can reopen 20 windows. So move
 	 * 20 entries in the VAS windows lost and reopen next 20 windows.
+	 * For partition migration, reopen all windows that are closed
+	 * during resume.
 	 */
-	if (vcaps->nr_close_wins > creds)
+	if ((vcaps->nr_close_wins > creds) && !migrate)
 		mv_ents = vcaps->nr_close_wins - creds;
 
 	list_for_each_entry_safe(win, tmp, &vcaps->list, win_list) {
@@ -613,12 +618,35 @@ static int reconfig_open_windows(struct vas_caps *vcaps, int creds)
 		mv_ents--;
 	}
 
+	/*
+	 * Open windows if they are closed only with migration or
+	 * DLPAR (lost credit) before.
+	 */
+	if (migrate)
+		flag = VAS_WIN_MIGRATE_CLOSE;
+	else
+		flag = VAS_WIN_NO_CRED_CLOSE;
+
 	list_for_each_entry_safe_from(win, tmp, &vcaps->list, win_list) {
+		/*
+		 * This window is closed with DLPAR and migration events.
+		 * So reopen the window with the last event.
+		 * The user space is not suspended with the current
+		 * migration notifier. So the user space can issue DLPAR
+		 * CPU hotplug while migration in progress. In this case
+		 * this window will be opened with the last event.
+		 */
+		if ((win->vas_win.status & VAS_WIN_NO_CRED_CLOSE) &&
+			(win->vas_win.status & VAS_WIN_MIGRATE_CLOSE)) {
+			win->vas_win.status &= ~flag;
+			continue;
+		}
+
 		/*
 		 * Nothing to do on this window if it is not closed
-		 * with VAS_WIN_NO_CRED_CLOSE
+		 * with this flag
 		 */
-		if (!(win->vas_win.status & VAS_WIN_NO_CRED_CLOSE))
+		if (!(win->vas_win.status & flag))
 			continue;
 
 		rc = allocate_setup_window(win, (u64 *)&domain[0],
@@ -634,7 +662,7 @@ static int reconfig_open_windows(struct vas_caps *vcaps, int creds)
 		/*
 		 * Set window status to active
 		 */
-		win->vas_win.status &= ~VAS_WIN_NO_CRED_CLOSE;
+		win->vas_win.status &= ~flag;
 		mutex_unlock(&win->vas_win.task_ref.mmap_mutex);
 		win->win_type = caps->win_type;
 		if (!--vcaps->nr_close_wins)
@@ -661,20 +689,32 @@ static int reconfig_open_windows(struct vas_caps *vcaps, int creds)
  * the user space to fall back to SW compression and manage with the
  * existing windows.
  */
-static int reconfig_close_windows(struct vas_caps *vcap, int excess_creds)
+static int reconfig_close_windows(struct vas_caps *vcap, int excess_creds,
+									bool migrate)
 {
 	struct pseries_vas_window *win, *tmp;
 	struct vas_user_win_ref *task_ref;
 	struct vm_area_struct *vma;
-	int rc = 0;
+	int rc = 0, flag;
+
+	if (migrate)
+		flag = VAS_WIN_MIGRATE_CLOSE;
+	else
+		flag = VAS_WIN_NO_CRED_CLOSE;
 
 	list_for_each_entry_safe(win, tmp, &vcap->list, win_list) {
 		/*
 		 * This window is already closed due to lost credit
-		 * before. Go for next window.
+		 * or for migration before. Go for next window.
+		 * For migration, nothing to do since this window
+		 * closed for DLPAR and will be reopened even on
+		 * the destination system with other DLPAR operation.
 		 */
-		if (win->vas_win.status & VAS_WIN_NO_CRED_CLOSE)
+		if ((win->vas_win.status & VAS_WIN_MIGRATE_CLOSE) ||
+			(win->vas_win.status & VAS_WIN_NO_CRED_CLOSE)) {
+			win->vas_win.status |= flag;
 			continue;
+		}
 
 		task_ref = &win->vas_win.task_ref;
 		mutex_lock(&task_ref->mmap_mutex);
@@ -683,7 +723,7 @@ static int reconfig_close_windows(struct vas_caps *vcap, int excess_creds)
 		 * Number of available credits are reduced, So select
 		 * and close windows.
 		 */
-		win->vas_win.status |= VAS_WIN_NO_CRED_CLOSE;
+		win->vas_win.status |= flag;
 
 		mmap_write_lock(task_ref->mm);
 		/*
@@ -706,12 +746,24 @@ static int reconfig_close_windows(struct vas_caps *vcap, int excess_creds)
 		 * later when the process issued with close(FD).
 		 */
 		rc = deallocate_free_window(win);
-		if (rc)
+		/*
+		 * This failure is from the hypervisor.
+		 * No way to stop migration for these failures.
+		 * So ignore error and continue closing other windows.
+		 */
+		if (rc && !migrate)
 			return rc;
 
 		vcap->nr_close_wins++;
 
-		if (!--excess_creds)
+		/*
+		 * For migration, do not depend on lpar_creds in case if
+		 * mismatch with the hypervisor value (should not happen).
+		 * So close all active windows in the list and will be
+		 * reopened windows based on the new lpar_creds on the
+		 * destination system during resume.
+		 */
+		if (!migrate && !--excess_creds)
 			break;
 	}
 
@@ -761,7 +813,8 @@ int vas_reconfig_capabilties(u8 type)
 		 * target, reopen windows if they are closed due to
 		 * the previous DLPAR (core removal).
 		 */
-		rc = reconfig_open_windows(vcaps, new_nr_creds - old_nr_creds);
+		rc = reconfig_open_windows(vcaps, new_nr_creds - old_nr_creds,
+					   false);
 	} else {
 		/*
 		 * # active windows is more than new LPAR available
@@ -771,7 +824,8 @@ int vas_reconfig_capabilties(u8 type)
 		nr_active_wins = vcaps->nr_open_windows - vcaps->nr_close_wins;
 		if (nr_active_wins > new_nr_creds)
 			rc = reconfig_close_windows(vcaps,
-					nr_active_wins - new_nr_creds);
+					nr_active_wins - new_nr_creds,
+					false);
 	}
 
 out:
-- 
2.27.0

[PATCH v3 3/4] powerpc/pseries/vas: Add VAS migration handler

[Haren Myneni]

Since the VAS windows belong to the VAS hardware resource, the
hypervisor expects the partition to close them on source partition
and reopen them after the partition migrated on the destination
machine.

This handler is called before pseries_suspend() to close these
windows and again invoked after migration. All active windows
for both default and QoS types will be closed and mark them
in-active and reopened after migration with this handler.
During the migration, the user space receives paste instruction
failure if it issues copy/paste on these in-active windows.

Signed-off-by: Haren Myneni <haren@linux.ibm.com>
---
 arch/powerpc/platforms/pseries/mobility.c |  5 ++
 arch/powerpc/platforms/pseries/vas.c      | 86 +++++++++++++++++++++++
 arch/powerpc/platforms/pseries/vas.h      |  6 ++
 3 files changed, 97 insertions(+)

diff --git a/arch/powerpc/platforms/pseries/mobility.c b/arch/powerpc/platforms/pseries/mobility.c
index 85033f392c78..70004243e25e 100644
--- a/arch/powerpc/platforms/pseries/mobility.c
+++ b/arch/powerpc/platforms/pseries/mobility.c
@@ -26,6 +26,7 @@
 #include <asm/machdep.h>
 #include <asm/rtas.h>
 #include "pseries.h"
+#include "vas.h"	/* vas_migration_handler() */
 #include "../../kernel/cacheinfo.h"
 
 static struct kobject *mobility_kobj;
@@ -669,12 +670,16 @@ static int pseries_migrate_partition(u64 handle)
 	if (ret)
 		return ret;
 
+	vas_migration_handler(VAS_SUSPEND);
+
 	ret = pseries_suspend(handle);
 	if (ret == 0)
 		post_mobility_fixup();
 	else
 		pseries_cancel_migration(handle, ret);
 
+	vas_migration_handler(VAS_RESUME);
+
 	return ret;
 }
 
diff --git a/arch/powerpc/platforms/pseries/vas.c b/arch/powerpc/platforms/pseries/vas.c
index fbcf311da0ec..df22827969db 100644
--- a/arch/powerpc/platforms/pseries/vas.c
+++ b/arch/powerpc/platforms/pseries/vas.c
@@ -869,6 +869,92 @@ static struct notifier_block pseries_vas_nb = {
 	.notifier_call = pseries_vas_notifier,
 };
 
+/*
+ * For LPM, all windows have to be closed on the source partition
+ * before migration and reopen them on the destination partition
+ * after migration. So closing windows during suspend and
+ * reopen them during resume.
+ */
+int vas_migration_handler(int action)
+{
+	struct vas_cop_feat_caps *caps;
+	int old_nr_creds, new_nr_creds = 0;
+	struct vas_caps *vcaps;
+	int i, rc = 0;
+
+	/*
+	 * NX-GZIP is not enabled. Nothing to do for migration.
+	 */
+	if (!copypaste_feat)
+		return rc;
+
+	mutex_lock(&vas_pseries_mutex);
+
+	for (i = 0; i < VAS_MAX_FEAT_TYPE; i++) {
+		vcaps = &vascaps[i];
+		caps = &vcaps->caps;
+		old_nr_creds = atomic_read(&caps->nr_total_credits);
+
+		rc = h_query_vas_capabilities(H_QUERY_VAS_CAPABILITIES,
+					      vcaps->feat,
+					      (u64)virt_to_phys(&hv_cop_caps));
+		if (!rc) {
+			new_nr_creds = be16_to_cpu(hv_cop_caps.target_lpar_creds);
+			/*
+			 * Should not happen. But incase print messages, close
+			 * all windows in the list during suspend and reopen
+			 * windows based on new lpar_creds on the destination
+			 * system.
+			 */
+			if (old_nr_creds != new_nr_creds) {
+				pr_err("state(%d): lpar creds: %d HV lpar creds: %d\n",
+					action, old_nr_creds, new_nr_creds);
+				pr_err("Used creds: %d, Active creds: %d\n",
+					atomic_read(&caps->nr_used_credits),
+					vcaps->nr_open_windows - vcaps->nr_close_wins);
+			}
+		} else {
+			pr_err("state(%d): Get VAS capabilities failed with %d\n",
+				action, rc);
+			/*
+			 * We can not stop migration with the current lpm
+			 * implementation. So continue closing all windows in
+			 * the list (during suspend) and return without
+			 * opening windows (during resume) if VAS capabilities
+			 * HCALL failed.
+			 */
+			if (action == VAS_RESUME)
+				goto out;
+		}
+
+		switch (action) {
+		case VAS_SUSPEND:
+			rc = reconfig_close_windows(vcaps, vcaps->nr_open_windows,
+							true);
+			break;
+		case VAS_RESUME:
+			atomic_set(&caps->nr_total_credits, new_nr_creds);
+			rc = reconfig_open_windows(vcaps, new_nr_creds, true);
+			break;
+		default:
+			/* should not happen */
+			pr_err("Invalid migration action %d\n", action);
+			rc = -EINVAL;
+			goto out;
+		}
+
+		/*
+		 * Ignore errors during suspend and return for resume.
+		 */
+		if (rc && (action == VAS_RESUME))
+			goto out;
+	}
+
+out:
+	mutex_unlock(&vas_pseries_mutex);
+	return rc;
+}
+
 static int __init pseries_vas_init(void)
 {
 	struct hv_vas_all_caps *hv_caps;
diff --git a/arch/powerpc/platforms/pseries/vas.h b/arch/powerpc/platforms/pseries/vas.h
index 4ddb1001a0aa..f7568d8c6ab0 100644
--- a/arch/powerpc/platforms/pseries/vas.h
+++ b/arch/powerpc/platforms/pseries/vas.h
@@ -33,6 +33,11 @@
 #define VAS_GZIP_QOS_CAPABILITIES	0x56516F73477A6970
 #define VAS_GZIP_DEFAULT_CAPABILITIES	0x56446566477A6970
 
+enum vas_migrate_action {
+	VAS_SUSPEND,
+	VAS_RESUME,
+};
+
 /*
  * Co-processor feature - GZIP QoS windows or GZIP default windows
  */
@@ -132,4 +137,5 @@ struct pseries_vas_window {
 int sysfs_add_vas_caps(struct vas_cop_feat_caps *caps);
 int vas_reconfig_capabilties(u8 type);
 int __init sysfs_pseries_vas_init(struct vas_all_caps *vas_caps);
+int vas_migration_handler(int action);
 #endif /* _VAS_H */
-- 
2.27.0

[PATCH v3 4/4] powerpc/pseries/vas: Disable window open during migration

[Haren Myneni]

The current partition migration implementation does not freeze the
user space and the user space can continue open VAS windows. So
when migration_in_progress flag is enabled, VAS open window
API returns -EBUSY.

Signed-off-by: Haren Myneni <haren@linux.ibm.com>
---
 arch/powerpc/platforms/pseries/vas.c | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/arch/powerpc/platforms/pseries/vas.c b/arch/powerpc/platforms/pseries/vas.c
index df22827969db..4be80112b05e 100644
--- a/arch/powerpc/platforms/pseries/vas.c
+++ b/arch/powerpc/platforms/pseries/vas.c
@@ -30,6 +30,7 @@ static struct hv_vas_cop_feat_caps hv_cop_caps;
 
 static struct vas_caps vascaps[VAS_MAX_FEAT_TYPE];
 static DEFINE_MUTEX(vas_pseries_mutex);
+static bool migration_in_progress;
 
 static long hcall_return_busy_check(long rc)
 {
@@ -356,8 +357,11 @@ static struct vas_window *vas_allocate_window(int vas_id, u64 flags,
 	 * same fault IRQ is not freed by the OS before.
 	 */
 	mutex_lock(&vas_pseries_mutex);
-	rc = allocate_setup_window(txwin, (u64 *)&domain[0],
-				   cop_feat_caps->win_type);
+	if (migration_in_progress)
+		rc = -EBUSY;
+	else
+		rc = allocate_setup_window(txwin, (u64 *)&domain[0],
+					   cop_feat_caps->win_type);
 	mutex_unlock(&vas_pseries_mutex);
 	if (rc)
 		goto out;
@@ -890,6 +894,11 @@ int vas_migration_handler(int action)
 
 	mutex_lock(&vas_pseries_mutex);
 
+	if (action == VAS_SUSPEND)
+		migration_in_progress = true;
+	else
+		migration_in_progress = false;
+
 	for (i = 0; i < VAS_MAX_FEAT_TYPE; i++) {
 		vcaps = &vascaps[i];
 		caps = &vcaps->caps;
-- 
2.27.0

Re: [PATCH v3 0/4] powerpc/pseries/vas: VAS/NXGZIP support with LPM

[Nicholas Piggin]

Excerpts from Haren Myneni's message of February 20, 2022 6:04 am:
> 
> Virtual Accelerator Switchboard (VAS) is an engine stays on the
> chip. So all windows opened on a specific engine belongs to VAS
> the chip.

The problem is more that PAPR does not virtualise the VAS windows,
right? That's a whole other gripe but nothing you can do about it
here.

Thanks,
Nick

> The hypervisor expects the partition to close all
> active windows on the sources system and reopen them after
> migration on the destination machine.
> 
> This patch series adds VAS support with the partition migration.
> When the migration initiates, the VAS migration handler will be
> invoked before pseries_suspend() to close all active windows and
> mark them in-active with VAS_WIN_MIGRATE_CLOSE status. Whereas
> this migration handler is called after migration to reopen all
> windows which has VAS_WIN_MIGRATE_CLOSE status and make them
> active again. The user space gets paste instruction failure
> when it sends requests on these in-active windows.
> 
> These patches depend on VAS/DLPAR support patch series
> 
> Changes in v2:
> - Added new patch "Define global hv_cop_caps struct" to eliminate
>   memory allocation failure during migration (suggestion by
>   Nathan Lynch)
> 
> Changes in v3:
> - Rebase on 5.17-rc4
> - Naming changes for VAS capability struct elemets based on the V4 DLPAR
>   support patch series.
> 
> Haren Myneni (4):
>   powerpc/pseries/vas: Define global hv_cop_caps struct
>   powerpc/pseries/vas: Modify reconfig open/close functions for
>     migration
>   powerpc/pseries/vas: Add VAS migration handler
>   powerpc/pseries/vas: Disable window open during migration
> 
>  arch/powerpc/include/asm/vas.h            |   2 +
>  arch/powerpc/platforms/pseries/mobility.c |   5 +
>  arch/powerpc/platforms/pseries/vas.c      | 234 +++++++++++++++++-----
>  arch/powerpc/platforms/pseries/vas.h      |   6 +
>  4 files changed, 201 insertions(+), 46 deletions(-)
> 
> -- 
> 2.27.0
> 
> 
> 

Re: [PATCH v3 1/4] powerpc/pseries/vas: Define global hv_cop_caps struct

[Nicholas Piggin]

Excerpts from Haren Myneni's message of February 20, 2022 6:05 am:
> 
> The coprocessor capabilities struct is used to get default and
> QoS capabilities from the hypervisor during init, DLPAR event and
> migration. So instead of allocating this struct for each event,
> define global struct and reuse it which allows the migration code
> to avoid adding an error path.
> 
> Also disable copy/paste feature flag if any capabilities HCALL
> is failed.
> 

Reviewed-by: Nicholas Piggin <npiggin@gmail.com>

> Signed-off-by: Haren Myneni <haren@linux.ibm.com>
> Acked-by: Nathan Lynch <nathanl@linux.ibm.com>
> ---
>  arch/powerpc/platforms/pseries/vas.c | 47 ++++++++++++----------------
>  1 file changed, 20 insertions(+), 27 deletions(-)
> 
> diff --git a/arch/powerpc/platforms/pseries/vas.c b/arch/powerpc/platforms/pseries/vas.c
> index 591c7597db5a..3bb219f54806 100644
> --- a/arch/powerpc/platforms/pseries/vas.c
> +++ b/arch/powerpc/platforms/pseries/vas.c
> @@ -26,6 +26,7 @@
>  
>  static struct vas_all_caps caps_all;
>  static bool copypaste_feat;
> +static struct hv_vas_cop_feat_caps hv_cop_caps;
>  
>  static struct vas_caps vascaps[VAS_MAX_FEAT_TYPE];
>  static DEFINE_MUTEX(vas_pseries_mutex);
> @@ -724,7 +725,6 @@ static int reconfig_close_windows(struct vas_caps *vcap, int excess_creds)
>   */
>  int vas_reconfig_capabilties(u8 type)
>  {
> -	struct hv_vas_cop_feat_caps *hv_caps;
>  	struct vas_cop_feat_caps *caps;
>  	int old_nr_creds, new_nr_creds;
>  	struct vas_caps *vcaps;
> @@ -738,17 +738,13 @@ int vas_reconfig_capabilties(u8 type)
>  	vcaps = &vascaps[type];
>  	caps = &vcaps->caps;
>  
> -	hv_caps = kmalloc(sizeof(*hv_caps), GFP_KERNEL);
> -	if (!hv_caps)
> -		return -ENOMEM;
> -
>  	mutex_lock(&vas_pseries_mutex);
>  	rc = h_query_vas_capabilities(H_QUERY_VAS_CAPABILITIES, vcaps->feat,
> -				      (u64)virt_to_phys(hv_caps));
> +				      (u64)virt_to_phys(&hv_cop_caps));
>  	if (rc)
>  		goto out;
>  
> -	new_nr_creds = be16_to_cpu(hv_caps->target_lpar_creds);
> +	new_nr_creds = be16_to_cpu(hv_cop_caps.target_lpar_creds);
>  
>  	old_nr_creds = atomic_read(&caps->nr_total_credits);
>  
> @@ -780,7 +776,6 @@ int vas_reconfig_capabilties(u8 type)
>  
>  out:
>  	mutex_unlock(&vas_pseries_mutex);
> -	kfree(hv_caps);
>  	return rc;
>  }
>  /*
> @@ -822,9 +817,8 @@ static struct notifier_block pseries_vas_nb = {
>  
>  static int __init pseries_vas_init(void)
>  {
> -	struct hv_vas_cop_feat_caps *hv_cop_caps;
>  	struct hv_vas_all_caps *hv_caps;
> -	int rc;
> +	int rc = 0;
>  
>  	/*
>  	 * Linux supports user space COPY/PASTE only with Radix
> @@ -850,38 +844,37 @@ static int __init pseries_vas_init(void)
>  
>  	sysfs_pseries_vas_init(&caps_all);
>  
> -	hv_cop_caps = kmalloc(sizeof(*hv_cop_caps), GFP_KERNEL);
> -	if (!hv_cop_caps) {
> -		rc = -ENOMEM;
> -		goto out;
> -	}
>  	/*
>  	 * QOS capabilities available
>  	 */
>  	if (caps_all.feat_type & VAS_GZIP_QOS_FEAT_BIT) {
>  		rc = get_vas_capabilities(VAS_GZIP_QOS_FEAT,
> -					  VAS_GZIP_QOS_FEAT_TYPE, hv_cop_caps);
> +					  VAS_GZIP_QOS_FEAT_TYPE, &hv_cop_caps);
>  
>  		if (rc)
> -			goto out_cop;
> +			goto out;
>  	}
>  	/*
>  	 * Default capabilities available
>  	 */
> -	if (caps_all.feat_type & VAS_GZIP_DEF_FEAT_BIT) {
> +	if (caps_all.feat_type & VAS_GZIP_DEF_FEAT_BIT)
>  		rc = get_vas_capabilities(VAS_GZIP_DEF_FEAT,
> -					  VAS_GZIP_DEF_FEAT_TYPE, hv_cop_caps);
> -		if (rc)
> -			goto out_cop;
> -	}
> +					  VAS_GZIP_DEF_FEAT_TYPE, &hv_cop_caps);
>  
> -	if (copypaste_feat && firmware_has_feature(FW_FEATURE_LPAR))
> -		of_reconfig_notifier_register(&pseries_vas_nb);
> +	if (!rc && copypaste_feat) {
> +		if (firmware_has_feature(FW_FEATURE_LPAR))
> +			of_reconfig_notifier_register(&pseries_vas_nb);
>  
> -	pr_info("GZIP feature is available\n");
> +		pr_info("GZIP feature is available\n");
> +	} else {
> +		/*
> +		 * Should not happen, but only when get default
> +		 * capabilities HCALL failed. So disable copy paste
> +		 * feature.
> +		 */
> +		copypaste_feat = false;
> +	}
>  
> -out_cop:
> -	kfree(hv_cop_caps);
>  out:
>  	kfree(hv_caps);
>  	return rc;
> -- 
> 2.27.0
> 
> 
> 

Re: [PATCH v3 0/4] powerpc/pseries/vas: VAS/NXGZIP support with LPM

[Haren Myneni]

On Wed, 2022-02-23 at 19:38 +1000, Nicholas Piggin wrote:
> Excerpts from Haren Myneni's message of February 20, 2022 6:04 am:
> > Virtual Accelerator Switchboard (VAS) is an engine stays on the
> > chip. So all windows opened on a specific engine belongs to VAS
> > the chip.
> 
> The problem is more that PAPR does not virtualise the VAS windows,
> right? That's a whole other gripe but nothing you can do about it
> here.

Yes, There is no virtualization with VAS windows and they are specific
to the chip. 

> 
> Thanks,
> Nick
> 
> > The hypervisor expects the partition to close all
> > active windows on the sources system and reopen them after
> > migration on the destination machine.
> > 
> > This patch series adds VAS support with the partition migration.
> > When the migration initiates, the VAS migration handler will be
> > invoked before pseries_suspend() to close all active windows and
> > mark them in-active with VAS_WIN_MIGRATE_CLOSE status. Whereas
> > this migration handler is called after migration to reopen all
> > windows which has VAS_WIN_MIGRATE_CLOSE status and make them
> > active again. The user space gets paste instruction failure
> > when it sends requests on these in-active windows.
> > 
> > These patches depend on VAS/DLPAR support patch series
> > 
> > Changes in v2:
> > - Added new patch "Define global hv_cop_caps struct" to eliminate
> >   memory allocation failure during migration (suggestion by
> >   Nathan Lynch)
> > 
> > Changes in v3:
> > - Rebase on 5.17-rc4
> > - Naming changes for VAS capability struct elemets based on the V4
> > DLPAR
> >   support patch series.
> > 
> > Haren Myneni (4):
> >   powerpc/pseries/vas: Define global hv_cop_caps struct
> >   powerpc/pseries/vas: Modify reconfig open/close functions for
> >     migration
> >   powerpc/pseries/vas: Add VAS migration handler
> >   powerpc/pseries/vas: Disable window open during migration
> > 
> >  arch/powerpc/include/asm/vas.h            |   2 +
> >  arch/powerpc/platforms/pseries/mobility.c |   5 +
> >  arch/powerpc/platforms/pseries/vas.c      | 234 +++++++++++++++++-
> > ----
> >  arch/powerpc/platforms/pseries/vas.h      |   6 +
> >  4 files changed, 201 insertions(+), 46 deletions(-)
> > 
> > -- 
> > 2.27.0
> > 
> > 
> > 

Re: [PATCH v3 2/4] powerpc/pseries/vas: Modify reconfig open/close functions for migration

[Nicholas Piggin]

Excerpts from Haren Myneni's message of February 20, 2022 6:05 am:
> 
> VAS is a hardware engine stays on the chip. So when the partition
> migrates, all VAS windows on the source system have to be closed
> and reopen them on the destination after migration.
> 
> This patch make changes to the current reconfig_open/close_windows
> functions to support migration:
> - Set VAS_WIN_MIGRATE_CLOSE to the window status when closes and
>   reopen windows with the same status during resume.
> - Continue to close all windows even if deallocate HCALL failed
>   (should not happen) since no way to stop migration with the
>   current LPM implementation.

Hmm.  pseries_migrate_partition *can* fail?

> - If the DLPAR CPU event happens while migration is in progress,
>   set VAS_WIN_NO_CRED_CLOSE to the window status. Close window
>   happens with the first event (migration or DLPAR) and Reopen
>   window happens only with the last event (migration or DLPAR).

Can DLPAR happen while migration is in progress? Couldn't
this cause your source and destination credits to go out of
whack?

Why do you need two close window types, what if you finish
LPM and just open as many as possible regardless how they
are closed?

Thanks,
Nick

> 
> Signed-off-by: Haren Myneni <haren@linux.ibm.com>
> ---
>  arch/powerpc/include/asm/vas.h       |  2 +
>  arch/powerpc/platforms/pseries/vas.c | 88 ++++++++++++++++++++++------
>  2 files changed, 73 insertions(+), 17 deletions(-)
> 
> diff --git a/arch/powerpc/include/asm/vas.h b/arch/powerpc/include/asm/vas.h
> index 6baf7b9ffed4..83afcb6c194b 100644
> --- a/arch/powerpc/include/asm/vas.h
> +++ b/arch/powerpc/include/asm/vas.h
> @@ -36,6 +36,8 @@
>  					/* vas mmap() */
>  /* Window is closed in the hypervisor due to lost credit */
>  #define VAS_WIN_NO_CRED_CLOSE	0x00000001
> +/* Window is closed due to migration */
> +#define VAS_WIN_MIGRATE_CLOSE	0x00000002
>  
>  /*
>   * Get/Set bit fields
> diff --git a/arch/powerpc/platforms/pseries/vas.c b/arch/powerpc/platforms/pseries/vas.c
> index 3bb219f54806..fbcf311da0ec 100644
> --- a/arch/powerpc/platforms/pseries/vas.c
> +++ b/arch/powerpc/platforms/pseries/vas.c
> @@ -457,11 +457,12 @@ static int vas_deallocate_window(struct vas_window *vwin)
>  	mutex_lock(&vas_pseries_mutex);
>  	/*
>  	 * VAS window is already closed in the hypervisor when
> -	 * lost the credit. So just remove the entry from
> -	 * the list, remove task references and free vas_window
> +	 * lost the credit or with migration. So just remove the entry
> +	 * from the list, remove task references and free vas_window
>  	 * struct.
>  	 */
> -	if (win->vas_win.status & VAS_WIN_NO_CRED_CLOSE) {
> +	if (!(win->vas_win.status & VAS_WIN_NO_CRED_CLOSE) &&
> +		!(win->vas_win.status & VAS_WIN_MIGRATE_CLOSE)) {
>  		rc = deallocate_free_window(win);
>  		if (rc) {
>  			mutex_unlock(&vas_pseries_mutex);
> @@ -578,12 +579,14 @@ static int __init get_vas_capabilities(u8 feat, enum vas_cop_feat_type type,
>   * by setting the remapping to new paste address if the window is
>   * active.
>   */
> -static int reconfig_open_windows(struct vas_caps *vcaps, int creds)
> +static int reconfig_open_windows(struct vas_caps *vcaps, int creds,
> +				 bool migrate)
>  {
>  	long domain[PLPAR_HCALL9_BUFSIZE] = {VAS_DEFAULT_DOMAIN_ID};
>  	struct vas_cop_feat_caps *caps = &vcaps->caps;
>  	struct pseries_vas_window *win = NULL, *tmp;
>  	int rc, mv_ents = 0;
> +	int flag;
>  
>  	/*
>  	 * Nothing to do if there are no closed windows.
> @@ -602,8 +605,10 @@ static int reconfig_open_windows(struct vas_caps *vcaps, int creds)
>  	 * (dedicated). If 1 core is added, this LPAR can have 20 more
>  	 * credits. It means the kernel can reopen 20 windows. So move
>  	 * 20 entries in the VAS windows lost and reopen next 20 windows.
> +	 * For partition migration, reopen all windows that are closed
> +	 * during resume.
>  	 */
> -	if (vcaps->nr_close_wins > creds)
> +	if ((vcaps->nr_close_wins > creds) && !migrate)
>  		mv_ents = vcaps->nr_close_wins - creds;
>  
>  	list_for_each_entry_safe(win, tmp, &vcaps->list, win_list) {
> @@ -613,12 +618,35 @@ static int reconfig_open_windows(struct vas_caps *vcaps, int creds)
>  		mv_ents--;
>  	}
>  
> +	/*
> +	 * Open windows if they are closed only with migration or
> +	 * DLPAR (lost credit) before.
> +	 */
> +	if (migrate)
> +		flag = VAS_WIN_MIGRATE_CLOSE;
> +	else
> +		flag = VAS_WIN_NO_CRED_CLOSE;
> +
>  	list_for_each_entry_safe_from(win, tmp, &vcaps->list, win_list) {
> +		/*
> +		 * This window is closed with DLPAR and migration events.
> +		 * So reopen the window with the last event.
> +		 * The user space is not suspended with the current
> +		 * migration notifier. So the user space can issue DLPAR
> +		 * CPU hotplug while migration in progress. In this case
> +		 * this window will be opened with the last event.
> +		 */
> +		if ((win->vas_win.status & VAS_WIN_NO_CRED_CLOSE) &&
> +			(win->vas_win.status & VAS_WIN_MIGRATE_CLOSE)) {
> +			win->vas_win.status &= ~flag;
> +			continue;
> +		}
> +
>  		/*
>  		 * Nothing to do on this window if it is not closed
> -		 * with VAS_WIN_NO_CRED_CLOSE
> +		 * with this flag
>  		 */
> -		if (!(win->vas_win.status & VAS_WIN_NO_CRED_CLOSE))
> +		if (!(win->vas_win.status & flag))
>  			continue;
>  
>  		rc = allocate_setup_window(win, (u64 *)&domain[0],
> @@ -634,7 +662,7 @@ static int reconfig_open_windows(struct vas_caps *vcaps, int creds)
>  		/*
>  		 * Set window status to active
>  		 */
> -		win->vas_win.status &= ~VAS_WIN_NO_CRED_CLOSE;
> +		win->vas_win.status &= ~flag;
>  		mutex_unlock(&win->vas_win.task_ref.mmap_mutex);
>  		win->win_type = caps->win_type;
>  		if (!--vcaps->nr_close_wins)
> @@ -661,20 +689,32 @@ static int reconfig_open_windows(struct vas_caps *vcaps, int creds)
>   * the user space to fall back to SW compression and manage with the
>   * existing windows.
>   */
> -static int reconfig_close_windows(struct vas_caps *vcap, int excess_creds)
> +static int reconfig_close_windows(struct vas_caps *vcap, int excess_creds,
> +									bool migrate)
>  {
>  	struct pseries_vas_window *win, *tmp;
>  	struct vas_user_win_ref *task_ref;
>  	struct vm_area_struct *vma;
> -	int rc = 0;
> +	int rc = 0, flag;
> +
> +	if (migrate)
> +		flag = VAS_WIN_MIGRATE_CLOSE;
> +	else
> +		flag = VAS_WIN_NO_CRED_CLOSE;
>  
>  	list_for_each_entry_safe(win, tmp, &vcap->list, win_list) {
>  		/*
>  		 * This window is already closed due to lost credit
> -		 * before. Go for next window.
> +		 * or for migration before. Go for next window.
> +		 * For migration, nothing to do since this window
> +		 * closed for DLPAR and will be reopened even on
> +		 * the destination system with other DLPAR operation.
>  		 */
> -		if (win->vas_win.status & VAS_WIN_NO_CRED_CLOSE)
> +		if ((win->vas_win.status & VAS_WIN_MIGRATE_CLOSE) ||
> +			(win->vas_win.status & VAS_WIN_NO_CRED_CLOSE)) {
> +			win->vas_win.status |= flag;
>  			continue;
> +		}
>  
>  		task_ref = &win->vas_win.task_ref;
>  		mutex_lock(&task_ref->mmap_mutex);
> @@ -683,7 +723,7 @@ static int reconfig_close_windows(struct vas_caps *vcap, int excess_creds)
>  		 * Number of available credits are reduced, So select
>  		 * and close windows.
>  		 */
> -		win->vas_win.status |= VAS_WIN_NO_CRED_CLOSE;
> +		win->vas_win.status |= flag;
>  
>  		mmap_write_lock(task_ref->mm);
>  		/*
> @@ -706,12 +746,24 @@ static int reconfig_close_windows(struct vas_caps *vcap, int excess_creds)
>  		 * later when the process issued with close(FD).
>  		 */
>  		rc = deallocate_free_window(win);
> -		if (rc)
> +		/*
> +		 * This failure is from the hypervisor.
> +		 * No way to stop migration for these failures.
> +		 * So ignore error and continue closing other windows.
> +		 */
> +		if (rc && !migrate)
>  			return rc;
>  
>  		vcap->nr_close_wins++;
>  
> -		if (!--excess_creds)
> +		/*
> +		 * For migration, do not depend on lpar_creds in case if
> +		 * mismatch with the hypervisor value (should not happen).
> +		 * So close all active windows in the list and will be
> +		 * reopened windows based on the new lpar_creds on the
> +		 * destination system during resume.
> +		 */
> +		if (!migrate && !--excess_creds)
>  			break;
>  	}
>  
> @@ -761,7 +813,8 @@ int vas_reconfig_capabilties(u8 type)
>  		 * target, reopen windows if they are closed due to
>  		 * the previous DLPAR (core removal).
>  		 */
> -		rc = reconfig_open_windows(vcaps, new_nr_creds - old_nr_creds);
> +		rc = reconfig_open_windows(vcaps, new_nr_creds - old_nr_creds,
> +					   false);
>  	} else {
>  		/*
>  		 * # active windows is more than new LPAR available
> @@ -771,7 +824,8 @@ int vas_reconfig_capabilties(u8 type)
>  		nr_active_wins = vcaps->nr_open_windows - vcaps->nr_close_wins;
>  		if (nr_active_wins > new_nr_creds)
>  			rc = reconfig_close_windows(vcaps,
> -					nr_active_wins - new_nr_creds);
> +					nr_active_wins - new_nr_creds,
> +					false);
>  	}
>  
>  out:
> -- 
> 2.27.0
> 
> 
> 

Re: [PATCH v3 3/4] powerpc/pseries/vas: Add VAS migration handler

[Nicholas Piggin]

Excerpts from Haren Myneni's message of February 20, 2022 6:06 am:
> 
> Since the VAS windows belong to the VAS hardware resource, the
> hypervisor expects the partition to close them on source partition
> and reopen them after the partition migrated on the destination
> machine.
> 
> This handler is called before pseries_suspend() to close these
> windows and again invoked after migration. All active windows
> for both default and QoS types will be closed and mark them
> in-active and reopened after migration with this handler.
> During the migration, the user space receives paste instruction
> failure if it issues copy/paste on these in-active windows.
> 
> Signed-off-by: Haren Myneni <haren@linux.ibm.com>
> ---
>  arch/powerpc/platforms/pseries/mobility.c |  5 ++
>  arch/powerpc/platforms/pseries/vas.c      | 86 +++++++++++++++++++++++
>  arch/powerpc/platforms/pseries/vas.h      |  6 ++
>  3 files changed, 97 insertions(+)
> 
> diff --git a/arch/powerpc/platforms/pseries/mobility.c b/arch/powerpc/platforms/pseries/mobility.c
> index 85033f392c78..70004243e25e 100644
> --- a/arch/powerpc/platforms/pseries/mobility.c
> +++ b/arch/powerpc/platforms/pseries/mobility.c
> @@ -26,6 +26,7 @@
>  #include <asm/machdep.h>
>  #include <asm/rtas.h>
>  #include "pseries.h"
> +#include "vas.h"	/* vas_migration_handler() */
>  #include "../../kernel/cacheinfo.h"
>  
>  static struct kobject *mobility_kobj;
> @@ -669,12 +670,16 @@ static int pseries_migrate_partition(u64 handle)
>  	if (ret)
>  		return ret;
>  
> +	vas_migration_handler(VAS_SUSPEND);

Not sure if there is much point having a "handler" like this that only
takes two operations. vas_migration_begin()/vas_migration_end() is
better isn't it?

Other question is why can't the suspend handler return error and handle
it here?

> +
>  	ret = pseries_suspend(handle);
>  	if (ret == 0)
>  		post_mobility_fixup();
>  	else
>  		pseries_cancel_migration(handle, ret);
>  
> +	vas_migration_handler(VAS_RESUME);
> +
>  	return ret;
>  }
>  
> diff --git a/arch/powerpc/platforms/pseries/vas.c b/arch/powerpc/platforms/pseries/vas.c
> index fbcf311da0ec..df22827969db 100644
> --- a/arch/powerpc/platforms/pseries/vas.c
> +++ b/arch/powerpc/platforms/pseries/vas.c
> @@ -869,6 +869,92 @@ static struct notifier_block pseries_vas_nb = {
>  	.notifier_call = pseries_vas_notifier,
>  };
>  
> +/*
> + * For LPM, all windows have to be closed on the source partition
> + * before migration and reopen them on the destination partition
> + * after migration. So closing windows during suspend and
> + * reopen them during resume.
> + */
> +int vas_migration_handler(int action)
> +{
> +	struct vas_cop_feat_caps *caps;
> +	int old_nr_creds, new_nr_creds = 0;
> +	struct vas_caps *vcaps;
> +	int i, rc = 0;
> +
> +	/*
> +	 * NX-GZIP is not enabled. Nothing to do for migration.
> +	 */
> +	if (!copypaste_feat)
> +		return rc;
> +
> +	mutex_lock(&vas_pseries_mutex);
> +
> +	for (i = 0; i < VAS_MAX_FEAT_TYPE; i++) {
> +		vcaps = &vascaps[i];
> +		caps = &vcaps->caps;
> +		old_nr_creds = atomic_read(&caps->nr_total_credits);
> +
> +		rc = h_query_vas_capabilities(H_QUERY_VAS_CAPABILITIES,
> +					      vcaps->feat,
> +					      (u64)virt_to_phys(&hv_cop_caps));
> +		if (!rc) {
> +			new_nr_creds = be16_to_cpu(hv_cop_caps.target_lpar_creds);
> +			/*
> +			 * Should not happen. But incase print messages, close
> +			 * all windows in the list during suspend and reopen
> +			 * windows based on new lpar_creds on the destination
> +			 * system.
> +			 */
> +			if (old_nr_creds != new_nr_creds) {
> +				pr_err("state(%d): lpar creds: %d HV lpar creds: %d\n",
> +					action, old_nr_creds, new_nr_creds);
> +				pr_err("Used creds: %d, Active creds: %d\n",
> +					atomic_read(&caps->nr_used_credits),
> +					vcaps->nr_open_windows - vcaps->nr_close_wins);

Error messages should have some vague use to the administrator even if 
you don't expect it or they aren't expected to know much.

"VAS NX error during LPM: credit mismatch blah"

Otherwise if it's a Linux or hypervisor bug then make it a WARN_ON, at 
least then by convention the administrator knows that should be reported
and it's (possibly) non-fatal.

Thanks,
Nick

> +			}
> +		} else {
> +			pr_err("state(%d): Get VAS capabilities failed with %d\n",
> +				action, rc);
> +			/*
> +			 * We can not stop migration with the current lpm
> +			 * implementation. So continue closing all windows in
> +			 * the list (during suspend) and return without
> +			 * opening windows (during resume) if VAS capabilities
> +			 * HCALL failed.
> +			 */
> +			if (action == VAS_RESUME)
> +				goto out;
> +		}
> +
> +		switch (action) {
> +		case VAS_SUSPEND:
> +			rc = reconfig_close_windows(vcaps, vcaps->nr_open_windows,
> +							true);
> +			break;
> +		case VAS_RESUME:
> +			atomic_set(&caps->nr_total_credits, new_nr_creds);
> +			rc = reconfig_open_windows(vcaps, new_nr_creds, true);
> +			break;
> +		default:
> +			/* should not happen */
> +			pr_err("Invalid migration action %d\n", action);
> +			rc = -EINVAL;
> +			goto out;
> +		}
> +
> +		/*
> +		 * Ignore errors during suspend and return for resume.
> +		 */
> +		if (rc && (action == VAS_RESUME))
> +			goto out;
> +	}
> +
> +out:
> +	mutex_unlock(&vas_pseries_mutex);
> +	return rc;
> +}
> +
>  static int __init pseries_vas_init(void)
>  {
>  	struct hv_vas_all_caps *hv_caps;
> diff --git a/arch/powerpc/platforms/pseries/vas.h b/arch/powerpc/platforms/pseries/vas.h
> index 4ddb1001a0aa..f7568d8c6ab0 100644
> --- a/arch/powerpc/platforms/pseries/vas.h
> +++ b/arch/powerpc/platforms/pseries/vas.h
> @@ -33,6 +33,11 @@
>  #define VAS_GZIP_QOS_CAPABILITIES	0x56516F73477A6970
>  #define VAS_GZIP_DEFAULT_CAPABILITIES	0x56446566477A6970
>  
> +enum vas_migrate_action {
> +	VAS_SUSPEND,
> +	VAS_RESUME,
> +};
> +
>  /*
>   * Co-processor feature - GZIP QoS windows or GZIP default windows
>   */
> @@ -132,4 +137,5 @@ struct pseries_vas_window {
>  int sysfs_add_vas_caps(struct vas_cop_feat_caps *caps);
>  int vas_reconfig_capabilties(u8 type);
>  int __init sysfs_pseries_vas_init(struct vas_all_caps *vas_caps);
> +int vas_migration_handler(int action);
>  #endif /* _VAS_H */
> -- 
> 2.27.0
> 
> 
> 

Re: [PATCH v3 4/4] powerpc/pseries/vas: Disable window open during migration

[Nicholas Piggin]

Excerpts from Haren Myneni's message of February 20, 2022 6:08 am:
> 
> The current partition migration implementation does not freeze the
> user space and the user space can continue open VAS windows. So
> when migration_in_progress flag is enabled, VAS open window
> API returns -EBUSY.

Seems like it could be merged with the previous patch. Unless
you're trying to specifically call out the -EBUSY chane to the
API?

Thanks,
Nick

> 
> Signed-off-by: Haren Myneni <haren@linux.ibm.com>
> ---
>  arch/powerpc/platforms/pseries/vas.c | 13 +++++++++++--
>  1 file changed, 11 insertions(+), 2 deletions(-)
> 
> diff --git a/arch/powerpc/platforms/pseries/vas.c b/arch/powerpc/platforms/pseries/vas.c
> index df22827969db..4be80112b05e 100644
> --- a/arch/powerpc/platforms/pseries/vas.c
> +++ b/arch/powerpc/platforms/pseries/vas.c
> @@ -30,6 +30,7 @@ static struct hv_vas_cop_feat_caps hv_cop_caps;
>  
>  static struct vas_caps vascaps[VAS_MAX_FEAT_TYPE];
>  static DEFINE_MUTEX(vas_pseries_mutex);
> +static bool migration_in_progress;
>  
>  static long hcall_return_busy_check(long rc)
>  {
> @@ -356,8 +357,11 @@ static struct vas_window *vas_allocate_window(int vas_id, u64 flags,
>  	 * same fault IRQ is not freed by the OS before.
>  	 */
>  	mutex_lock(&vas_pseries_mutex);
> -	rc = allocate_setup_window(txwin, (u64 *)&domain[0],
> -				   cop_feat_caps->win_type);
> +	if (migration_in_progress)
> +		rc = -EBUSY;
> +	else
> +		rc = allocate_setup_window(txwin, (u64 *)&domain[0],
> +					   cop_feat_caps->win_type);
>  	mutex_unlock(&vas_pseries_mutex);
>  	if (rc)
>  		goto out;
> @@ -890,6 +894,11 @@ int vas_migration_handler(int action)
>  
>  	mutex_lock(&vas_pseries_mutex);
>  
> +	if (action == VAS_SUSPEND)
> +		migration_in_progress = true;
> +	else
> +		migration_in_progress = false;
> +
>  	for (i = 0; i < VAS_MAX_FEAT_TYPE; i++) {
>  		vcaps = &vascaps[i];
>  		caps = &vcaps->caps;
> -- 
> 2.27.0
> 
> 
> 

Re: [PATCH v3 2/4] powerpc/pseries/vas: Modify reconfig open/close functions for migration

[Haren Myneni]

On Wed, 2022-02-23 at 19:54 +1000, Nicholas Piggin wrote:
> Excerpts from Haren Myneni's message of February 20, 2022 6:05 am:
> > VAS is a hardware engine stays on the chip. So when the partition
> > migrates, all VAS windows on the source system have to be closed
> > and reopen them on the destination after migration.
> > 
> > This patch make changes to the current reconfig_open/close_windows
> > functions to support migration:
> > - Set VAS_WIN_MIGRATE_CLOSE to the window status when closes and
> >   reopen windows with the same status during resume.
> > - Continue to close all windows even if deallocate HCALL failed
> >   (should not happen) since no way to stop migration with the
> >   current LPM implementation.
> 
> Hmm.  pseries_migrate_partition *can* fail?

Yes, it can fail. If pseries_suspend() fails, all VAS windows will be
reopened again without migration. vas_migration_handler(VAS_RESUME) is
called whether pseries_suspend() returns 0 or not.

> 
> > - If the DLPAR CPU event happens while migration is in progress,
> >   set VAS_WIN_NO_CRED_CLOSE to the window status. Close window
> >   happens with the first event (migration or DLPAR) and Reopen
> >   window happens only with the last event (migration or DLPAR).
> 
> Can DLPAR happen while migration is in progress? Couldn't
> this cause your source and destination credits to go out of
> whack?

Should not be, If the DLPAR event happens while migration is in
progress, windows will be closed in the hypervisor (and mark inactive
with migration status bit in OS) for migration. For DLPAR event, mark
the DLPAR_CLOSED status bits for the necessary windows. Then after the
migration, we open windows in the hypervisor and set them active in OS
that have only migration status. Open the other remaining windows only
after the other DLPAR core add event. 

Regarding the traget credits on the destination, we get the new
capabilities after migration and use the new value for reopen. 

Ex: Used the following test case -
- Configuted 2 dedicated cores (40 credits) and exeuted the test case
which opened 35 credits / windows
- Removed 1 core, means available 20 credits. So closed 15 windows and
set them with DLPAR closed status
- Migration start: Closed the remaining 20 windows and set all windows
(means 35) migration status
- After migration, opened windows that have only migration status - 20
windows, and also clear migration stats for the remaining 15 widnows
- Add core which gives the system 20 more credits, So opened the
remaining 15 windows and these have only DLPAR closed status. 

> 
> Why do you need two close window types, what if you finish
> LPM and just open as many as possible regardless how they
> are closed?

Adding 2 different status bits to support DLPAR and LPM closed staus.
As I mentioned above, windows will be active only after both bits are
cleared.

Thanks
Haren

> 
> Thanks,
> Nick
> 
> > Signed-off-by: Haren Myneni <haren@linux.ibm.com>
> > ---
> >  arch/powerpc/include/asm/vas.h       |  2 +
> >  arch/powerpc/platforms/pseries/vas.c | 88 ++++++++++++++++++++++
> > ------
> >  2 files changed, 73 insertions(+), 17 deletions(-)
> > 
> > diff --git a/arch/powerpc/include/asm/vas.h
> > b/arch/powerpc/include/asm/vas.h
> > index 6baf7b9ffed4..83afcb6c194b 100644
> > --- a/arch/powerpc/include/asm/vas.h
> > +++ b/arch/powerpc/include/asm/vas.h
> > @@ -36,6 +36,8 @@
> >  					/* vas mmap() */
> >  /* Window is closed in the hypervisor due to lost credit */
> >  #define VAS_WIN_NO_CRED_CLOSE	0x00000001
> > +/* Window is closed due to migration */
> > +#define VAS_WIN_MIGRATE_CLOSE	0x00000002
> >  
> >  /*
> >   * Get/Set bit fields
> > diff --git a/arch/powerpc/platforms/pseries/vas.c
> > b/arch/powerpc/platforms/pseries/vas.c
> > index 3bb219f54806..fbcf311da0ec 100644
> > --- a/arch/powerpc/platforms/pseries/vas.c
> > +++ b/arch/powerpc/platforms/pseries/vas.c
> > @@ -457,11 +457,12 @@ static int vas_deallocate_window(struct
> > vas_window *vwin)
> >  	mutex_lock(&vas_pseries_mutex);
> >  	/*
> >  	 * VAS window is already closed in the hypervisor when
> > -	 * lost the credit. So just remove the entry from
> > -	 * the list, remove task references and free vas_window
> > +	 * lost the credit or with migration. So just remove the entry
> > +	 * from the list, remove task references and free vas_window
> >  	 * struct.
> >  	 */
> > -	if (win->vas_win.status & VAS_WIN_NO_CRED_CLOSE) {
> > +	if (!(win->vas_win.status & VAS_WIN_NO_CRED_CLOSE) &&
> > +		!(win->vas_win.status & VAS_WIN_MIGRATE_CLOSE)) {
> >  		rc = deallocate_free_window(win);
> >  		if (rc) {
> >  			mutex_unlock(&vas_pseries_mutex);
> > @@ -578,12 +579,14 @@ static int __init get_vas_capabilities(u8
> > feat, enum vas_cop_feat_type type,
> >   * by setting the remapping to new paste address if the window is
> >   * active.
> >   */
> > -static int reconfig_open_windows(struct vas_caps *vcaps, int
> > creds)
> > +static int reconfig_open_windows(struct vas_caps *vcaps, int
> > creds,
> > +				 bool migrate)
> >  {
> >  	long domain[PLPAR_HCALL9_BUFSIZE] = {VAS_DEFAULT_DOMAIN_ID};
> >  	struct vas_cop_feat_caps *caps = &vcaps->caps;
> >  	struct pseries_vas_window *win = NULL, *tmp;
> >  	int rc, mv_ents = 0;
> > +	int flag;
> >  
> >  	/*
> >  	 * Nothing to do if there are no closed windows.
> > @@ -602,8 +605,10 @@ static int reconfig_open_windows(struct
> > vas_caps *vcaps, int creds)
> >  	 * (dedicated). If 1 core is added, this LPAR can have 20 more
> >  	 * credits. It means the kernel can reopen 20 windows. So move
> >  	 * 20 entries in the VAS windows lost and reopen next 20
> > windows.
> > +	 * For partition migration, reopen all windows that are closed
> > +	 * during resume.
> >  	 */
> > -	if (vcaps->nr_close_wins > creds)
> > +	if ((vcaps->nr_close_wins > creds) && !migrate)
> >  		mv_ents = vcaps->nr_close_wins - creds;
> >  
> >  	list_for_each_entry_safe(win, tmp, &vcaps->list, win_list) {
> > @@ -613,12 +618,35 @@ static int reconfig_open_windows(struct
> > vas_caps *vcaps, int creds)
> >  		mv_ents--;
> >  	}
> >  
> > +	/*
> > +	 * Open windows if they are closed only with migration or
> > +	 * DLPAR (lost credit) before.
> > +	 */
> > +	if (migrate)
> > +		flag = VAS_WIN_MIGRATE_CLOSE;
> > +	else
> > +		flag = VAS_WIN_NO_CRED_CLOSE;
> > +
> >  	list_for_each_entry_safe_from(win, tmp, &vcaps->list, win_list)
> > {
> > +		/*
> > +		 * This window is closed with DLPAR and migration
> > events.
> > +		 * So reopen the window with the last event.
> > +		 * The user space is not suspended with the current
> > +		 * migration notifier. So the user space can issue
> > DLPAR
> > +		 * CPU hotplug while migration in progress. In this
> > case
> > +		 * this window will be opened with the last event.
> > +		 */
> > +		if ((win->vas_win.status & VAS_WIN_NO_CRED_CLOSE) &&
> > +			(win->vas_win.status & VAS_WIN_MIGRATE_CLOSE))
> > {
> > +			win->vas_win.status &= ~flag;
> > +			continue;
> > +		}
> > +
> >  		/*
> >  		 * Nothing to do on this window if it is not closed
> > -		 * with VAS_WIN_NO_CRED_CLOSE
> > +		 * with this flag
> >  		 */
> > -		if (!(win->vas_win.status & VAS_WIN_NO_CRED_CLOSE))
> > +		if (!(win->vas_win.status & flag))
> >  			continue;
> >  
> >  		rc = allocate_setup_window(win, (u64 *)&domain[0],
> > @@ -634,7 +662,7 @@ static int reconfig_open_windows(struct
> > vas_caps *vcaps, int creds)
> >  		/*
> >  		 * Set window status to active
> >  		 */
> > -		win->vas_win.status &= ~VAS_WIN_NO_CRED_CLOSE;
> > +		win->vas_win.status &= ~flag;
> >  		mutex_unlock(&win->vas_win.task_ref.mmap_mutex);
> >  		win->win_type = caps->win_type;
> >  		if (!--vcaps->nr_close_wins)
> > @@ -661,20 +689,32 @@ static int reconfig_open_windows(struct
> > vas_caps *vcaps, int creds)
> >   * the user space to fall back to SW compression and manage with
> > the
> >   * existing windows.
> >   */
> > -static int reconfig_close_windows(struct vas_caps *vcap, int
> > excess_creds)
> > +static int reconfig_close_windows(struct vas_caps *vcap, int
> > excess_creds,
> > +									
> > bool migrate)
> >  {
> >  	struct pseries_vas_window *win, *tmp;
> >  	struct vas_user_win_ref *task_ref;
> >  	struct vm_area_struct *vma;
> > -	int rc = 0;
> > +	int rc = 0, flag;
> > +
> > +	if (migrate)
> > +		flag = VAS_WIN_MIGRATE_CLOSE;
> > +	else
> > +		flag = VAS_WIN_NO_CRED_CLOSE;
> >  
> >  	list_for_each_entry_safe(win, tmp, &vcap->list, win_list) {
> >  		/*
> >  		 * This window is already closed due to lost credit
> > -		 * before. Go for next window.
> > +		 * or for migration before. Go for next window.
> > +		 * For migration, nothing to do since this window
> > +		 * closed for DLPAR and will be reopened even on
> > +		 * the destination system with other DLPAR operation.
> >  		 */
> > -		if (win->vas_win.status & VAS_WIN_NO_CRED_CLOSE)
> > +		if ((win->vas_win.status & VAS_WIN_MIGRATE_CLOSE) ||
> > +			(win->vas_win.status & VAS_WIN_NO_CRED_CLOSE))
> > {
> > +			win->vas_win.status |= flag;
> >  			continue;
> > +		}
> >  
> >  		task_ref = &win->vas_win.task_ref;
> >  		mutex_lock(&task_ref->mmap_mutex);
> > @@ -683,7 +723,7 @@ static int reconfig_close_windows(struct
> > vas_caps *vcap, int excess_creds)
> >  		 * Number of available credits are reduced, So select
> >  		 * and close windows.
> >  		 */
> > -		win->vas_win.status |= VAS_WIN_NO_CRED_CLOSE;
> > +		win->vas_win.status |= flag;
> >  
> >  		mmap_write_lock(task_ref->mm);
> >  		/*
> > @@ -706,12 +746,24 @@ static int reconfig_close_windows(struct
> > vas_caps *vcap, int excess_creds)
> >  		 * later when the process issued with close(FD).
> >  		 */
> >  		rc = deallocate_free_window(win);
> > -		if (rc)
> > +		/*
> > +		 * This failure is from the hypervisor.
> > +		 * No way to stop migration for these failures.
> > +		 * So ignore error and continue closing other windows.
> > +		 */
> > +		if (rc && !migrate)
> >  			return rc;
> >  
> >  		vcap->nr_close_wins++;
> >  
> > -		if (!--excess_creds)
> > +		/*
> > +		 * For migration, do not depend on lpar_creds in case
> > if
> > +		 * mismatch with the hypervisor value (should not
> > happen).
> > +		 * So close all active windows in the list and will be
> > +		 * reopened windows based on the new lpar_creds on the
> > +		 * destination system during resume.
> > +		 */
> > +		if (!migrate && !--excess_creds)
> >  			break;
> >  	}
> >  
> > @@ -761,7 +813,8 @@ int vas_reconfig_capabilties(u8 type)
> >  		 * target, reopen windows if they are closed due to
> >  		 * the previous DLPAR (core removal).
> >  		 */
> > -		rc = reconfig_open_windows(vcaps, new_nr_creds -
> > old_nr_creds);
> > +		rc = reconfig_open_windows(vcaps, new_nr_creds -
> > old_nr_creds,
> > +					   false);
> >  	} else {
> >  		/*
> >  		 * # active windows is more than new LPAR available
> > @@ -771,7 +824,8 @@ int vas_reconfig_capabilties(u8 type)
> >  		nr_active_wins = vcaps->nr_open_windows - vcaps-
> > >nr_close_wins;
> >  		if (nr_active_wins > new_nr_creds)
> >  			rc = reconfig_close_windows(vcaps,
> > -					nr_active_wins - new_nr_creds);
> > +					nr_active_wins - new_nr_creds,
> > +					false);
> >  	}
> >  
> >  out:
> > -- 
> > 2.27.0
> > 
> > 
> > 

Re: [PATCH v3 3/4] powerpc/pseries/vas: Add VAS migration handler

[Haren Myneni]

On Wed, 2022-02-23 at 20:03 +1000, Nicholas Piggin wrote:
> Excerpts from Haren Myneni's message of February 20, 2022 6:06 am:
> > Since the VAS windows belong to the VAS hardware resource, the
> > hypervisor expects the partition to close them on source partition
> > and reopen them after the partition migrated on the destination
> > machine.
> > 
> > This handler is called before pseries_suspend() to close these
> > windows and again invoked after migration. All active windows
> > for both default and QoS types will be closed and mark them
> > in-active and reopened after migration with this handler.
> > During the migration, the user space receives paste instruction
> > failure if it issues copy/paste on these in-active windows.
> > 
> > Signed-off-by: Haren Myneni <haren@linux.ibm.com>
> > ---
> >  arch/powerpc/platforms/pseries/mobility.c |  5 ++
> >  arch/powerpc/platforms/pseries/vas.c      | 86
> > +++++++++++++++++++++++
> >  arch/powerpc/platforms/pseries/vas.h      |  6 ++
> >  3 files changed, 97 insertions(+)
> > 
> > diff --git a/arch/powerpc/platforms/pseries/mobility.c
> > b/arch/powerpc/platforms/pseries/mobility.c
> > index 85033f392c78..70004243e25e 100644
> > --- a/arch/powerpc/platforms/pseries/mobility.c
> > +++ b/arch/powerpc/platforms/pseries/mobility.c
> > @@ -26,6 +26,7 @@
> >  #include <asm/machdep.h>
> >  #include <asm/rtas.h>
> >  #include "pseries.h"
> > +#include "vas.h"	/* vas_migration_handler() */
> >  #include "../../kernel/cacheinfo.h"
> >  
> >  static struct kobject *mobility_kobj;
> > @@ -669,12 +670,16 @@ static int pseries_migrate_partition(u64
> > handle)
> >  	if (ret)
> >  		return ret;
> >  
> > +	vas_migration_handler(VAS_SUSPEND);
> 
> Not sure if there is much point having a "handler" like this that
> only
> takes two operations. vas_migration_begin()/vas_migration_end() is
> better isn't it?

The actual suspend / resume framework will be added later. So using the
VAS_SUSPEND/VAS_RESUME right now, but will be removed later after
having the permanent fix. 

> 
> Other question is why can't the suspend handler return error and
> handle
> it here?

We can, but has to call pseries_cancel_migration() if VAS suspend
handler returns failure. We should expect this failure only from
H_DEALLOCATE_VAS_WINDOW and H_QUERY_VAS_CAPABILITIES HCALLs wich should
not happen generally.

> 
> > +
> >  	ret = pseries_suspend(handle);
> >  	if (ret == 0)
> >  		post_mobility_fixup();
> >  	else
> >  		pseries_cancel_migration(handle, ret);
> >  
> > +	vas_migration_handler(VAS_RESUME);
> > +
> >  	return ret;
> >  }
> >  
> > diff --git a/arch/powerpc/platforms/pseries/vas.c
> > b/arch/powerpc/platforms/pseries/vas.c
> > index fbcf311da0ec..df22827969db 100644
> > --- a/arch/powerpc/platforms/pseries/vas.c
> > +++ b/arch/powerpc/platforms/pseries/vas.c
> > @@ -869,6 +869,92 @@ static struct notifier_block pseries_vas_nb =
> > {
> >  	.notifier_call = pseries_vas_notifier,
> >  };
> >  
> > +/*
> > + * For LPM, all windows have to be closed on the source partition
> > + * before migration and reopen them on the destination partition
> > + * after migration. So closing windows during suspend and
> > + * reopen them during resume.
> > + */
> > +int vas_migration_handler(int action)
> > +{
> > +	struct vas_cop_feat_caps *caps;
> > +	int old_nr_creds, new_nr_creds = 0;
> > +	struct vas_caps *vcaps;
> > +	int i, rc = 0;
> > +
> > +	/*
> > +	 * NX-GZIP is not enabled. Nothing to do for migration.
> > +	 */
> > +	if (!copypaste_feat)
> > +		return rc;
> > +
> > +	mutex_lock(&vas_pseries_mutex);
> > +
> > +	for (i = 0; i < VAS_MAX_FEAT_TYPE; i++) {
> > +		vcaps = &vascaps[i];
> > +		caps = &vcaps->caps;
> > +		old_nr_creds = atomic_read(&caps->nr_total_credits);
> > +
> > +		rc = h_query_vas_capabilities(H_QUERY_VAS_CAPABILITIES,
> > +					      vcaps->feat,
> > +					      (u64)virt_to_phys(&hv_cop
> > _caps));
> > +		if (!rc) {
> > +			new_nr_creds =
> > be16_to_cpu(hv_cop_caps.target_lpar_creds);
> > +			/*
> > +			 * Should not happen. But incase print
> > messages, close
> > +			 * all windows in the list during suspend and
> > reopen
> > +			 * windows based on new lpar_creds on the
> > destination
> > +			 * system.
> > +			 */
> > +			if (old_nr_creds != new_nr_creds) {
> > +				pr_err("state(%d): lpar creds: %d HV
> > lpar creds: %d\n",
> > +					action, old_nr_creds,
> > new_nr_creds);
> > +				pr_err("Used creds: %d, Active creds:
> > %d\n",
> > +					atomic_read(&caps-
> > >nr_used_credits),
> > +					vcaps->nr_open_windows - vcaps-
> > >nr_close_wins);
> 
> Error messages should have some vague use to the administrator even
> if 
> you don't expect it or they aren't expected to know much.
> 
> "VAS NX error during LPM: credit mismatch blah"
> 
> Otherwise if it's a Linux or hypervisor bug then make it a WARN_ON,
> at 
> least then by convention the administrator knows that should be
> reported
> and it's (possibly) non-fatal.

I can use pr_warn(), but my intention was to give attention to user by
notifying it as error message. This credits mismatch should not happen
and the hypervisor has to make sure credits match before the migration.

Thanks
Haren

> 
> Thanks,
> Nick
> 
> > +			}
> > +		} else {
> > +			pr_err("state(%d): Get VAS capabilities failed
> > with %d\n",
> > +				action, rc);
> > +			/*
> > +			 * We can not stop migration with the current
> > lpm
> > +			 * implementation. So continue closing all
> > windows in
> > +			 * the list (during suspend) and return without
> > +			 * opening windows (during resume) if VAS
> > capabilities
> > +			 * HCALL failed.
> > +			 */
> > +			if (action == VAS_RESUME)
> > +				goto out;
> > +		}
> > +
> > +		switch (action) {
> > +		case VAS_SUSPEND:
> > +			rc = reconfig_close_windows(vcaps, vcaps-
> > >nr_open_windows,
> > +							true);
> > +			break;
> > +		case VAS_RESUME:
> > +			atomic_set(&caps->nr_total_credits,
> > new_nr_creds);
> > +			rc = reconfig_open_windows(vcaps, new_nr_creds,
> > true);
> > +			break;
> > +		default:
> > +			/* should not happen */
> > +			pr_err("Invalid migration action %d\n",
> > action);
> > +			rc = -EINVAL;
> > +			goto out;
> > +		}
> > +
> > +		/*
> > +		 * Ignore errors during suspend and return for resume.
> > +		 */
> > +		if (rc && (action == VAS_RESUME))
> > +			goto out;
> > +	}
> > +
> > +out:
> > +	mutex_unlock(&vas_pseries_mutex);
> > +	return rc;
> > +}
> > +
> >  static int __init pseries_vas_init(void)
> >  {
> >  	struct hv_vas_all_caps *hv_caps;
> > diff --git a/arch/powerpc/platforms/pseries/vas.h
> > b/arch/powerpc/platforms/pseries/vas.h
> > index 4ddb1001a0aa..f7568d8c6ab0 100644
> > --- a/arch/powerpc/platforms/pseries/vas.h
> > +++ b/arch/powerpc/platforms/pseries/vas.h
> > @@ -33,6 +33,11 @@
> >  #define VAS_GZIP_QOS_CAPABILITIES	0x56516F73477A6970
> >  #define VAS_GZIP_DEFAULT_CAPABILITIES	0x56446566477A6970
> >  
> > +enum vas_migrate_action {
> > +	VAS_SUSPEND,
> > +	VAS_RESUME,
> > +};
> > +
> >  /*
> >   * Co-processor feature - GZIP QoS windows or GZIP default windows
> >   */
> > @@ -132,4 +137,5 @@ struct pseries_vas_window {
> >  int sysfs_add_vas_caps(struct vas_cop_feat_caps *caps);
> >  int vas_reconfig_capabilties(u8 type);
> >  int __init sysfs_pseries_vas_init(struct vas_all_caps *vas_caps);
> > +int vas_migration_handler(int action);
> >  #endif /* _VAS_H */
> > -- 
> > 2.27.0
> > 
> > 
> >