* [PATCH v2 01/20] vmw_balloon: handle commands in a single function.
2018-09-20 17:30 [PATCH v2 00/20] vmw_balloon: compaction, shrinker, 64-bit, etc Nadav Amit
@ 2018-09-20 17:30 ` Nadav Amit
2018-09-20 17:30 ` [PATCH v2 02/20] vmw_balloon: unify commands tracing and stats Nadav Amit
` (19 subsequent siblings)
20 siblings, 0 replies; 24+ messages in thread
From: Nadav Amit @ 2018-09-20 17:30 UTC (permalink / raw)
To: Greg Kroah-Hartman, Arnd Bergmann
Cc: linux-kernel, Xavier Deguillard, Nadav Amit
By inlining the hypercall interface, we can unify several operations
into one central point in the code:
- Updating the target.
- Updating when a reset is needed.
- Update statistics (which will be done later in the patch-set).
- Print debug-messages (although they cannot be enabled as selectively).
Reviewed-by: Xavier Deguillard <xdeguillard@vmware.com>
Signed-off-by: Nadav Amit <namit@vmware.com>
---
drivers/misc/vmw_balloon.c | 223 +++++++++++++++++++------------------
1 file changed, 116 insertions(+), 107 deletions(-)
diff --git a/drivers/misc/vmw_balloon.c b/drivers/misc/vmw_balloon.c
index 2543ef1ece17..0a4d5501f805 100644
--- a/drivers/misc/vmw_balloon.c
+++ b/drivers/misc/vmw_balloon.c
@@ -138,6 +138,15 @@ enum vmwballoon_capabilities {
#define VMW_BALLOON_BATCH_STATUS_MASK ((1UL << 5) - 1)
#define VMW_BALLOON_BATCH_PAGE_MASK (~((1UL << PAGE_SHIFT) - 1))
+#define VMW_BALLOON_CMD_WITH_TARGET_MASK \
+ ((1UL << VMW_BALLOON_CMD_GET_TARGET) | \
+ (1UL << VMW_BALLOON_CMD_LOCK) | \
+ (1UL << VMW_BALLOON_CMD_UNLOCK) | \
+ (1UL << VMW_BALLOON_CMD_BATCHED_LOCK) | \
+ (1UL << VMW_BALLOON_CMD_BATCHED_UNLOCK) | \
+ (1UL << VMW_BALLOON_CMD_BATCHED_2M_LOCK) | \
+ (1UL << VMW_BALLOON_CMD_BATCHED_2M_UNLOCK))
+
struct vmballoon_batch_page {
u64 pages[VMW_BALLOON_BATCH_MAX_PAGES];
};
@@ -159,28 +168,6 @@ static void vmballoon_batch_set_pa(struct vmballoon_batch_page *batch, int idx,
batch->pages[idx] = pa;
}
-
-#define VMWARE_BALLOON_CMD(cmd, arg1, arg2, result) \
-({ \
- unsigned long __status, __dummy1, __dummy2, __dummy3; \
- __asm__ __volatile__ ("inl %%dx" : \
- "=a"(__status), \
- "=c"(__dummy1), \
- "=d"(__dummy2), \
- "=b"(result), \
- "=S" (__dummy3) : \
- "0"(VMW_BALLOON_HV_MAGIC), \
- "1"(VMW_BALLOON_CMD_##cmd), \
- "2"(VMW_BALLOON_HV_PORT), \
- "3"(arg1), \
- "4" (arg2) : \
- "memory"); \
- if (VMW_BALLOON_CMD_##cmd == VMW_BALLOON_CMD_START) \
- result = __dummy1; \
- result &= -1UL; \
- __status & -1UL; \
-})
-
#ifdef CONFIG_DEBUG_FS
struct vmballoon_stats {
unsigned int timer;
@@ -220,9 +207,9 @@ struct vmballoon;
struct vmballoon_ops {
void (*add_page)(struct vmballoon *b, int idx, struct page *p);
int (*lock)(struct vmballoon *b, unsigned int num_pages,
- bool is_2m_pages, unsigned int *target);
+ bool is_2m_pages);
int (*unlock)(struct vmballoon *b, unsigned int num_pages,
- bool is_2m_pages, unsigned int *target);
+ bool is_2m_pages);
};
struct vmballoon_page_size {
@@ -272,18 +259,64 @@ struct vmballoon {
static struct vmballoon balloon;
+static inline unsigned long
+__vmballoon_cmd(struct vmballoon *b, unsigned long cmd, unsigned long arg1,
+ unsigned long arg2, unsigned long *result)
+{
+ unsigned long status, dummy1, dummy2, dummy3, local_result;
+
+ asm volatile ("inl %%dx" :
+ "=a"(status),
+ "=c"(dummy1),
+ "=d"(dummy2),
+ "=b"(local_result),
+ "=S"(dummy3) :
+ "0"(VMW_BALLOON_HV_MAGIC),
+ "1"(cmd),
+ "2"(VMW_BALLOON_HV_PORT),
+ "3"(arg1),
+ "4"(arg2) :
+ "memory");
+
+ /* update the result if needed */
+ if (result)
+ *result = (cmd == VMW_BALLOON_CMD_START) ? dummy1 :
+ local_result;
+
+ /* update target when applicable */
+ if (status == VMW_BALLOON_SUCCESS &&
+ ((1ul << cmd) & VMW_BALLOON_CMD_WITH_TARGET_MASK))
+ b->target = local_result;
+
+ /* mark reset required accordingly */
+ if (status == VMW_BALLOON_ERROR_RESET)
+ b->reset_required = true;
+
+ return status;
+}
+
+static __always_inline unsigned long
+vmballoon_cmd(struct vmballoon *b, unsigned long cmd, unsigned long arg1,
+ unsigned long arg2)
+{
+ unsigned long dummy;
+
+ return __vmballoon_cmd(b, cmd, arg1, arg2, &dummy);
+}
+
/*
* Send "start" command to the host, communicating supported version
* of the protocol.
*/
static bool vmballoon_send_start(struct vmballoon *b, unsigned long req_caps)
{
- unsigned long status, capabilities, dummy = 0;
+ unsigned long status, capabilities;
bool success;
STATS_INC(b->stats.start);
- status = VMWARE_BALLOON_CMD(START, req_caps, dummy, capabilities);
+ status = __vmballoon_cmd(b, VMW_BALLOON_CMD_START, req_caps, 0,
+ &capabilities);
switch (status) {
case VMW_BALLOON_SUCCESS_WITH_CAPABILITIES:
@@ -316,21 +349,6 @@ static bool vmballoon_send_start(struct vmballoon *b, unsigned long req_caps)
return success;
}
-static bool vmballoon_check_status(struct vmballoon *b, unsigned long status)
-{
- switch (status) {
- case VMW_BALLOON_SUCCESS:
- return true;
-
- case VMW_BALLOON_ERROR_RESET:
- b->reset_required = true;
- /* fall through */
-
- default:
- return false;
- }
-}
-
/*
* Communicate guest type to the host so that it can adjust ballooning
* algorithm to the one most appropriate for the guest. This command
@@ -339,14 +357,14 @@ static bool vmballoon_check_status(struct vmballoon *b, unsigned long status)
*/
static bool vmballoon_send_guest_id(struct vmballoon *b)
{
- unsigned long status, dummy = 0;
+ unsigned long status;
- status = VMWARE_BALLOON_CMD(GUEST_ID, VMW_BALLOON_GUEST_ID, dummy,
- dummy);
+ status = vmballoon_cmd(b, VMW_BALLOON_CMD_GUEST_ID,
+ VMW_BALLOON_GUEST_ID, 0);
STATS_INC(b->stats.guest_type);
- if (vmballoon_check_status(b, status))
+ if (status == VMW_BALLOON_SUCCESS)
return true;
pr_debug("%s - failed, hv returns %ld\n", __func__, status);
@@ -365,12 +383,10 @@ static u16 vmballoon_page_size(bool is_2m_page)
/*
* Retrieve desired balloon size from the host.
*/
-static bool vmballoon_send_get_target(struct vmballoon *b, u32 *new_target)
+static bool vmballoon_send_get_target(struct vmballoon *b)
{
unsigned long status;
- unsigned long target;
unsigned long limit;
- unsigned long dummy = 0;
u32 limit32;
/*
@@ -389,11 +405,10 @@ static bool vmballoon_send_get_target(struct vmballoon *b, u32 *new_target)
/* update stats */
STATS_INC(b->stats.target);
- status = VMWARE_BALLOON_CMD(GET_TARGET, limit, dummy, target);
- if (vmballoon_check_status(b, status)) {
- *new_target = target;
+ status = vmballoon_cmd(b, VMW_BALLOON_CMD_GET_TARGET, limit, 0);
+
+ if (status == VMW_BALLOON_SUCCESS)
return true;
- }
pr_debug("%s - failed, hv returns %ld\n", __func__, status);
STATS_INC(b->stats.target_fail);
@@ -406,9 +421,9 @@ static bool vmballoon_send_get_target(struct vmballoon *b, u32 *new_target)
* check the return value and maybe submit a different page.
*/
static int vmballoon_send_lock_page(struct vmballoon *b, unsigned long pfn,
- unsigned int *hv_status, unsigned int *target)
+ unsigned int *hv_status)
{
- unsigned long status, dummy = 0;
+ unsigned long status;
u32 pfn32;
pfn32 = (u32)pfn;
@@ -417,8 +432,9 @@ static int vmballoon_send_lock_page(struct vmballoon *b, unsigned long pfn,
STATS_INC(b->stats.lock[false]);
- *hv_status = status = VMWARE_BALLOON_CMD(LOCK, pfn, dummy, *target);
- if (vmballoon_check_status(b, status))
+ *hv_status = status = vmballoon_cmd(b, VMW_BALLOON_CMD_LOCK, pfn, 0);
+
+ if (status == VMW_BALLOON_SUCCESS)
return 0;
pr_debug("%s - ppn %lx, hv returns %ld\n", __func__, pfn, status);
@@ -427,21 +443,19 @@ static int vmballoon_send_lock_page(struct vmballoon *b, unsigned long pfn,
}
static int vmballoon_send_batched_lock(struct vmballoon *b,
- unsigned int num_pages, bool is_2m_pages, unsigned int *target)
+ unsigned int num_pages, bool is_2m_pages)
{
- unsigned long status;
unsigned long pfn = PHYS_PFN(virt_to_phys(b->batch_page));
+ unsigned long status, cmd;
STATS_INC(b->stats.lock[is_2m_pages]);
- if (is_2m_pages)
- status = VMWARE_BALLOON_CMD(BATCHED_2M_LOCK, pfn, num_pages,
- *target);
- else
- status = VMWARE_BALLOON_CMD(BATCHED_LOCK, pfn, num_pages,
- *target);
+ cmd = is_2m_pages ? VMW_BALLOON_CMD_BATCHED_2M_LOCK :
+ VMW_BALLOON_CMD_BATCHED_LOCK;
- if (vmballoon_check_status(b, status))
+ status = vmballoon_cmd(b, cmd, pfn, num_pages);
+
+ if (status == VMW_BALLOON_SUCCESS)
return 0;
pr_debug("%s - batch ppn %lx, hv returns %ld\n", __func__, pfn, status);
@@ -453,10 +467,9 @@ static int vmballoon_send_batched_lock(struct vmballoon *b,
* Notify the host that guest intends to release given page back into
* the pool of available (to the guest) pages.
*/
-static bool vmballoon_send_unlock_page(struct vmballoon *b, unsigned long pfn,
- unsigned int *target)
+static bool vmballoon_send_unlock_page(struct vmballoon *b, unsigned long pfn)
{
- unsigned long status, dummy = 0;
+ unsigned long status;
u32 pfn32;
pfn32 = (u32)pfn;
@@ -465,8 +478,8 @@ static bool vmballoon_send_unlock_page(struct vmballoon *b, unsigned long pfn,
STATS_INC(b->stats.unlock[false]);
- status = VMWARE_BALLOON_CMD(UNLOCK, pfn, dummy, *target);
- if (vmballoon_check_status(b, status))
+ status = vmballoon_cmd(b, VMW_BALLOON_CMD_UNLOCK, pfn, 0);
+ if (status == VMW_BALLOON_SUCCESS)
return true;
pr_debug("%s - ppn %lx, hv returns %ld\n", __func__, pfn, status);
@@ -475,21 +488,19 @@ static bool vmballoon_send_unlock_page(struct vmballoon *b, unsigned long pfn,
}
static bool vmballoon_send_batched_unlock(struct vmballoon *b,
- unsigned int num_pages, bool is_2m_pages, unsigned int *target)
+ unsigned int num_pages, bool is_2m_pages)
{
- unsigned long status;
unsigned long pfn = PHYS_PFN(virt_to_phys(b->batch_page));
+ unsigned long status, cmd;
STATS_INC(b->stats.unlock[is_2m_pages]);
- if (is_2m_pages)
- status = VMWARE_BALLOON_CMD(BATCHED_2M_UNLOCK, pfn, num_pages,
- *target);
- else
- status = VMWARE_BALLOON_CMD(BATCHED_UNLOCK, pfn, num_pages,
- *target);
+ cmd = is_2m_pages ? VMW_BALLOON_CMD_BATCHED_2M_UNLOCK :
+ VMW_BALLOON_CMD_BATCHED_UNLOCK;
+
+ status = vmballoon_cmd(b, cmd, pfn, num_pages);
- if (vmballoon_check_status(b, status))
+ if (status == VMW_BALLOON_SUCCESS)
return true;
pr_debug("%s - batch ppn %lx, hv returns %ld\n", __func__, pfn, status);
@@ -550,7 +561,7 @@ static void vmballoon_pop(struct vmballoon *b)
* inflation cycle.
*/
static int vmballoon_lock_page(struct vmballoon *b, unsigned int num_pages,
- bool is_2m_pages, unsigned int *target)
+ bool is_2m_pages)
{
int locked, hv_status;
struct page *page = b->page;
@@ -558,8 +569,8 @@ static int vmballoon_lock_page(struct vmballoon *b, unsigned int num_pages,
/* is_2m_pages can never happen as 2m pages support implies batching */
- locked = vmballoon_send_lock_page(b, page_to_pfn(page), &hv_status,
- target);
+ locked = vmballoon_send_lock_page(b, page_to_pfn(page), &hv_status);
+
if (locked) {
STATS_INC(b->stats.refused_alloc[false]);
@@ -594,13 +605,13 @@ static int vmballoon_lock_page(struct vmballoon *b, unsigned int num_pages,
}
static int vmballoon_lock_batched_page(struct vmballoon *b,
- unsigned int num_pages, bool is_2m_pages, unsigned int *target)
+ unsigned int num_pages, bool is_2m_pages)
{
int locked, i;
u16 size_per_page = vmballoon_page_size(is_2m_pages);
- locked = vmballoon_send_batched_lock(b, num_pages, is_2m_pages,
- target);
+ locked = vmballoon_send_batched_lock(b, num_pages, is_2m_pages);
+
if (locked > 0) {
for (i = 0; i < num_pages; i++) {
u64 pa = vmballoon_batch_get_pa(b->batch_page, i);
@@ -653,14 +664,14 @@ static int vmballoon_lock_batched_page(struct vmballoon *b,
* to use, if needed.
*/
static int vmballoon_unlock_page(struct vmballoon *b, unsigned int num_pages,
- bool is_2m_pages, unsigned int *target)
+ bool is_2m_pages)
{
struct page *page = b->page;
struct vmballoon_page_size *page_size = &b->page_sizes[false];
/* is_2m_pages can never happen as 2m pages support implies batching */
- if (!vmballoon_send_unlock_page(b, page_to_pfn(page), target)) {
+ if (!vmballoon_send_unlock_page(b, page_to_pfn(page))) {
list_add(&page->lru, &page_size->pages);
return -EIO;
}
@@ -676,15 +687,14 @@ static int vmballoon_unlock_page(struct vmballoon *b, unsigned int num_pages,
}
static int vmballoon_unlock_batched_page(struct vmballoon *b,
- unsigned int num_pages, bool is_2m_pages,
- unsigned int *target)
+ unsigned int num_pages, bool is_2m_pages)
{
int locked, i, ret = 0;
bool hv_success;
u16 size_per_page = vmballoon_page_size(is_2m_pages);
- hv_success = vmballoon_send_batched_unlock(b, num_pages, is_2m_pages,
- target);
+ hv_success = vmballoon_send_batched_unlock(b, num_pages, is_2m_pages);
+
if (!hv_success)
ret = -EIO;
@@ -799,7 +809,7 @@ static void vmballoon_inflate(struct vmballoon *b)
STATS_INC(b->stats.alloc_fail[is_2m_pages]);
if (is_2m_pages) {
- b->ops->lock(b, num_pages, true, &b->target);
+ b->ops->lock(b, num_pages, true);
/*
* ignore errors from locking as we now switch
@@ -838,8 +848,8 @@ static void vmballoon_inflate(struct vmballoon *b)
b->ops->add_page(b, num_pages++, page);
if (num_pages == b->batch_max_pages) {
- error = b->ops->lock(b, num_pages, is_2m_pages,
- &b->target);
+ error = b->ops->lock(b, num_pages, is_2m_pages);
+
num_pages = 0;
if (error)
break;
@@ -849,7 +859,7 @@ static void vmballoon_inflate(struct vmballoon *b)
}
if (num_pages > 0)
- b->ops->lock(b, num_pages, is_2m_pages, &b->target);
+ b->ops->lock(b, num_pages, is_2m_pages);
vmballoon_release_refused_pages(b, true);
vmballoon_release_refused_pages(b, false);
@@ -887,7 +897,7 @@ static void vmballoon_deflate(struct vmballoon *b)
int error;
error = b->ops->unlock(b, num_pages,
- is_2m_pages, &b->target);
+ is_2m_pages);
num_pages = 0;
if (error)
return;
@@ -897,7 +907,7 @@ static void vmballoon_deflate(struct vmballoon *b)
}
if (num_pages > 0)
- b->ops->unlock(b, num_pages, is_2m_pages, &b->target);
+ b->ops->unlock(b, num_pages, is_2m_pages);
}
}
@@ -942,10 +952,9 @@ static void vmballoon_doorbell(void *client_data)
*/
static void vmballoon_vmci_cleanup(struct vmballoon *b)
{
- int error;
+ vmballoon_cmd(b, VMW_BALLOON_CMD_VMCI_DOORBELL_SET,
+ VMCI_INVALID_ID, VMCI_INVALID_ID);
- VMWARE_BALLOON_CMD(VMCI_DOORBELL_SET, VMCI_INVALID_ID,
- VMCI_INVALID_ID, error);
STATS_INC(b->stats.doorbell_unset);
if (!vmci_handle_is_invalid(b->vmci_doorbell)) {
@@ -959,7 +968,7 @@ static void vmballoon_vmci_cleanup(struct vmballoon *b)
*/
static int vmballoon_vmci_init(struct vmballoon *b)
{
- unsigned long error, dummy;
+ unsigned long error;
if ((b->capabilities & VMW_BALLOON_SIGNALLED_WAKEUP_CMD) == 0)
return 0;
@@ -971,8 +980,9 @@ static int vmballoon_vmci_init(struct vmballoon *b)
if (error != VMCI_SUCCESS)
goto fail;
- error = VMWARE_BALLOON_CMD(VMCI_DOORBELL_SET, b->vmci_doorbell.context,
- b->vmci_doorbell.resource, dummy);
+ error = __vmballoon_cmd(b, VMW_BALLOON_CMD_VMCI_DOORBELL_SET,
+ b->vmci_doorbell.context,
+ b->vmci_doorbell.resource, NULL);
STATS_INC(b->stats.doorbell_set);
@@ -1038,17 +1048,16 @@ static void vmballoon_work(struct work_struct *work)
{
struct delayed_work *dwork = to_delayed_work(work);
struct vmballoon *b = container_of(dwork, struct vmballoon, dwork);
- unsigned int target;
STATS_INC(b->stats.timer);
if (b->reset_required)
vmballoon_reset(b);
- if (!b->reset_required && vmballoon_send_get_target(b, &target)) {
- /* update target, adjust size */
- b->target = target;
+ if (!b->reset_required && vmballoon_send_get_target(b)) {
+ unsigned long target = b->target;
+ /* update target, adjust size */
if (b->size < target)
vmballoon_inflate(b);
else if (target == 0 ||
--
2.17.1
^ permalink raw reply related [flat|nested] 24+ messages in thread* [PATCH v2 02/20] vmw_balloon: unify commands tracing and stats
2018-09-20 17:30 [PATCH v2 00/20] vmw_balloon: compaction, shrinker, 64-bit, etc Nadav Amit
2018-09-20 17:30 ` [PATCH v2 01/20] vmw_balloon: handle commands in a single function Nadav Amit
@ 2018-09-20 17:30 ` Nadav Amit
2018-09-20 17:30 ` [PATCH v2 03/20] vmw_balloon: merge send_lock and send_unlock path Nadav Amit
` (18 subsequent siblings)
20 siblings, 0 replies; 24+ messages in thread
From: Nadav Amit @ 2018-09-20 17:30 UTC (permalink / raw)
To: Greg Kroah-Hartman, Arnd Bergmann
Cc: linux-kernel, Xavier Deguillard, Nadav Amit
Now that we have a single point, unify the tracing and collecting the
statistics for commands and their failure. While it might somewhat
reduce the control over debugging, it cleans the code a lot.
Reviewed-by: Xavier Deguillard <xdeguillard@vmware.com>
Signed-off-by: Nadav Amit <namit@vmware.com>
---
drivers/misc/vmw_balloon.c | 116 +++++++++++++------------------------
1 file changed, 41 insertions(+), 75 deletions(-)
diff --git a/drivers/misc/vmw_balloon.c b/drivers/misc/vmw_balloon.c
index 0a4d5501f805..2ed4875319c8 100644
--- a/drivers/misc/vmw_balloon.c
+++ b/drivers/misc/vmw_balloon.c
@@ -105,6 +105,7 @@ enum vmwballoon_capabilities {
#define VMW_BALLOON_CMD_BATCHED_2M_UNLOCK 9
#define VMW_BALLOON_CMD_VMCI_DOORBELL_SET 10
+#define VMW_BALLOON_CMD_NUM 11
/* error codes */
#define VMW_BALLOON_SUCCESS 0
@@ -147,6 +148,19 @@ enum vmwballoon_capabilities {
(1UL << VMW_BALLOON_CMD_BATCHED_2M_LOCK) | \
(1UL << VMW_BALLOON_CMD_BATCHED_2M_UNLOCK))
+static const char * const vmballoon_cmd_names[] = {
+ [VMW_BALLOON_CMD_START] = "start",
+ [VMW_BALLOON_CMD_GET_TARGET] = "target",
+ [VMW_BALLOON_CMD_LOCK] = "lock",
+ [VMW_BALLOON_CMD_UNLOCK] = "unlock",
+ [VMW_BALLOON_CMD_GUEST_ID] = "guestType",
+ [VMW_BALLOON_CMD_BATCHED_LOCK] = "batchLock",
+ [VMW_BALLOON_CMD_BATCHED_UNLOCK] = "batchUnlock",
+ [VMW_BALLOON_CMD_BATCHED_2M_LOCK] = "2m-lock",
+ [VMW_BALLOON_CMD_BATCHED_2M_UNLOCK] = "2m-unlock",
+ [VMW_BALLOON_CMD_VMCI_DOORBELL_SET] = "doorbellSet"
+};
+
struct vmballoon_batch_page {
u64 pages[VMW_BALLOON_BATCH_MAX_PAGES];
};
@@ -182,19 +196,9 @@ struct vmballoon_stats {
unsigned int refused_free[VMW_BALLOON_NUM_PAGE_SIZES];
unsigned int free[VMW_BALLOON_NUM_PAGE_SIZES];
- /* monitor operations */
- unsigned int lock[VMW_BALLOON_NUM_PAGE_SIZES];
- unsigned int lock_fail[VMW_BALLOON_NUM_PAGE_SIZES];
- unsigned int unlock[VMW_BALLOON_NUM_PAGE_SIZES];
- unsigned int unlock_fail[VMW_BALLOON_NUM_PAGE_SIZES];
- unsigned int target;
- unsigned int target_fail;
- unsigned int start;
- unsigned int start_fail;
- unsigned int guest_type;
- unsigned int guest_type_fail;
- unsigned int doorbell_set;
- unsigned int doorbell_unset;
+ /* Monitor operations. */
+ unsigned long ops[VMW_BALLOON_CMD_NUM];
+ unsigned long ops_fail[VMW_BALLOON_CMD_NUM];
};
#define STATS_INC(stat) (stat)++
@@ -265,6 +269,8 @@ __vmballoon_cmd(struct vmballoon *b, unsigned long cmd, unsigned long arg1,
{
unsigned long status, dummy1, dummy2, dummy3, local_result;
+ STATS_INC(b->stats.ops[cmd]);
+
asm volatile ("inl %%dx" :
"=a"(status),
"=c"(dummy1),
@@ -288,6 +294,14 @@ __vmballoon_cmd(struct vmballoon *b, unsigned long cmd, unsigned long arg1,
((1ul << cmd) & VMW_BALLOON_CMD_WITH_TARGET_MASK))
b->target = local_result;
+ if (status != VMW_BALLOON_SUCCESS &&
+ status != VMW_BALLOON_SUCCESS_WITH_CAPABILITIES) {
+ STATS_INC(b->stats.ops_fail[cmd]);
+ pr_debug("%s: %s [0x%lx,0x%lx) failed, returned %ld\n",
+ __func__, vmballoon_cmd_names[cmd], arg1, arg2,
+ status);
+ }
+
/* mark reset required accordingly */
if (status == VMW_BALLOON_ERROR_RESET)
b->reset_required = true;
@@ -313,8 +327,6 @@ static bool vmballoon_send_start(struct vmballoon *b, unsigned long req_caps)
unsigned long status, capabilities;
bool success;
- STATS_INC(b->stats.start);
-
status = __vmballoon_cmd(b, VMW_BALLOON_CMD_START, req_caps, 0,
&capabilities);
@@ -342,10 +354,6 @@ static bool vmballoon_send_start(struct vmballoon *b, unsigned long req_caps)
else
b->supported_page_sizes = 1;
- if (!success) {
- pr_debug("%s - failed, hv returns %ld\n", __func__, status);
- STATS_INC(b->stats.start_fail);
- }
return success;
}
@@ -362,13 +370,9 @@ static bool vmballoon_send_guest_id(struct vmballoon *b)
status = vmballoon_cmd(b, VMW_BALLOON_CMD_GUEST_ID,
VMW_BALLOON_GUEST_ID, 0);
- STATS_INC(b->stats.guest_type);
-
if (status == VMW_BALLOON_SUCCESS)
return true;
- pr_debug("%s - failed, hv returns %ld\n", __func__, status);
- STATS_INC(b->stats.guest_type_fail);
return false;
}
@@ -402,16 +406,11 @@ static bool vmballoon_send_get_target(struct vmballoon *b)
if (limit != limit32)
return false;
- /* update stats */
- STATS_INC(b->stats.target);
-
status = vmballoon_cmd(b, VMW_BALLOON_CMD_GET_TARGET, limit, 0);
if (status == VMW_BALLOON_SUCCESS)
return true;
- pr_debug("%s - failed, hv returns %ld\n", __func__, status);
- STATS_INC(b->stats.target_fail);
return false;
}
@@ -430,15 +429,11 @@ static int vmballoon_send_lock_page(struct vmballoon *b, unsigned long pfn,
if (pfn32 != pfn)
return -EINVAL;
- STATS_INC(b->stats.lock[false]);
-
*hv_status = status = vmballoon_cmd(b, VMW_BALLOON_CMD_LOCK, pfn, 0);
if (status == VMW_BALLOON_SUCCESS)
return 0;
- pr_debug("%s - ppn %lx, hv returns %ld\n", __func__, pfn, status);
- STATS_INC(b->stats.lock_fail[false]);
return -EIO;
}
@@ -448,8 +443,6 @@ static int vmballoon_send_batched_lock(struct vmballoon *b,
unsigned long pfn = PHYS_PFN(virt_to_phys(b->batch_page));
unsigned long status, cmd;
- STATS_INC(b->stats.lock[is_2m_pages]);
-
cmd = is_2m_pages ? VMW_BALLOON_CMD_BATCHED_2M_LOCK :
VMW_BALLOON_CMD_BATCHED_LOCK;
@@ -458,8 +451,6 @@ static int vmballoon_send_batched_lock(struct vmballoon *b,
if (status == VMW_BALLOON_SUCCESS)
return 0;
- pr_debug("%s - batch ppn %lx, hv returns %ld\n", __func__, pfn, status);
- STATS_INC(b->stats.lock_fail[is_2m_pages]);
return 1;
}
@@ -476,15 +467,8 @@ static bool vmballoon_send_unlock_page(struct vmballoon *b, unsigned long pfn)
if (pfn32 != pfn)
return false;
- STATS_INC(b->stats.unlock[false]);
-
status = vmballoon_cmd(b, VMW_BALLOON_CMD_UNLOCK, pfn, 0);
- if (status == VMW_BALLOON_SUCCESS)
- return true;
-
- pr_debug("%s - ppn %lx, hv returns %ld\n", __func__, pfn, status);
- STATS_INC(b->stats.unlock_fail[false]);
- return false;
+ return status == VMW_BALLOON_SUCCESS;
}
static bool vmballoon_send_batched_unlock(struct vmballoon *b,
@@ -493,19 +477,12 @@ static bool vmballoon_send_batched_unlock(struct vmballoon *b,
unsigned long pfn = PHYS_PFN(virt_to_phys(b->batch_page));
unsigned long status, cmd;
- STATS_INC(b->stats.unlock[is_2m_pages]);
-
cmd = is_2m_pages ? VMW_BALLOON_CMD_BATCHED_2M_UNLOCK :
VMW_BALLOON_CMD_BATCHED_UNLOCK;
status = vmballoon_cmd(b, cmd, pfn, num_pages);
- if (status == VMW_BALLOON_SUCCESS)
- return true;
-
- pr_debug("%s - batch ppn %lx, hv returns %ld\n", __func__, pfn, status);
- STATS_INC(b->stats.unlock_fail[is_2m_pages]);
- return false;
+ return status == VMW_BALLOON_SUCCESS;
}
static struct page *vmballoon_alloc_page(gfp_t flags, bool is_2m_page)
@@ -955,8 +932,6 @@ static void vmballoon_vmci_cleanup(struct vmballoon *b)
vmballoon_cmd(b, VMW_BALLOON_CMD_VMCI_DOORBELL_SET,
VMCI_INVALID_ID, VMCI_INVALID_ID);
- STATS_INC(b->stats.doorbell_unset);
-
if (!vmci_handle_is_invalid(b->vmci_doorbell)) {
vmci_doorbell_destroy(b->vmci_doorbell);
b->vmci_doorbell = VMCI_INVALID_HANDLE;
@@ -984,8 +959,6 @@ static int vmballoon_vmci_init(struct vmballoon *b)
b->vmci_doorbell.context,
b->vmci_doorbell.resource, NULL);
- STATS_INC(b->stats.doorbell_set);
-
if (error != VMW_BALLOON_SUCCESS)
goto fail;
@@ -1082,6 +1055,7 @@ static int vmballoon_debug_show(struct seq_file *f, void *offset)
{
struct vmballoon *b = f->private;
struct vmballoon_stats *stats = &b->stats;
+ int i;
/* format capabilities info */
seq_printf(f,
@@ -1097,17 +1071,19 @@ static int vmballoon_debug_show(struct seq_file *f, void *offset)
"current: %8d pages\n",
b->target, b->size);
+ for (i = 0; i < VMW_BALLOON_CMD_NUM; i++) {
+ if (vmballoon_cmd_names[i] == NULL)
+ continue;
+
+ seq_printf(f, "%-22s: %16lu (%lu failed)\n",
+ vmballoon_cmd_names[i], stats->ops[i],
+ stats->ops_fail[i]);
+ }
+
seq_printf(f,
"\n"
"timer: %8u\n"
"doorbell: %8u\n"
- "start: %8u (%4u failed)\n"
- "guestType: %8u (%4u failed)\n"
- "2m-lock: %8u (%4u failed)\n"
- "lock: %8u (%4u failed)\n"
- "2m-unlock: %8u (%4u failed)\n"
- "unlock: %8u (%4u failed)\n"
- "target: %8u (%4u failed)\n"
"prim2mAlloc: %8u (%4u failed)\n"
"primNoSleepAlloc: %8u (%4u failed)\n"
"primCanSleepAlloc: %8u (%4u failed)\n"
@@ -1116,26 +1092,16 @@ static int vmballoon_debug_show(struct seq_file *f, void *offset)
"err2mAlloc: %8u\n"
"errAlloc: %8u\n"
"err2mFree: %8u\n"
- "errFree: %8u\n"
- "doorbellSet: %8u\n"
- "doorbellUnset: %8u\n",
+ "errFree: %8u\n",
stats->timer,
stats->doorbell,
- stats->start, stats->start_fail,
- stats->guest_type, stats->guest_type_fail,
- stats->lock[true], stats->lock_fail[true],
- stats->lock[false], stats->lock_fail[false],
- stats->unlock[true], stats->unlock_fail[true],
- stats->unlock[false], stats->unlock_fail[false],
- stats->target, stats->target_fail,
stats->alloc[true], stats->alloc_fail[true],
stats->alloc[false], stats->alloc_fail[false],
stats->sleep_alloc, stats->sleep_alloc_fail,
stats->free[true],
stats->free[false],
stats->refused_alloc[true], stats->refused_alloc[false],
- stats->refused_free[true], stats->refused_free[false],
- stats->doorbell_set, stats->doorbell_unset);
+ stats->refused_free[true], stats->refused_free[false]);
return 0;
}
--
2.17.1
^ permalink raw reply related [flat|nested] 24+ messages in thread* [PATCH v2 03/20] vmw_balloon: merge send_lock and send_unlock path
2018-09-20 17:30 [PATCH v2 00/20] vmw_balloon: compaction, shrinker, 64-bit, etc Nadav Amit
2018-09-20 17:30 ` [PATCH v2 01/20] vmw_balloon: handle commands in a single function Nadav Amit
2018-09-20 17:30 ` [PATCH v2 02/20] vmw_balloon: unify commands tracing and stats Nadav Amit
@ 2018-09-20 17:30 ` Nadav Amit
2018-09-20 17:30 ` [PATCH v2 04/20] vmw_balloon: simplifying batch access Nadav Amit
` (17 subsequent siblings)
20 siblings, 0 replies; 24+ messages in thread
From: Nadav Amit @ 2018-09-20 17:30 UTC (permalink / raw)
To: Greg Kroah-Hartman, Arnd Bergmann
Cc: linux-kernel, Xavier Deguillard, Nadav Amit
The lock and unlock code paths are very similar, so avoid the duplicate
code by merging them together.
Reviewed-by: Xavier Deguillard <xdeguillard@vmware.com>
Signed-off-by: Nadav Amit <namit@vmware.com>
---
drivers/misc/vmw_balloon.c | 62 +++++++++++++-------------------------
1 file changed, 21 insertions(+), 41 deletions(-)
diff --git a/drivers/misc/vmw_balloon.c b/drivers/misc/vmw_balloon.c
index 2ed4875319c8..5e5f61f207f4 100644
--- a/drivers/misc/vmw_balloon.c
+++ b/drivers/misc/vmw_balloon.c
@@ -420,16 +420,18 @@ static bool vmballoon_send_get_target(struct vmballoon *b)
* check the return value and maybe submit a different page.
*/
static int vmballoon_send_lock_page(struct vmballoon *b, unsigned long pfn,
- unsigned int *hv_status)
+ unsigned int *hv_status, bool lock)
{
- unsigned long status;
+ unsigned long status, cmd;
u32 pfn32;
pfn32 = (u32)pfn;
if (pfn32 != pfn)
return -EINVAL;
- *hv_status = status = vmballoon_cmd(b, VMW_BALLOON_CMD_LOCK, pfn, 0);
+ cmd = lock ? VMW_BALLOON_CMD_LOCK : VMW_BALLOON_CMD_UNLOCK;
+
+ *hv_status = status = vmballoon_cmd(b, cmd, pfn, 0);
if (status == VMW_BALLOON_SUCCESS)
return 0;
@@ -438,13 +440,18 @@ static int vmballoon_send_lock_page(struct vmballoon *b, unsigned long pfn,
}
static int vmballoon_send_batched_lock(struct vmballoon *b,
- unsigned int num_pages, bool is_2m_pages)
+ unsigned int num_pages, bool is_2m_pages,
+ bool lock)
{
unsigned long pfn = PHYS_PFN(virt_to_phys(b->batch_page));
unsigned long status, cmd;
- cmd = is_2m_pages ? VMW_BALLOON_CMD_BATCHED_2M_LOCK :
- VMW_BALLOON_CMD_BATCHED_LOCK;
+ if (lock)
+ cmd = is_2m_pages ? VMW_BALLOON_CMD_BATCHED_2M_LOCK :
+ VMW_BALLOON_CMD_BATCHED_LOCK;
+ else
+ cmd = is_2m_pages ? VMW_BALLOON_CMD_BATCHED_2M_UNLOCK :
+ VMW_BALLOON_CMD_BATCHED_UNLOCK;
status = vmballoon_cmd(b, cmd, pfn, num_pages);
@@ -454,37 +461,6 @@ static int vmballoon_send_batched_lock(struct vmballoon *b,
return 1;
}
-/*
- * Notify the host that guest intends to release given page back into
- * the pool of available (to the guest) pages.
- */
-static bool vmballoon_send_unlock_page(struct vmballoon *b, unsigned long pfn)
-{
- unsigned long status;
- u32 pfn32;
-
- pfn32 = (u32)pfn;
- if (pfn32 != pfn)
- return false;
-
- status = vmballoon_cmd(b, VMW_BALLOON_CMD_UNLOCK, pfn, 0);
- return status == VMW_BALLOON_SUCCESS;
-}
-
-static bool vmballoon_send_batched_unlock(struct vmballoon *b,
- unsigned int num_pages, bool is_2m_pages)
-{
- unsigned long pfn = PHYS_PFN(virt_to_phys(b->batch_page));
- unsigned long status, cmd;
-
- cmd = is_2m_pages ? VMW_BALLOON_CMD_BATCHED_2M_UNLOCK :
- VMW_BALLOON_CMD_BATCHED_UNLOCK;
-
- status = vmballoon_cmd(b, cmd, pfn, num_pages);
-
- return status == VMW_BALLOON_SUCCESS;
-}
-
static struct page *vmballoon_alloc_page(gfp_t flags, bool is_2m_page)
{
if (is_2m_page)
@@ -546,7 +522,8 @@ static int vmballoon_lock_page(struct vmballoon *b, unsigned int num_pages,
/* is_2m_pages can never happen as 2m pages support implies batching */
- locked = vmballoon_send_lock_page(b, page_to_pfn(page), &hv_status);
+ locked = vmballoon_send_lock_page(b, page_to_pfn(page), &hv_status,
+ true);
if (locked) {
STATS_INC(b->stats.refused_alloc[false]);
@@ -587,7 +564,7 @@ static int vmballoon_lock_batched_page(struct vmballoon *b,
int locked, i;
u16 size_per_page = vmballoon_page_size(is_2m_pages);
- locked = vmballoon_send_batched_lock(b, num_pages, is_2m_pages);
+ locked = vmballoon_send_batched_lock(b, num_pages, is_2m_pages, true);
if (locked > 0) {
for (i = 0; i < num_pages; i++) {
@@ -645,10 +622,12 @@ static int vmballoon_unlock_page(struct vmballoon *b, unsigned int num_pages,
{
struct page *page = b->page;
struct vmballoon_page_size *page_size = &b->page_sizes[false];
+ unsigned int hv_status;
/* is_2m_pages can never happen as 2m pages support implies batching */
- if (!vmballoon_send_unlock_page(b, page_to_pfn(page))) {
+ if (!vmballoon_send_lock_page(b, page_to_pfn(page), &hv_status,
+ false)) {
list_add(&page->lru, &page_size->pages);
return -EIO;
}
@@ -670,7 +649,8 @@ static int vmballoon_unlock_batched_page(struct vmballoon *b,
bool hv_success;
u16 size_per_page = vmballoon_page_size(is_2m_pages);
- hv_success = vmballoon_send_batched_unlock(b, num_pages, is_2m_pages);
+ hv_success = vmballoon_send_batched_lock(b, num_pages, is_2m_pages,
+ false);
if (!hv_success)
ret = -EIO;
--
2.17.1
^ permalink raw reply related [flat|nested] 24+ messages in thread* [PATCH v2 04/20] vmw_balloon: simplifying batch access
2018-09-20 17:30 [PATCH v2 00/20] vmw_balloon: compaction, shrinker, 64-bit, etc Nadav Amit
` (2 preceding siblings ...)
2018-09-20 17:30 ` [PATCH v2 03/20] vmw_balloon: merge send_lock and send_unlock path Nadav Amit
@ 2018-09-20 17:30 ` Nadav Amit
2018-09-20 17:30 ` [PATCH v2 05/20] vmw_balloon: remove sleeping allocations Nadav Amit
` (16 subsequent siblings)
20 siblings, 0 replies; 24+ messages in thread
From: Nadav Amit @ 2018-09-20 17:30 UTC (permalink / raw)
To: Greg Kroah-Hartman, Arnd Bergmann
Cc: linux-kernel, Xavier Deguillard, Nadav Amit
The use of accessors for batch entries complicates the code and makes it
less readable. Remove it an instead use bit-fields.
Reviewed-by: Xavier Deguillard <xdeguillard@vmware.com>
Signed-off-by: Nadav Amit <namit@vmware.com>
---
drivers/misc/vmw_balloon.c | 81 ++++++++++++++------------------------
1 file changed, 30 insertions(+), 51 deletions(-)
diff --git a/drivers/misc/vmw_balloon.c b/drivers/misc/vmw_balloon.c
index 5e5f61f207f4..7af70fc988b9 100644
--- a/drivers/misc/vmw_balloon.c
+++ b/drivers/misc/vmw_balloon.c
@@ -121,24 +121,6 @@ enum vmwballoon_capabilities {
#define VMW_BALLOON_SUCCESS_WITH_CAPABILITIES (0x03000000)
-/* Batch page description */
-
-/*
- * Layout of a page in the batch page:
- *
- * +-------------+----------+--------+
- * | | | |
- * | Page number | Reserved | Status |
- * | | | |
- * +-------------+----------+--------+
- * 64 PAGE_SHIFT 6 0
- *
- * The reserved field should be set to 0.
- */
-#define VMW_BALLOON_BATCH_MAX_PAGES (PAGE_SIZE / sizeof(u64))
-#define VMW_BALLOON_BATCH_STATUS_MASK ((1UL << 5) - 1)
-#define VMW_BALLOON_BATCH_PAGE_MASK (~((1UL << PAGE_SHIFT) - 1))
-
#define VMW_BALLOON_CMD_WITH_TARGET_MASK \
((1UL << VMW_BALLOON_CMD_GET_TARGET) | \
(1UL << VMW_BALLOON_CMD_LOCK) | \
@@ -161,27 +143,6 @@ static const char * const vmballoon_cmd_names[] = {
[VMW_BALLOON_CMD_VMCI_DOORBELL_SET] = "doorbellSet"
};
-struct vmballoon_batch_page {
- u64 pages[VMW_BALLOON_BATCH_MAX_PAGES];
-};
-
-static u64 vmballoon_batch_get_pa(struct vmballoon_batch_page *batch, int idx)
-{
- return batch->pages[idx] & VMW_BALLOON_BATCH_PAGE_MASK;
-}
-
-static int vmballoon_batch_get_status(struct vmballoon_batch_page *batch,
- int idx)
-{
- return (int)(batch->pages[idx] & VMW_BALLOON_BATCH_STATUS_MASK);
-}
-
-static void vmballoon_batch_set_pa(struct vmballoon_batch_page *batch, int idx,
- u64 pa)
-{
- batch->pages[idx] = pa;
-}
-
#ifdef CONFIG_DEBUG_FS
struct vmballoon_stats {
unsigned int timer;
@@ -225,6 +186,19 @@ struct vmballoon_page_size {
unsigned int n_refused_pages;
};
+/**
+ * struct vmballoon_batch_entry - a batch entry for lock or unlock.
+ *
+ * @status: the status of the operation, which is written by the hypervisor.
+ * @reserved: reserved for future use. Must be set to zero.
+ * @pfn: the physical frame number of the page to be locked or unlocked.
+ */
+struct vmballoon_batch_entry {
+ u64 status : 5;
+ u64 reserved : PAGE_SHIFT - 5;
+ u64 pfn : 52;
+} __packed;
+
struct vmballoon {
struct vmballoon_page_size page_sizes[VMW_BALLOON_NUM_PAGE_SIZES];
@@ -240,7 +214,14 @@ struct vmballoon {
unsigned long capabilities;
- struct vmballoon_batch_page *batch_page;
+ /**
+ * @batch_page: pointer to communication batch page.
+ *
+ * When batching is used, batch_page points to a page, which holds up to
+ * %VMW_BALLOON_BATCH_MAX_PAGES entries for locking or unlocking.
+ */
+ struct vmballoon_batch_entry *batch_page;
+
unsigned int batch_max_pages;
struct page *page;
@@ -568,8 +549,7 @@ static int vmballoon_lock_batched_page(struct vmballoon *b,
if (locked > 0) {
for (i = 0; i < num_pages; i++) {
- u64 pa = vmballoon_batch_get_pa(b->batch_page, i);
- struct page *p = pfn_to_page(pa >> PAGE_SHIFT);
+ struct page *p = pfn_to_page(b->batch_page[i].pfn);
vmballoon_free_page(p, is_2m_pages);
}
@@ -578,12 +558,11 @@ static int vmballoon_lock_batched_page(struct vmballoon *b,
}
for (i = 0; i < num_pages; i++) {
- u64 pa = vmballoon_batch_get_pa(b->batch_page, i);
- struct page *p = pfn_to_page(pa >> PAGE_SHIFT);
+ struct page *p = pfn_to_page(b->batch_page[i].pfn);
struct vmballoon_page_size *page_size =
&b->page_sizes[is_2m_pages];
- locked = vmballoon_batch_get_status(b->batch_page, i);
+ locked = b->batch_page[i].status;
switch (locked) {
case VMW_BALLOON_SUCCESS:
@@ -656,12 +635,11 @@ static int vmballoon_unlock_batched_page(struct vmballoon *b,
ret = -EIO;
for (i = 0; i < num_pages; i++) {
- u64 pa = vmballoon_batch_get_pa(b->batch_page, i);
- struct page *p = pfn_to_page(pa >> PAGE_SHIFT);
+ struct page *p = pfn_to_page(b->batch_page[i].pfn);
struct vmballoon_page_size *page_size =
&b->page_sizes[is_2m_pages];
- locked = vmballoon_batch_get_status(b->batch_page, i);
+ locked = b->batch_page[i].status;
if (!hv_success || locked != VMW_BALLOON_SUCCESS) {
/*
* That page wasn't successfully unlocked by the
@@ -710,8 +688,8 @@ static void vmballoon_add_page(struct vmballoon *b, int idx, struct page *p)
static void vmballoon_add_batched_page(struct vmballoon *b, int idx,
struct page *p)
{
- vmballoon_batch_set_pa(b->batch_page, idx,
- (u64)page_to_pfn(p) << PAGE_SHIFT);
+ b->batch_page[idx] = (struct vmballoon_batch_entry)
+ { .pfn = page_to_pfn(p) };
}
/*
@@ -967,7 +945,8 @@ static void vmballoon_reset(struct vmballoon *b)
if ((b->capabilities & VMW_BALLOON_BATCHED_CMDS) != 0) {
b->ops = &vmballoon_batched_ops;
- b->batch_max_pages = VMW_BALLOON_BATCH_MAX_PAGES;
+ b->batch_max_pages = PAGE_SIZE / sizeof(struct
+ vmballoon_batch_entry);
if (!vmballoon_init_batching(b)) {
/*
* We failed to initialize batching, inform the monitor
--
2.17.1
^ permalink raw reply related [flat|nested] 24+ messages in thread* [PATCH v2 05/20] vmw_balloon: remove sleeping allocations
2018-09-20 17:30 [PATCH v2 00/20] vmw_balloon: compaction, shrinker, 64-bit, etc Nadav Amit
` (3 preceding siblings ...)
2018-09-20 17:30 ` [PATCH v2 04/20] vmw_balloon: simplifying batch access Nadav Amit
@ 2018-09-20 17:30 ` Nadav Amit
2018-09-20 17:30 ` [PATCH v2 06/20] vmw_balloon: change batch/single lock abstractions Nadav Amit
` (15 subsequent siblings)
20 siblings, 0 replies; 24+ messages in thread
From: Nadav Amit @ 2018-09-20 17:30 UTC (permalink / raw)
To: Greg Kroah-Hartman, Arnd Bergmann
Cc: linux-kernel, Xavier Deguillard, Nadav Amit
Splitting the allocations between sleeping and non-sleeping made some
sort of sense as long as rate-limiting was enabled. Now that it is
removed, we need to decide - either we want sleeping allocations or not.
Since no other Linux balloon driver (hv, Xen, virtio) uses sleeping
allocations, use the same approach.
We do distinguish, however, between 2MB allocations and 4kB allocations
and prevent reclamation on 2MB. In both cases, we avoid using emergency
low-memory pools, as it may cause undesired effects.
Reviewed-by: Xavier Deguillard <xdeguillard@vmware.com>
Signed-off-by: Nadav Amit <namit@vmware.com>
---
drivers/misc/vmw_balloon.c | 67 ++++++++++----------------------------
1 file changed, 18 insertions(+), 49 deletions(-)
diff --git a/drivers/misc/vmw_balloon.c b/drivers/misc/vmw_balloon.c
index 7af70fc988b9..952308997499 100644
--- a/drivers/misc/vmw_balloon.c
+++ b/drivers/misc/vmw_balloon.c
@@ -37,20 +37,20 @@ MODULE_ALIAS("vmware_vmmemctl");
MODULE_LICENSE("GPL");
/*
- * Use __GFP_HIGHMEM to allow pages from HIGHMEM zone. We don't
- * allow wait (__GFP_RECLAIM) for NOSLEEP page allocations. Use
- * __GFP_NOWARN, to suppress page allocation failure warnings.
+ * Use __GFP_HIGHMEM to allow pages from HIGHMEM zone. We don't allow wait
+ * (__GFP_RECLAIM) for huge page allocations. Use __GFP_NOWARN, to suppress page
+ * allocation failure warnings. Disallow access to emergency low-memory pools.
*/
-#define VMW_PAGE_ALLOC_NOSLEEP (__GFP_HIGHMEM|__GFP_NOWARN)
+#define VMW_HUGE_PAGE_ALLOC_FLAGS (__GFP_HIGHMEM|__GFP_NOWARN| \
+ __GFP_NOMEMALLOC)
/*
- * Use GFP_HIGHUSER when executing in a separate kernel thread
- * context and allocation can sleep. This is less stressful to
- * the guest memory system, since it allows the thread to block
- * while memory is reclaimed, and won't take pages from emergency
- * low-memory pools.
+ * Use __GFP_HIGHMEM to allow pages from HIGHMEM zone. We allow lightweight
+ * reclamation (__GFP_NORETRY). Use __GFP_NOWARN, to suppress page allocation
+ * failure warnings. Disallow access to emergency low-memory pools.
*/
-#define VMW_PAGE_ALLOC_CANSLEEP (GFP_HIGHUSER)
+#define VMW_PAGE_ALLOC_FLAGS (__GFP_HIGHMEM|__GFP_NOWARN| \
+ __GFP_NOMEMALLOC|__GFP_NORETRY)
/* Maximum number of refused pages we accumulate during inflation cycle */
#define VMW_BALLOON_MAX_REFUSED 16
@@ -151,8 +151,6 @@ struct vmballoon_stats {
/* allocation statistics */
unsigned int alloc[VMW_BALLOON_NUM_PAGE_SIZES];
unsigned int alloc_fail[VMW_BALLOON_NUM_PAGE_SIZES];
- unsigned int sleep_alloc;
- unsigned int sleep_alloc_fail;
unsigned int refused_alloc[VMW_BALLOON_NUM_PAGE_SIZES];
unsigned int refused_free[VMW_BALLOON_NUM_PAGE_SIZES];
unsigned int free[VMW_BALLOON_NUM_PAGE_SIZES];
@@ -442,12 +440,13 @@ static int vmballoon_send_batched_lock(struct vmballoon *b,
return 1;
}
-static struct page *vmballoon_alloc_page(gfp_t flags, bool is_2m_page)
+static struct page *vmballoon_alloc_page(bool is_2m_page)
{
if (is_2m_page)
- return alloc_pages(flags, VMW_BALLOON_2M_SHIFT);
+ return alloc_pages(VMW_HUGE_PAGE_ALLOC_FLAGS,
+ VMW_BALLOON_2M_SHIFT);
- return alloc_page(flags);
+ return alloc_page(VMW_PAGE_ALLOC_FLAGS);
}
static void vmballoon_free_page(struct page *page, bool is_2m_page)
@@ -701,7 +700,6 @@ static void vmballoon_inflate(struct vmballoon *b)
{
unsigned int num_pages = 0;
int error = 0;
- gfp_t flags = VMW_PAGE_ALLOC_NOSLEEP;
bool is_2m_pages;
pr_debug("%s - size: %d, target %d\n", __func__, b->size, b->target);
@@ -734,15 +732,10 @@ static void vmballoon_inflate(struct vmballoon *b)
< b->target) {
struct page *page;
- if (flags == VMW_PAGE_ALLOC_NOSLEEP)
- STATS_INC(b->stats.alloc[is_2m_pages]);
- else
- STATS_INC(b->stats.sleep_alloc);
-
- page = vmballoon_alloc_page(flags, is_2m_pages);
+ STATS_INC(b->stats.alloc[is_2m_pages]);
+ page = vmballoon_alloc_page(is_2m_pages);
if (!page) {
STATS_INC(b->stats.alloc_fail[is_2m_pages]);
-
if (is_2m_pages) {
b->ops->lock(b, num_pages, true);
@@ -756,29 +749,7 @@ static void vmballoon_inflate(struct vmballoon *b)
is_2m_pages = false;
continue;
}
-
- if (flags == VMW_PAGE_ALLOC_CANSLEEP) {
- /*
- * CANSLEEP page allocation failed, so guest
- * is under severe memory pressure. We just log
- * the event, but do not stop the inflation
- * due to its negative impact on performance.
- */
- STATS_INC(b->stats.sleep_alloc_fail);
- break;
- }
-
- /*
- * NOSLEEP page allocation failed, so the guest is
- * under memory pressure. Slowing down page alloctions
- * seems to be reasonable, but doing so might actually
- * cause the hypervisor to throttle us down, resulting
- * in degraded performance. We will count on the
- * scheduler and standard memory management mechanisms
- * for now.
- */
- flags = VMW_PAGE_ALLOC_CANSLEEP;
- continue;
+ break;
}
b->ops->add_page(b, num_pages++, page);
@@ -1044,8 +1015,7 @@ static int vmballoon_debug_show(struct seq_file *f, void *offset)
"timer: %8u\n"
"doorbell: %8u\n"
"prim2mAlloc: %8u (%4u failed)\n"
- "primNoSleepAlloc: %8u (%4u failed)\n"
- "primCanSleepAlloc: %8u (%4u failed)\n"
+ "prim4kAlloc: %8u (%4u failed)\n"
"prim2mFree: %8u\n"
"primFree: %8u\n"
"err2mAlloc: %8u\n"
@@ -1056,7 +1026,6 @@ static int vmballoon_debug_show(struct seq_file *f, void *offset)
stats->doorbell,
stats->alloc[true], stats->alloc_fail[true],
stats->alloc[false], stats->alloc_fail[false],
- stats->sleep_alloc, stats->sleep_alloc_fail,
stats->free[true],
stats->free[false],
stats->refused_alloc[true], stats->refused_alloc[false],
--
2.17.1
^ permalink raw reply related [flat|nested] 24+ messages in thread* [PATCH v2 06/20] vmw_balloon: change batch/single lock abstractions
2018-09-20 17:30 [PATCH v2 00/20] vmw_balloon: compaction, shrinker, 64-bit, etc Nadav Amit
` (4 preceding siblings ...)
2018-09-20 17:30 ` [PATCH v2 05/20] vmw_balloon: remove sleeping allocations Nadav Amit
@ 2018-09-20 17:30 ` Nadav Amit
2018-09-20 17:30 ` [PATCH v2 07/20] vmw_balloon: treat all refused pages equally Nadav Amit
` (14 subsequent siblings)
20 siblings, 0 replies; 24+ messages in thread
From: Nadav Amit @ 2018-09-20 17:30 UTC (permalink / raw)
To: Greg Kroah-Hartman, Arnd Bergmann
Cc: linux-kernel, Xavier Deguillard, Nadav Amit
The current abstractions for batch vs single operations seem suboptimal
and complicate the implementation of additional features (OOM,
compaction).
The immediate problem of the current abstractions is that they cause
differences in how operations are handled when batching is on or off.
For example, the refused_alloc counter is not updated when batching is
on. These discrepancies are caused by code redundancies.
Instead, this patch presents three type of operations, according to
whether batching is on or off: (1) add page, (2) communication with the
hypervisor and (3) retrieving the status of a page.
To avoid the overhead of virtual functions, and since we do not expect
additional interfaces for communication with the hypervisor, we use
static keys instead of virtual functions.
Finally, while we are at it, change vmballoon_init_batching() to return
int instead of bool, to be consistent in the return type and avoid
potential coding errors.
Reviewed-by: Xavier Deguillard <xdeguillard@vmware.com>
Signed-off-by: Nadav Amit <namit@vmware.com>
---
drivers/misc/vmw_balloon.c | 358 +++++++++++++++++--------------------
1 file changed, 165 insertions(+), 193 deletions(-)
diff --git a/drivers/misc/vmw_balloon.c b/drivers/misc/vmw_balloon.c
index 952308997499..96dde120bbd5 100644
--- a/drivers/misc/vmw_balloon.c
+++ b/drivers/misc/vmw_balloon.c
@@ -165,15 +165,7 @@ struct vmballoon_stats {
#define STATS_INC(stat)
#endif
-struct vmballoon;
-
-struct vmballoon_ops {
- void (*add_page)(struct vmballoon *b, int idx, struct page *p);
- int (*lock)(struct vmballoon *b, unsigned int num_pages,
- bool is_2m_pages);
- int (*unlock)(struct vmballoon *b, unsigned int num_pages,
- bool is_2m_pages);
-};
+static DEFINE_STATIC_KEY_TRUE(vmw_balloon_batching);
struct vmballoon_page_size {
/* list of reserved physical pages */
@@ -223,8 +215,6 @@ struct vmballoon {
unsigned int batch_max_pages;
struct page *page;
- const struct vmballoon_ops *ops;
-
#ifdef CONFIG_DEBUG_FS
/* statistics */
struct vmballoon_stats stats;
@@ -393,53 +383,6 @@ static bool vmballoon_send_get_target(struct vmballoon *b)
return false;
}
-/*
- * Notify the host about allocated page so that host can use it without
- * fear that guest will need it. Host may reject some pages, we need to
- * check the return value and maybe submit a different page.
- */
-static int vmballoon_send_lock_page(struct vmballoon *b, unsigned long pfn,
- unsigned int *hv_status, bool lock)
-{
- unsigned long status, cmd;
- u32 pfn32;
-
- pfn32 = (u32)pfn;
- if (pfn32 != pfn)
- return -EINVAL;
-
- cmd = lock ? VMW_BALLOON_CMD_LOCK : VMW_BALLOON_CMD_UNLOCK;
-
- *hv_status = status = vmballoon_cmd(b, cmd, pfn, 0);
-
- if (status == VMW_BALLOON_SUCCESS)
- return 0;
-
- return -EIO;
-}
-
-static int vmballoon_send_batched_lock(struct vmballoon *b,
- unsigned int num_pages, bool is_2m_pages,
- bool lock)
-{
- unsigned long pfn = PHYS_PFN(virt_to_phys(b->batch_page));
- unsigned long status, cmd;
-
- if (lock)
- cmd = is_2m_pages ? VMW_BALLOON_CMD_BATCHED_2M_LOCK :
- VMW_BALLOON_CMD_BATCHED_LOCK;
- else
- cmd = is_2m_pages ? VMW_BALLOON_CMD_BATCHED_2M_UNLOCK :
- VMW_BALLOON_CMD_BATCHED_UNLOCK;
-
- status = vmballoon_cmd(b, cmd, pfn, num_pages);
-
- if (status == VMW_BALLOON_SUCCESS)
- return 0;
-
- return 1;
-}
-
static struct page *vmballoon_alloc_page(bool is_2m_page)
{
if (is_2m_page)
@@ -488,88 +431,126 @@ static void vmballoon_pop(struct vmballoon *b)
b->batch_page = NULL;
}
-/*
- * Notify the host of a ballooned page. If host rejects the page put it on the
- * refuse list, those refused page are then released at the end of the
- * inflation cycle.
+/**
+ * vmballoon_status_page - returns the status of (un)lock operation
+ *
+ * @b: pointer to the balloon.
+ * @idx: index for the page for which the operation is performed.
+ * @p: pointer to where the page struct is returned.
+ *
+ * Following a lock or unlock operation, returns the status of the operation for
+ * an individual page. Provides the page that the operation was performed on on
+ * the @page argument.
+ *
+ * Returns: The status of a lock or unlock operation for an individual page.
*/
-static int vmballoon_lock_page(struct vmballoon *b, unsigned int num_pages,
- bool is_2m_pages)
+static unsigned long vmballoon_status_page(struct vmballoon *b, int idx,
+ struct page **p)
{
- int locked, hv_status;
- struct page *page = b->page;
- struct vmballoon_page_size *page_size = &b->page_sizes[false];
-
- /* is_2m_pages can never happen as 2m pages support implies batching */
-
- locked = vmballoon_send_lock_page(b, page_to_pfn(page), &hv_status,
- true);
+ if (static_branch_likely(&vmw_balloon_batching)) {
+ /* batching mode */
+ *p = pfn_to_page(b->batch_page[idx].pfn);
+ return b->batch_page[idx].status;
+ }
- if (locked) {
- STATS_INC(b->stats.refused_alloc[false]);
+ /* non-batching mode */
+ *p = b->page;
- if (locked == -EIO &&
- (hv_status == VMW_BALLOON_ERROR_RESET ||
- hv_status == VMW_BALLOON_ERROR_PPN_NOTNEEDED)) {
- vmballoon_free_page(page, false);
- return -EIO;
- }
+ /*
+ * If a failure occurs, the indication will be provided in the status
+ * of the entire operation, which is considered before the individual
+ * page status. So for non-batching mode, the indication is always of
+ * success.
+ */
+ return VMW_BALLOON_SUCCESS;
+}
- /*
- * Place page on the list of non-balloonable pages
- * and retry allocation, unless we already accumulated
- * too many of them, in which case take a breather.
- */
- if (page_size->n_refused_pages < VMW_BALLOON_MAX_REFUSED) {
- page_size->n_refused_pages++;
- list_add(&page->lru, &page_size->refused_pages);
- } else {
- vmballoon_free_page(page, false);
- }
- return locked;
+/**
+ * vmballoon_lock_op - notifies the host about inflated/deflated pages.
+ * @b: pointer to the balloon.
+ * @num_pages: number of inflated/deflated pages.
+ * @is_2m_pages: whether the page(s) are 2M (or 4k).
+ * @lock: whether the operation is lock (or unlock).
+ *
+ * Notify the host about page(s) that were ballooned (or removed from the
+ * balloon) so that host can use it without fear that guest will need it (or
+ * stop using them since the VM does). Host may reject some pages, we need to
+ * check the return value and maybe submit a different page. The pages that are
+ * inflated/deflated are pointed by @b->page.
+ *
+ * Return: result as provided by the hypervisor.
+ */
+static unsigned long vmballoon_lock_op(struct vmballoon *b,
+ unsigned int num_pages,
+ bool is_2m_pages, bool lock)
+{
+ unsigned long cmd, pfn;
+
+ if (static_branch_likely(&vmw_balloon_batching)) {
+ if (lock)
+ cmd = is_2m_pages ? VMW_BALLOON_CMD_BATCHED_2M_LOCK :
+ VMW_BALLOON_CMD_BATCHED_LOCK;
+ else
+ cmd = is_2m_pages ? VMW_BALLOON_CMD_BATCHED_2M_UNLOCK :
+ VMW_BALLOON_CMD_BATCHED_UNLOCK;
+
+ pfn = PHYS_PFN(virt_to_phys(b->batch_page));
+ } else {
+ cmd = lock ? VMW_BALLOON_CMD_LOCK : VMW_BALLOON_CMD_UNLOCK;
+ pfn = page_to_pfn(b->page);
+
+ /* In non-batching mode, PFNs must fit in 32-bit */
+ if (unlikely(pfn != (u32)pfn))
+ return VMW_BALLOON_ERROR_PPN_INVALID;
}
- /* track allocated page */
- list_add(&page->lru, &page_size->pages);
-
- /* update balloon size */
- b->size++;
-
- return 0;
+ return vmballoon_cmd(b, cmd, pfn, num_pages);
}
-static int vmballoon_lock_batched_page(struct vmballoon *b,
- unsigned int num_pages, bool is_2m_pages)
+static int vmballoon_lock(struct vmballoon *b, unsigned int num_pages,
+ bool is_2m_pages)
{
- int locked, i;
+ unsigned long batch_status;
+ int i;
u16 size_per_page = vmballoon_page_size(is_2m_pages);
- locked = vmballoon_send_batched_lock(b, num_pages, is_2m_pages, true);
-
- if (locked > 0) {
- for (i = 0; i < num_pages; i++) {
- struct page *p = pfn_to_page(b->batch_page[i].pfn);
-
- vmballoon_free_page(p, is_2m_pages);
- }
-
- return -EIO;
- }
+ batch_status = vmballoon_lock_op(b, num_pages, is_2m_pages, true);
for (i = 0; i < num_pages; i++) {
- struct page *p = pfn_to_page(b->batch_page[i].pfn);
+ unsigned long status;
+ struct page *p;
struct vmballoon_page_size *page_size =
&b->page_sizes[is_2m_pages];
- locked = b->batch_page[i].status;
+ status = vmballoon_status_page(b, i, &p);
+
+ /*
+ * Failure of the whole batch overrides a single operation
+ * results.
+ */
+ if (batch_status != VMW_BALLOON_SUCCESS)
+ status = batch_status;
- switch (locked) {
- case VMW_BALLOON_SUCCESS:
+ if (status == VMW_BALLOON_SUCCESS) {
+ /* track allocated page */
list_add(&p->lru, &page_size->pages);
+
+ /* update balloon size */
b->size += size_per_page;
- break;
+ continue;
+ }
+
+ /* Error occurred */
+ STATS_INC(b->stats.refused_alloc[is_2m_pages]);
+
+ switch (status) {
case VMW_BALLOON_ERROR_PPN_PINNED:
case VMW_BALLOON_ERROR_PPN_INVALID:
+ /*
+ * Place page on the list of non-balloonable pages
+ * and retry allocation, unless we already accumulated
+ * too many of them, in which case take a breather.
+ */
if (page_size->n_refused_pages
< VMW_BALLOON_MAX_REFUSED) {
list_add(&p->lru, &page_size->refused_pages);
@@ -587,7 +568,7 @@ static int vmballoon_lock_batched_page(struct vmballoon *b,
}
}
- return 0;
+ return batch_status == VMW_BALLOON_SUCCESS ? 0 : -EIO;
}
/*
@@ -595,51 +576,31 @@ static int vmballoon_lock_batched_page(struct vmballoon *b,
* the host so it can make sure the page will be available for the guest
* to use, if needed.
*/
-static int vmballoon_unlock_page(struct vmballoon *b, unsigned int num_pages,
- bool is_2m_pages)
+static int vmballoon_unlock(struct vmballoon *b, unsigned int num_pages,
+ bool is_2m_pages)
{
- struct page *page = b->page;
- struct vmballoon_page_size *page_size = &b->page_sizes[false];
- unsigned int hv_status;
-
- /* is_2m_pages can never happen as 2m pages support implies batching */
-
- if (!vmballoon_send_lock_page(b, page_to_pfn(page), &hv_status,
- false)) {
- list_add(&page->lru, &page_size->pages);
- return -EIO;
- }
-
- /* deallocate page */
- vmballoon_free_page(page, false);
- STATS_INC(b->stats.free[false]);
-
- /* update balloon size */
- b->size--;
-
- return 0;
-}
-
-static int vmballoon_unlock_batched_page(struct vmballoon *b,
- unsigned int num_pages, bool is_2m_pages)
-{
- int locked, i, ret = 0;
- bool hv_success;
+ int i;
+ unsigned long batch_status;
u16 size_per_page = vmballoon_page_size(is_2m_pages);
- hv_success = vmballoon_send_batched_lock(b, num_pages, is_2m_pages,
- false);
-
- if (!hv_success)
- ret = -EIO;
+ batch_status = vmballoon_lock_op(b, num_pages, is_2m_pages, false);
for (i = 0; i < num_pages; i++) {
- struct page *p = pfn_to_page(b->batch_page[i].pfn);
- struct vmballoon_page_size *page_size =
- &b->page_sizes[is_2m_pages];
+ struct vmballoon_page_size *page_size;
+ unsigned long status;
+ struct page *p;
- locked = b->batch_page[i].status;
- if (!hv_success || locked != VMW_BALLOON_SUCCESS) {
+ status = vmballoon_status_page(b, i, &p);
+ page_size = &b->page_sizes[is_2m_pages];
+
+ /*
+ * Failure of the whole batch overrides a single operation
+ * results.
+ */
+ if (batch_status != VMW_BALLOON_SUCCESS)
+ status = batch_status;
+
+ if (status != VMW_BALLOON_SUCCESS) {
/*
* That page wasn't successfully unlocked by the
* hypervisor, re-add it to the list of pages owned by
@@ -656,7 +617,7 @@ static int vmballoon_unlock_batched_page(struct vmballoon *b,
}
}
- return ret;
+ return batch_status == VMW_BALLOON_SUCCESS ? 0 : -EIO;
}
/*
@@ -681,14 +642,11 @@ static void vmballoon_release_refused_pages(struct vmballoon *b,
static void vmballoon_add_page(struct vmballoon *b, int idx, struct page *p)
{
- b->page = p;
-}
-
-static void vmballoon_add_batched_page(struct vmballoon *b, int idx,
- struct page *p)
-{
- b->batch_page[idx] = (struct vmballoon_batch_entry)
+ if (static_branch_likely(&vmw_balloon_batching))
+ b->batch_page[idx] = (struct vmballoon_batch_entry)
{ .pfn = page_to_pfn(p) };
+ else
+ b->page = p;
}
/*
@@ -737,7 +695,7 @@ static void vmballoon_inflate(struct vmballoon *b)
if (!page) {
STATS_INC(b->stats.alloc_fail[is_2m_pages]);
if (is_2m_pages) {
- b->ops->lock(b, num_pages, true);
+ vmballoon_lock(b, num_pages, true);
/*
* ignore errors from locking as we now switch
@@ -752,9 +710,9 @@ static void vmballoon_inflate(struct vmballoon *b)
break;
}
- b->ops->add_page(b, num_pages++, page);
+ vmballoon_add_page(b, num_pages++, page);
if (num_pages == b->batch_max_pages) {
- error = b->ops->lock(b, num_pages, is_2m_pages);
+ error = vmballoon_lock(b, num_pages, is_2m_pages);
num_pages = 0;
if (error)
@@ -765,7 +723,7 @@ static void vmballoon_inflate(struct vmballoon *b)
}
if (num_pages > 0)
- b->ops->lock(b, num_pages, is_2m_pages);
+ vmballoon_lock(b, num_pages, is_2m_pages);
vmballoon_release_refused_pages(b, true);
vmballoon_release_refused_pages(b, false);
@@ -797,12 +755,12 @@ static void vmballoon_deflate(struct vmballoon *b)
break;
list_del(&page->lru);
- b->ops->add_page(b, num_pages++, page);
+ vmballoon_add_page(b, num_pages++, page);
if (num_pages == b->batch_max_pages) {
int error;
- error = b->ops->unlock(b, num_pages,
+ error = vmballoon_unlock(b, num_pages,
is_2m_pages);
num_pages = 0;
if (error)
@@ -813,32 +771,50 @@ static void vmballoon_deflate(struct vmballoon *b)
}
if (num_pages > 0)
- b->ops->unlock(b, num_pages, is_2m_pages);
+ vmballoon_unlock(b, num_pages, is_2m_pages);
}
}
-static const struct vmballoon_ops vmballoon_basic_ops = {
- .add_page = vmballoon_add_page,
- .lock = vmballoon_lock_page,
- .unlock = vmballoon_unlock_page
-};
-
-static const struct vmballoon_ops vmballoon_batched_ops = {
- .add_page = vmballoon_add_batched_page,
- .lock = vmballoon_lock_batched_page,
- .unlock = vmballoon_unlock_batched_page
-};
+/**
+ * vmballoon_deinit_batching - disables batching mode.
+ *
+ * @b: pointer to &struct vmballoon.
+ *
+ * Disables batching, by deallocating the page for communication with the
+ * hypervisor and disabling the static key to indicate that batching is off.
+ */
+static void vmballoon_deinit_batching(struct vmballoon *b)
+{
+ free_page((unsigned long)b->batch_page);
+ b->batch_page = NULL;
+ static_branch_disable(&vmw_balloon_batching);
+ b->batch_max_pages = 1;
+}
-static bool vmballoon_init_batching(struct vmballoon *b)
+/**
+ * vmballoon_init_batching - enable batching mode.
+ *
+ * @b: pointer to &struct vmballoon.
+ *
+ * Enables batching, by allocating a page for communication with the hypervisor
+ * and enabling the static_key to use batching.
+ *
+ * Return: zero on success or an appropriate error-code.
+ */
+static int vmballoon_init_batching(struct vmballoon *b)
{
struct page *page;
page = alloc_page(GFP_KERNEL | __GFP_ZERO);
if (!page)
- return false;
+ return -ENOMEM;
b->batch_page = page_address(page);
- return true;
+ b->batch_max_pages = PAGE_SIZE / sizeof(struct vmballoon_batch_entry);
+
+ static_branch_enable(&vmw_balloon_batching);
+
+ return 0;
}
/*
@@ -915,10 +891,7 @@ static void vmballoon_reset(struct vmballoon *b)
return;
if ((b->capabilities & VMW_BALLOON_BATCHED_CMDS) != 0) {
- b->ops = &vmballoon_batched_ops;
- b->batch_max_pages = PAGE_SIZE / sizeof(struct
- vmballoon_batch_entry);
- if (!vmballoon_init_batching(b)) {
+ if (vmballoon_init_batching(b)) {
/*
* We failed to initialize batching, inform the monitor
* about it by sending a null capability.
@@ -929,8 +902,7 @@ static void vmballoon_reset(struct vmballoon *b)
return;
}
} else if ((b->capabilities & VMW_BALLOON_BASIC_CMDS) != 0) {
- b->ops = &vmballoon_basic_ops;
- b->batch_max_pages = 1;
+ vmballoon_deinit_batching(b);
}
b->reset_required = false;
--
2.17.1
^ permalink raw reply related [flat|nested] 24+ messages in thread* [PATCH v2 07/20] vmw_balloon: treat all refused pages equally
2018-09-20 17:30 [PATCH v2 00/20] vmw_balloon: compaction, shrinker, 64-bit, etc Nadav Amit
` (5 preceding siblings ...)
2018-09-20 17:30 ` [PATCH v2 06/20] vmw_balloon: change batch/single lock abstractions Nadav Amit
@ 2018-09-20 17:30 ` Nadav Amit
2018-09-20 17:30 ` [PATCH v2 08/20] vmw_balloon: rename VMW_BALLOON_2M_SHIFT to VMW_BALLOON_2M_ORDER Nadav Amit
` (13 subsequent siblings)
20 siblings, 0 replies; 24+ messages in thread
From: Nadav Amit @ 2018-09-20 17:30 UTC (permalink / raw)
To: Greg Kroah-Hartman, Arnd Bergmann
Cc: linux-kernel, Xavier Deguillard, Nadav Amit
Currently, when the hypervisor rejects a page during lock operation, the
VM treats pages differently according to the error-code: in certain
cases the page is immediately freed, and in others it is put on a
rejection list and only freed later.
The behavior does not make too much sense. If the page is freed
immediately it is very likely to be used again in the next batch of
allocations, and be rejected again.
In addition, for support of compaction and OOM notifiers, we wish to
separate the logic that communicates with the hypervisor (as well as
analyzes the status of each page) from the logic that allocates or free
pages.
Treat all errors the same way, queuing the pages on the refuse list.
Move to the next allocation size (4k) when too many pages are refused.
Free the refused pages when moving to the next size to avoid situations
in which too much memory is waiting to be freed on the refused list.
Reviewed-by: Xavier Deguillard <xdeguillard@vmware.com>
Signed-off-by: Nadav Amit <namit@vmware.com>
---
drivers/misc/vmw_balloon.c | 52 +++++++++++++++++++++-----------------
1 file changed, 29 insertions(+), 23 deletions(-)
diff --git a/drivers/misc/vmw_balloon.c b/drivers/misc/vmw_balloon.c
index 96dde120bbd5..4e067d269706 100644
--- a/drivers/misc/vmw_balloon.c
+++ b/drivers/misc/vmw_balloon.c
@@ -543,29 +543,13 @@ static int vmballoon_lock(struct vmballoon *b, unsigned int num_pages,
/* Error occurred */
STATS_INC(b->stats.refused_alloc[is_2m_pages]);
- switch (status) {
- case VMW_BALLOON_ERROR_PPN_PINNED:
- case VMW_BALLOON_ERROR_PPN_INVALID:
- /*
- * Place page on the list of non-balloonable pages
- * and retry allocation, unless we already accumulated
- * too many of them, in which case take a breather.
- */
- if (page_size->n_refused_pages
- < VMW_BALLOON_MAX_REFUSED) {
- list_add(&p->lru, &page_size->refused_pages);
- page_size->n_refused_pages++;
- break;
- }
- /* Fallthrough */
- case VMW_BALLOON_ERROR_RESET:
- case VMW_BALLOON_ERROR_PPN_NOTNEEDED:
- vmballoon_free_page(p, is_2m_pages);
- break;
- default:
- /* This should never happen */
- WARN_ON_ONCE(true);
- }
+ /*
+ * Place page on the list of non-balloonable pages
+ * and retry allocation, unless we already accumulated
+ * too many of them, in which case take a breather.
+ */
+ list_add(&p->lru, &page_size->refused_pages);
+ page_size->n_refused_pages++;
}
return batch_status == VMW_BALLOON_SUCCESS ? 0 : -EIO;
@@ -712,9 +696,31 @@ static void vmballoon_inflate(struct vmballoon *b)
vmballoon_add_page(b, num_pages++, page);
if (num_pages == b->batch_max_pages) {
+ struct vmballoon_page_size *page_size =
+ &b->page_sizes[is_2m_pages];
+
error = vmballoon_lock(b, num_pages, is_2m_pages);
num_pages = 0;
+
+ /*
+ * Stop allocating this page size if we already
+ * accumulated too many pages that the hypervisor
+ * refused.
+ */
+ if (page_size->n_refused_pages >=
+ VMW_BALLOON_MAX_REFUSED) {
+ if (!is_2m_pages)
+ break;
+
+ /*
+ * Release the refused pages as we move to 4k
+ * pages.
+ */
+ vmballoon_release_refused_pages(b, true);
+ is_2m_pages = true;
+ }
+
if (error)
break;
}
--
2.17.1
^ permalink raw reply related [flat|nested] 24+ messages in thread* [PATCH v2 08/20] vmw_balloon: rename VMW_BALLOON_2M_SHIFT to VMW_BALLOON_2M_ORDER
2018-09-20 17:30 [PATCH v2 00/20] vmw_balloon: compaction, shrinker, 64-bit, etc Nadav Amit
` (6 preceding siblings ...)
2018-09-20 17:30 ` [PATCH v2 07/20] vmw_balloon: treat all refused pages equally Nadav Amit
@ 2018-09-20 17:30 ` Nadav Amit
2018-09-20 17:30 ` [PATCH v2 09/20] vmw_balloon: refactor change size from vmballoon_work Nadav Amit
` (12 subsequent siblings)
20 siblings, 0 replies; 24+ messages in thread
From: Nadav Amit @ 2018-09-20 17:30 UTC (permalink / raw)
To: Greg Kroah-Hartman, Arnd Bergmann
Cc: linux-kernel, Xavier Deguillard, Nadav Amit
The name of the macro'd VMW_BALLOON_2M_SHIFT is misleading. The value
reflects 2M huge-page order. Unfortunately, we cannot use
HPAGE_PMD_ORDER, since it is not defined when transparent huge-pages are
off, so we need to define our own one.
Rename it to VMW_BALLOON_2M_ORDER. No functional change.
Signed-off-by: Nadav Amit <namit@vmware.com>
---
drivers/misc/vmw_balloon.c | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/drivers/misc/vmw_balloon.c b/drivers/misc/vmw_balloon.c
index 4e067d269706..626abe65eddb 100644
--- a/drivers/misc/vmw_balloon.c
+++ b/drivers/misc/vmw_balloon.c
@@ -77,7 +77,7 @@ enum vmwballoon_capabilities {
| VMW_BALLOON_BATCHED_2M_CMDS \
| VMW_BALLOON_SIGNALLED_WAKEUP_CMD)
-#define VMW_BALLOON_2M_SHIFT (9)
+#define VMW_BALLOON_2M_ORDER (PMD_SHIFT - PAGE_SHIFT)
#define VMW_BALLOON_NUM_PAGE_SIZES (2)
/*
@@ -348,7 +348,7 @@ static bool vmballoon_send_guest_id(struct vmballoon *b)
static u16 vmballoon_page_size(bool is_2m_page)
{
if (is_2m_page)
- return 1 << VMW_BALLOON_2M_SHIFT;
+ return 1 << VMW_BALLOON_2M_ORDER;
return 1;
}
@@ -387,7 +387,7 @@ static struct page *vmballoon_alloc_page(bool is_2m_page)
{
if (is_2m_page)
return alloc_pages(VMW_HUGE_PAGE_ALLOC_FLAGS,
- VMW_BALLOON_2M_SHIFT);
+ VMW_BALLOON_2M_ORDER);
return alloc_page(VMW_PAGE_ALLOC_FLAGS);
}
@@ -395,7 +395,7 @@ static struct page *vmballoon_alloc_page(bool is_2m_page)
static void vmballoon_free_page(struct page *page, bool is_2m_page)
{
if (is_2m_page)
- __free_pages(page, VMW_BALLOON_2M_SHIFT);
+ __free_pages(page, VMW_BALLOON_2M_ORDER);
else
__free_page(page);
}
--
2.17.1
^ permalink raw reply related [flat|nested] 24+ messages in thread* [PATCH v2 09/20] vmw_balloon: refactor change size from vmballoon_work
2018-09-20 17:30 [PATCH v2 00/20] vmw_balloon: compaction, shrinker, 64-bit, etc Nadav Amit
` (7 preceding siblings ...)
2018-09-20 17:30 ` [PATCH v2 08/20] vmw_balloon: rename VMW_BALLOON_2M_SHIFT to VMW_BALLOON_2M_ORDER Nadav Amit
@ 2018-09-20 17:30 ` Nadav Amit
2018-09-20 17:30 ` [PATCH v2 10/20] vmw_balloon: simplify vmballoon_send_get_target() Nadav Amit
` (11 subsequent siblings)
20 siblings, 0 replies; 24+ messages in thread
From: Nadav Amit @ 2018-09-20 17:30 UTC (permalink / raw)
To: Greg Kroah-Hartman, Arnd Bergmann
Cc: linux-kernel, Xavier Deguillard, Nadav Amit
The required change in the balloon size is currently computed in
vmballoon_work(), vmballoon_inflate() and vmballoon_deflate(). Refactor
it to simplify the next patches.
Reviewed-by: Xavier Deguillard <xdeguillard@vmware.com>
Signed-off-by: Nadav Amit <namit@vmware.com>
---
drivers/misc/vmw_balloon.c | 75 ++++++++++++++++++++++++++------------
1 file changed, 52 insertions(+), 23 deletions(-)
diff --git a/drivers/misc/vmw_balloon.c b/drivers/misc/vmw_balloon.c
index 626abe65eddb..0d9f223d463e 100644
--- a/drivers/misc/vmw_balloon.c
+++ b/drivers/misc/vmw_balloon.c
@@ -633,6 +633,37 @@ static void vmballoon_add_page(struct vmballoon *b, int idx, struct page *p)
b->page = p;
}
+/**
+ * vmballoon_change - retrieve the required balloon change
+ *
+ * @b: pointer for the balloon.
+ *
+ * Return: the required change for the balloon size. A positive number
+ * indicates inflation, a negative number indicates a deflation.
+ */
+static int64_t vmballoon_change(struct vmballoon *b)
+{
+ int64_t size, target;
+
+ size = b->size;
+ target = b->target;
+
+ /*
+ * We must cast first because of int sizes
+ * Otherwise we might get huge positives instead of negatives
+ */
+
+ if (b->reset_required)
+ return 0;
+
+ /* consider a 2MB slack on deflate, unless the balloon is emptied */
+ if (target < size && size - target < vmballoon_page_size(true) &&
+ target != 0)
+ return 0;
+
+ return target - size;
+}
+
/*
* Inflate the balloon towards its target size. Note that we try to limit
* the rate of allocation to make sure we are not choking the rest of the
@@ -644,8 +675,6 @@ static void vmballoon_inflate(struct vmballoon *b)
int error = 0;
bool is_2m_pages;
- pr_debug("%s - size: %d, target %d\n", __func__, b->size, b->target);
-
/*
* First try NOSLEEP page allocations to inflate balloon.
*
@@ -667,11 +696,8 @@ static void vmballoon_inflate(struct vmballoon *b)
*/
is_2m_pages = b->supported_page_sizes == VMW_BALLOON_NUM_PAGE_SIZES;
- pr_debug("%s - goal: %d", __func__, b->target - b->size);
-
- while (!b->reset_required &&
- b->size + num_pages * vmballoon_page_size(is_2m_pages)
- < b->target) {
+ while ((int64_t)(num_pages * vmballoon_page_size(is_2m_pages)) <
+ vmballoon_change(b)) {
struct page *page;
STATS_INC(b->stats.alloc[is_2m_pages]);
@@ -742,8 +768,6 @@ static void vmballoon_deflate(struct vmballoon *b)
{
unsigned is_2m_pages;
- pr_debug("%s - size: %d, target %d\n", __func__, b->size, b->target);
-
/* free pages to reach target */
for (is_2m_pages = 0; is_2m_pages < b->supported_page_sizes;
is_2m_pages++) {
@@ -753,11 +777,9 @@ static void vmballoon_deflate(struct vmballoon *b)
&b->page_sizes[is_2m_pages];
list_for_each_entry_safe(page, next, &page_size->pages, lru) {
- if (b->reset_required ||
- (b->target > 0 &&
- b->size - num_pages
- * vmballoon_page_size(is_2m_pages)
- < b->target + vmballoon_page_size(true)))
+ if ((int64_t)(num_pages *
+ vmballoon_page_size(is_2m_pages)) >=
+ -vmballoon_change(b))
break;
list_del(&page->lru);
@@ -921,28 +943,35 @@ static void vmballoon_reset(struct vmballoon *b)
pr_err("failed to send guest ID to the host\n");
}
-/*
- * Balloon work function: reset protocol, if needed, get the new size and
- * adjust balloon as needed. Repeat in 1 sec.
+/**
+ * vmballoon_work - periodic balloon worker for reset, inflation and deflation.
+ *
+ * @work: pointer to the &work_struct which is provided by the workqueue.
+ *
+ * Resets the protocol if needed, gets the new size and adjusts balloon as
+ * needed. Repeat in 1 sec.
*/
static void vmballoon_work(struct work_struct *work)
{
struct delayed_work *dwork = to_delayed_work(work);
struct vmballoon *b = container_of(dwork, struct vmballoon, dwork);
+ int64_t change = 0;
STATS_INC(b->stats.timer);
if (b->reset_required)
vmballoon_reset(b);
- if (!b->reset_required && vmballoon_send_get_target(b)) {
- unsigned long target = b->target;
+ if (vmballoon_send_get_target(b))
+ change = vmballoon_change(b);
+
+ if (change != 0) {
+ pr_debug("%s - size: %u, target %u", __func__,
+ b->size, b->target);
- /* update target, adjust size */
- if (b->size < target)
+ if (change > 0)
vmballoon_inflate(b);
- else if (target == 0 ||
- b->size > target + vmballoon_page_size(true))
+ else /* (change < 0) */
vmballoon_deflate(b);
}
--
2.17.1
^ permalink raw reply related [flat|nested] 24+ messages in thread* [PATCH v2 10/20] vmw_balloon: simplify vmballoon_send_get_target()
2018-09-20 17:30 [PATCH v2 00/20] vmw_balloon: compaction, shrinker, 64-bit, etc Nadav Amit
` (8 preceding siblings ...)
2018-09-20 17:30 ` [PATCH v2 09/20] vmw_balloon: refactor change size from vmballoon_work Nadav Amit
@ 2018-09-20 17:30 ` Nadav Amit
2018-09-20 17:30 ` [PATCH v2 11/20] vmw_balloon: stats rework Nadav Amit
` (10 subsequent siblings)
20 siblings, 0 replies; 24+ messages in thread
From: Nadav Amit @ 2018-09-20 17:30 UTC (permalink / raw)
To: Greg Kroah-Hartman, Arnd Bergmann
Cc: linux-kernel, Xavier Deguillard, Nadav Amit
As we want to leave as little as possible on the global balloon
structure, to avoid possible future races, we want to get rid sysinfo.
We can actually get the total_ram directly, and simplify the logic of
vmballoon_send_get_target() a little.
While we are doing that, let's return int and avoid mistakes due to
bool/int conversions.
Reviewed-by: Xavier Deguillard <xdeguillard@vmware.com>
Signed-off-by: Nadav Amit <namit@vmware.com>
---
drivers/misc/vmw_balloon.c | 35 ++++++++++++++---------------------
1 file changed, 14 insertions(+), 21 deletions(-)
diff --git a/drivers/misc/vmw_balloon.c b/drivers/misc/vmw_balloon.c
index 0d9f223d463e..3c80a21e0f91 100644
--- a/drivers/misc/vmw_balloon.c
+++ b/drivers/misc/vmw_balloon.c
@@ -223,8 +223,6 @@ struct vmballoon {
struct dentry *dbg_entry;
#endif
- struct sysinfo sysinfo;
-
struct delayed_work dwork;
struct vmci_handle vmci_doorbell;
@@ -353,34 +351,29 @@ static u16 vmballoon_page_size(bool is_2m_page)
return 1;
}
-/*
- * Retrieve desired balloon size from the host.
+/**
+ * vmballoon_send_get_target() - Retrieve desired balloon size from the host.
+ *
+ * @b: pointer to the balloon.
+ *
+ * Return: zero on success, EINVAL if limit does not fit in 32-bit, as required
+ * by the host-guest protocol and EIO if an error occurred in communicating with
+ * the host.
*/
-static bool vmballoon_send_get_target(struct vmballoon *b)
+static int vmballoon_send_get_target(struct vmballoon *b)
{
unsigned long status;
unsigned long limit;
- u32 limit32;
- /*
- * si_meminfo() is cheap. Moreover, we want to provide dynamic
- * max balloon size later. So let us call si_meminfo() every
- * iteration.
- */
- si_meminfo(&b->sysinfo);
- limit = b->sysinfo.totalram;
+ limit = totalram_pages;
/* Ensure limit fits in 32-bits */
- limit32 = (u32)limit;
- if (limit != limit32)
- return false;
+ if (limit != (u32)limit)
+ return -EINVAL;
status = vmballoon_cmd(b, VMW_BALLOON_CMD_GET_TARGET, limit, 0);
- if (status == VMW_BALLOON_SUCCESS)
- return true;
-
- return false;
+ return status == VMW_BALLOON_SUCCESS ? 0 : -EIO;
}
static struct page *vmballoon_alloc_page(bool is_2m_page)
@@ -962,7 +955,7 @@ static void vmballoon_work(struct work_struct *work)
if (b->reset_required)
vmballoon_reset(b);
- if (vmballoon_send_get_target(b))
+ if (!vmballoon_send_get_target(b))
change = vmballoon_change(b);
if (change != 0) {
--
2.17.1
^ permalink raw reply related [flat|nested] 24+ messages in thread* [PATCH v2 11/20] vmw_balloon: stats rework
2018-09-20 17:30 [PATCH v2 00/20] vmw_balloon: compaction, shrinker, 64-bit, etc Nadav Amit
` (9 preceding siblings ...)
2018-09-20 17:30 ` [PATCH v2 10/20] vmw_balloon: simplify vmballoon_send_get_target() Nadav Amit
@ 2018-09-20 17:30 ` Nadav Amit
2018-09-20 17:30 ` [PATCH v2 12/20] vmw_balloon: rework the inflate and deflate loops Nadav Amit
` (9 subsequent siblings)
20 siblings, 0 replies; 24+ messages in thread
From: Nadav Amit @ 2018-09-20 17:30 UTC (permalink / raw)
To: Greg Kroah-Hartman, Arnd Bergmann
Cc: linux-kernel, Xavier Deguillard, Nadav Amit
To allow the balloon statistics to be updated concurrently, we change
the statistics to be held per core and aggregate it when needed.
To avoid the memory overhead of keeping the statistics per core, and
since it is likely not used by most users, we start updating the
statistics only after the first use. A read-write semaphore is used to
protect the statistics initialization and avoid races. This semaphore is
(and will) be used to protect configuration changes during reset.
While we are at it, address some other issues: change the statistics
update to inline functions instead of define; use ulong for saving the
statistics; and clean the statistics printouts.
Note that this patch changes the format of the outputs. If there are any
automatic tools that use the statistics, they might fail.
Reviewed-by: Xavier Deguillard <xdeguillard@vmware.com>
Signed-off-by: Nadav Amit <namit@vmware.com>
---
drivers/misc/vmw_balloon.c | 384 +++++++++++++++++++++++++++----------
1 file changed, 281 insertions(+), 103 deletions(-)
diff --git a/drivers/misc/vmw_balloon.c b/drivers/misc/vmw_balloon.c
index 3c80a21e0f91..0a2bdaf5773b 100644
--- a/drivers/misc/vmw_balloon.c
+++ b/drivers/misc/vmw_balloon.c
@@ -25,6 +25,8 @@
#include <linux/workqueue.h>
#include <linux/debugfs.h>
#include <linux/seq_file.h>
+#include <linux/rwsem.h>
+#include <linux/slab.h>
#include <linux/vmw_vmci_defs.h>
#include <linux/vmw_vmci_api.h>
#include <asm/hypervisor.h>
@@ -78,46 +80,94 @@ enum vmwballoon_capabilities {
| VMW_BALLOON_SIGNALLED_WAKEUP_CMD)
#define VMW_BALLOON_2M_ORDER (PMD_SHIFT - PAGE_SHIFT)
-#define VMW_BALLOON_NUM_PAGE_SIZES (2)
-/*
- * Backdoor commands availability:
+enum vmballoon_page_size_type {
+ VMW_BALLOON_4K_PAGE,
+ VMW_BALLOON_2M_PAGE,
+ VMW_BALLOON_LAST_SIZE = VMW_BALLOON_2M_PAGE
+};
+
+#define VMW_BALLOON_NUM_PAGE_SIZES (VMW_BALLOON_LAST_SIZE + 1)
+
+enum vmballoon_op_stat_type {
+ VMW_BALLOON_OP_STAT,
+ VMW_BALLOON_OP_FAIL_STAT
+};
+
+#define VMW_BALLOON_OP_STAT_TYPES (VMW_BALLOON_OP_FAIL_STAT + 1)
+
+/**
+ * enum vmballoon_cmd_type - backdoor commands.
+ *
+ * Availability of the commands is as followed:
+ *
+ * %VMW_BALLOON_CMD_START, %VMW_BALLOON_CMD_GET_TARGET and
+ * %VMW_BALLOON_CMD_GUEST_ID are always available.
+ *
+ * If the host reports %VMW_BALLOON_BASIC_CMDS are supported then
+ * %VMW_BALLOON_CMD_LOCK and %VMW_BALLOON_CMD_UNLOCK commands are available.
*
- * START, GET_TARGET and GUEST_ID are always available,
+ * If the host reports %VMW_BALLOON_BATCHED_CMDS are supported then
+ * %VMW_BALLOON_CMD_BATCHED_LOCK and VMW_BALLOON_CMD_BATCHED_UNLOCK commands
+ * are available.
*
- * VMW_BALLOON_BASIC_CMDS:
- * LOCK and UNLOCK commands,
- * VMW_BALLOON_BATCHED_CMDS:
- * BATCHED_LOCK and BATCHED_UNLOCK commands.
- * VMW BALLOON_BATCHED_2M_CMDS:
- * BATCHED_2M_LOCK and BATCHED_2M_UNLOCK commands,
- * VMW VMW_BALLOON_SIGNALLED_WAKEUP_CMD:
- * VMW_BALLOON_CMD_VMCI_DOORBELL_SET command.
+ * If the host reports %VMW_BALLOON_BATCHED_2M_CMDS are supported then
+ * %VMW_BALLOON_CMD_BATCHED_2M_LOCK and %VMW_BALLOON_CMD_BATCHED_2M_UNLOCK
+ * are supported.
+ *
+ * If the host reports VMW_BALLOON_SIGNALLED_WAKEUP_CMD is supported then
+ * VMW_BALLOON_CMD_VMCI_DOORBELL_SET command is supported.
+ *
+ * @VMW_BALLOON_CMD_START: Communicating supported version with the hypervisor.
+ * @VMW_BALLOON_CMD_GET_TARGET: Gets the balloon target size.
+ * @VMW_BALLOON_CMD_LOCK: Informs the hypervisor about a ballooned page.
+ * @VMW_BALLOON_CMD_UNLOCK: Informs the hypervisor about a page that is about
+ * to be deflated from the balloon.
+ * @VMW_BALLOON_CMD_GUEST_ID: Informs the hypervisor about the type of OS that
+ * runs in the VM.
+ * @VMW_BALLOON_CMD_BATCHED_LOCK: Inform the hypervisor about a batch of
+ * ballooned pages (up to 512).
+ * @VMW_BALLOON_CMD_BATCHED_UNLOCK: Inform the hypervisor about a batch of
+ * pages that are about to be deflated from the
+ * balloon (up to 512).
+ * @VMW_BALLOON_CMD_BATCHED_2M_LOCK: Similar to @VMW_BALLOON_CMD_BATCHED_LOCK
+ * for 2MB pages.
+ * @VMW_BALLOON_CMD_BATCHED_2M_UNLOCK: Similar to
+ * @VMW_BALLOON_CMD_BATCHED_UNLOCK for 2MB
+ * pages.
+ * @VMW_BALLOON_CMD_VMCI_DOORBELL_SET: A command to set doorbell notification
+ * that would be invoked when the balloon
+ * size changes.
+ * @VMW_BALLOON_CMD_LAST: Value of the last command.
*/
-#define VMW_BALLOON_CMD_START 0
-#define VMW_BALLOON_CMD_GET_TARGET 1
-#define VMW_BALLOON_CMD_LOCK 2
-#define VMW_BALLOON_CMD_UNLOCK 3
-#define VMW_BALLOON_CMD_GUEST_ID 4
-#define VMW_BALLOON_CMD_BATCHED_LOCK 6
-#define VMW_BALLOON_CMD_BATCHED_UNLOCK 7
-#define VMW_BALLOON_CMD_BATCHED_2M_LOCK 8
-#define VMW_BALLOON_CMD_BATCHED_2M_UNLOCK 9
-#define VMW_BALLOON_CMD_VMCI_DOORBELL_SET 10
-
-#define VMW_BALLOON_CMD_NUM 11
-
-/* error codes */
-#define VMW_BALLOON_SUCCESS 0
-#define VMW_BALLOON_FAILURE -1
-#define VMW_BALLOON_ERROR_CMD_INVALID 1
-#define VMW_BALLOON_ERROR_PPN_INVALID 2
-#define VMW_BALLOON_ERROR_PPN_LOCKED 3
-#define VMW_BALLOON_ERROR_PPN_UNLOCKED 4
-#define VMW_BALLOON_ERROR_PPN_PINNED 5
-#define VMW_BALLOON_ERROR_PPN_NOTNEEDED 6
-#define VMW_BALLOON_ERROR_RESET 7
-#define VMW_BALLOON_ERROR_BUSY 8
+enum vmballoon_cmd_type {
+ VMW_BALLOON_CMD_START,
+ VMW_BALLOON_CMD_GET_TARGET,
+ VMW_BALLOON_CMD_LOCK,
+ VMW_BALLOON_CMD_UNLOCK,
+ VMW_BALLOON_CMD_GUEST_ID,
+ /* No command 5 */
+ VMW_BALLOON_CMD_BATCHED_LOCK = 6,
+ VMW_BALLOON_CMD_BATCHED_UNLOCK,
+ VMW_BALLOON_CMD_BATCHED_2M_LOCK,
+ VMW_BALLOON_CMD_BATCHED_2M_UNLOCK,
+ VMW_BALLOON_CMD_VMCI_DOORBELL_SET,
+ VMW_BALLOON_CMD_LAST = VMW_BALLOON_CMD_VMCI_DOORBELL_SET,
+};
+
+#define VMW_BALLOON_CMD_NUM (VMW_BALLOON_CMD_LAST + 1)
+
+enum vmballoon_error_codes {
+ VMW_BALLOON_SUCCESS,
+ VMW_BALLOON_ERROR_CMD_INVALID,
+ VMW_BALLOON_ERROR_PPN_INVALID,
+ VMW_BALLOON_ERROR_PPN_LOCKED,
+ VMW_BALLOON_ERROR_PPN_UNLOCKED,
+ VMW_BALLOON_ERROR_PPN_PINNED,
+ VMW_BALLOON_ERROR_PPN_NOTNEEDED,
+ VMW_BALLOON_ERROR_RESET,
+ VMW_BALLOON_ERROR_BUSY
+};
#define VMW_BALLOON_SUCCESS_WITH_CAPABILITIES (0x03000000)
@@ -143,29 +193,28 @@ static const char * const vmballoon_cmd_names[] = {
[VMW_BALLOON_CMD_VMCI_DOORBELL_SET] = "doorbellSet"
};
-#ifdef CONFIG_DEBUG_FS
-struct vmballoon_stats {
- unsigned int timer;
- unsigned int doorbell;
-
- /* allocation statistics */
- unsigned int alloc[VMW_BALLOON_NUM_PAGE_SIZES];
- unsigned int alloc_fail[VMW_BALLOON_NUM_PAGE_SIZES];
- unsigned int refused_alloc[VMW_BALLOON_NUM_PAGE_SIZES];
- unsigned int refused_free[VMW_BALLOON_NUM_PAGE_SIZES];
- unsigned int free[VMW_BALLOON_NUM_PAGE_SIZES];
-
- /* Monitor operations. */
- unsigned long ops[VMW_BALLOON_CMD_NUM];
- unsigned long ops_fail[VMW_BALLOON_CMD_NUM];
+enum vmballoon_stat_page {
+ VMW_BALLOON_PAGE_STAT_ALLOC,
+ VMW_BALLOON_PAGE_STAT_ALLOC_FAIL,
+ VMW_BALLOON_PAGE_STAT_REFUSED_ALLOC,
+ VMW_BALLOON_PAGE_STAT_REFUSED_FREE,
+ VMW_BALLOON_PAGE_STAT_FREE,
+ VMW_BALLOON_PAGE_STAT_LAST = VMW_BALLOON_PAGE_STAT_FREE
};
-#define STATS_INC(stat) (stat)++
-#else
-#define STATS_INC(stat)
-#endif
+#define VMW_BALLOON_PAGE_STAT_NUM (VMW_BALLOON_PAGE_STAT_LAST + 1)
+
+enum vmballoon_stat_general {
+ VMW_BALLOON_STAT_TIMER,
+ VMW_BALLOON_STAT_DOORBELL,
+ VMW_BALLOON_STAT_LAST = VMW_BALLOON_STAT_DOORBELL
+};
+
+#define VMW_BALLOON_STAT_NUM (VMW_BALLOON_STAT_LAST + 1)
+
static DEFINE_STATIC_KEY_TRUE(vmw_balloon_batching);
+static DEFINE_STATIC_KEY_FALSE(balloon_stat_enabled);
struct vmballoon_page_size {
/* list of reserved physical pages */
@@ -215,10 +264,10 @@ struct vmballoon {
unsigned int batch_max_pages;
struct page *page;
-#ifdef CONFIG_DEBUG_FS
/* statistics */
- struct vmballoon_stats stats;
+ struct vmballoon_stats *stats;
+#ifdef CONFIG_DEBUG_FS
/* debugfs file exporting statistics */
struct dentry *dbg_entry;
#endif
@@ -226,17 +275,70 @@ struct vmballoon {
struct delayed_work dwork;
struct vmci_handle vmci_doorbell;
+
+ /**
+ * @conf_sem: semaphore to protect the configuration and the statistics.
+ */
+ struct rw_semaphore conf_sem;
};
static struct vmballoon balloon;
+struct vmballoon_stats {
+ /* timer / doorbell operations */
+ atomic64_t general_stat[VMW_BALLOON_STAT_NUM];
+
+ /* allocation statistics for huge and small pages */
+ atomic64_t
+ page_stat[VMW_BALLOON_PAGE_STAT_NUM][VMW_BALLOON_NUM_PAGE_SIZES];
+
+ /* Monitor operations: total operations, and failures */
+ atomic64_t ops[VMW_BALLOON_CMD_NUM][VMW_BALLOON_OP_STAT_TYPES];
+};
+
+static inline bool is_vmballoon_stats_on(void)
+{
+ return IS_ENABLED(CONFIG_DEBUG_FS) &&
+ static_branch_unlikely(&balloon_stat_enabled);
+}
+
+static inline void vmballoon_stats_op_inc(struct vmballoon *b, unsigned int op,
+ enum vmballoon_op_stat_type type)
+{
+ if (is_vmballoon_stats_on())
+ atomic64_inc(&b->stats->ops[op][type]);
+}
+
+static inline void vmballoon_stats_gen_inc(struct vmballoon *b,
+ enum vmballoon_stat_general stat)
+{
+ if (is_vmballoon_stats_on())
+ atomic64_inc(&b->stats->general_stat[stat]);
+}
+
+static inline void vmballoon_stats_gen_add(struct vmballoon *b,
+ enum vmballoon_stat_general stat,
+ unsigned int val)
+{
+ if (is_vmballoon_stats_on())
+ atomic64_add(val, &b->stats->general_stat[stat]);
+}
+
+static inline void vmballoon_stats_page_inc(struct vmballoon *b,
+ enum vmballoon_stat_page stat,
+ bool is_2m_page)
+{
+ if (is_vmballoon_stats_on())
+ atomic64_inc(&b->stats->page_stat[stat][is_2m_page]);
+}
+
static inline unsigned long
__vmballoon_cmd(struct vmballoon *b, unsigned long cmd, unsigned long arg1,
unsigned long arg2, unsigned long *result)
{
unsigned long status, dummy1, dummy2, dummy3, local_result;
- STATS_INC(b->stats.ops[cmd]);
+ vmballoon_stats_op_inc(b, cmd, VMW_BALLOON_OP_STAT);
asm volatile ("inl %%dx" :
"=a"(status),
@@ -263,7 +365,7 @@ __vmballoon_cmd(struct vmballoon *b, unsigned long cmd, unsigned long arg1,
if (status != VMW_BALLOON_SUCCESS &&
status != VMW_BALLOON_SUCCESS_WITH_CAPABILITIES) {
- STATS_INC(b->stats.ops_fail[cmd]);
+ vmballoon_stats_op_inc(b, cmd, VMW_BALLOON_OP_FAIL_STAT);
pr_debug("%s: %s [0x%lx,0x%lx) failed, returned %ld\n",
__func__, vmballoon_cmd_names[cmd], arg1, arg2,
status);
@@ -413,7 +515,8 @@ static void vmballoon_pop(struct vmballoon *b)
list_for_each_entry_safe(page, next, &page_size->pages, lru) {
list_del(&page->lru);
vmballoon_free_page(page, is_2m_pages);
- STATS_INC(b->stats.free[is_2m_pages]);
+ vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_FREE,
+ is_2m_pages);
b->size -= size_per_page;
cond_resched();
}
@@ -534,7 +637,8 @@ static int vmballoon_lock(struct vmballoon *b, unsigned int num_pages,
}
/* Error occurred */
- STATS_INC(b->stats.refused_alloc[is_2m_pages]);
+ vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_REFUSED_ALLOC,
+ is_2m_pages);
/*
* Place page on the list of non-balloonable pages
@@ -587,7 +691,8 @@ static int vmballoon_unlock(struct vmballoon *b, unsigned int num_pages,
} else {
/* deallocate page */
vmballoon_free_page(p, is_2m_pages);
- STATS_INC(b->stats.free[is_2m_pages]);
+ vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_FREE,
+ is_2m_pages);
/* update balloon size */
b->size -= size_per_page;
@@ -611,7 +716,8 @@ static void vmballoon_release_refused_pages(struct vmballoon *b,
list_for_each_entry_safe(page, next, &page_size->refused_pages, lru) {
list_del(&page->lru);
vmballoon_free_page(page, is_2m_pages);
- STATS_INC(b->stats.refused_free[is_2m_pages]);
+ vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_REFUSED_FREE,
+ is_2m_pages);
}
page_size->n_refused_pages = 0;
@@ -693,10 +799,14 @@ static void vmballoon_inflate(struct vmballoon *b)
vmballoon_change(b)) {
struct page *page;
- STATS_INC(b->stats.alloc[is_2m_pages]);
+ vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_ALLOC,
+ is_2m_pages);
+
page = vmballoon_alloc_page(is_2m_pages);
if (!page) {
- STATS_INC(b->stats.alloc_fail[is_2m_pages]);
+ vmballoon_stats_page_inc(b,
+ VMW_BALLOON_PAGE_STAT_ALLOC_FAIL, is_2m_pages);
+
if (is_2m_pages) {
vmballoon_lock(b, num_pages, true);
@@ -845,7 +955,7 @@ static void vmballoon_doorbell(void *client_data)
{
struct vmballoon *b = client_data;
- STATS_INC(b->stats.doorbell);
+ vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_DOORBELL);
mod_delayed_work(system_freezable_wq, &b->dwork, 0);
}
@@ -903,6 +1013,8 @@ static void vmballoon_reset(struct vmballoon *b)
{
int error;
+ down_write(&b->conf_sem);
+
vmballoon_vmci_cleanup(b);
/* free all pages, skipping monitor unlock */
@@ -934,6 +1046,8 @@ static void vmballoon_reset(struct vmballoon *b)
if (!vmballoon_send_guest_id(b))
pr_err("failed to send guest ID to the host\n");
+
+ up_write(&b->conf_sem);
}
/**
@@ -950,11 +1064,18 @@ static void vmballoon_work(struct work_struct *work)
struct vmballoon *b = container_of(dwork, struct vmballoon, dwork);
int64_t change = 0;
- STATS_INC(b->stats.timer);
-
if (b->reset_required)
vmballoon_reset(b);
+ down_read(&b->conf_sem);
+
+ /*
+ * Update the stats while holding the semaphore to ensure that
+ * @stats_enabled is consistent with whether the stats are actually
+ * enabled
+ */
+ vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_TIMER);
+
if (!vmballoon_send_get_target(b))
change = vmballoon_change(b);
@@ -968,12 +1089,15 @@ static void vmballoon_work(struct work_struct *work)
vmballoon_deflate(b);
}
+ up_read(&b->conf_sem);
+
/*
* We are using a freezable workqueue so that balloon operations are
* stopped while the system transitions to/from sleep/hibernation.
*/
queue_delayed_work(system_freezable_wq,
dwork, round_jiffies_relative(HZ));
+
}
/*
@@ -981,55 +1105,105 @@ static void vmballoon_work(struct work_struct *work)
*/
#ifdef CONFIG_DEBUG_FS
+static const char * const vmballoon_stat_page_names[] = {
+ [VMW_BALLOON_PAGE_STAT_ALLOC] = "alloc",
+ [VMW_BALLOON_PAGE_STAT_ALLOC_FAIL] = "allocFail",
+ [VMW_BALLOON_PAGE_STAT_REFUSED_ALLOC] = "errAlloc",
+ [VMW_BALLOON_PAGE_STAT_REFUSED_FREE] = "errFree",
+ [VMW_BALLOON_PAGE_STAT_FREE] = "free"
+};
+
+static const char * const vmballoon_stat_names[] = {
+ [VMW_BALLOON_STAT_TIMER] = "timer",
+ [VMW_BALLOON_STAT_DOORBELL] = "doorbell"
+};
+
+static const char * const vmballoon_page_size_names[] = {
+ [VMW_BALLOON_4K_PAGE] = "4k",
+ [VMW_BALLOON_2M_PAGE] = "2M"
+};
+
+static int vmballoon_enable_stats(struct vmballoon *b)
+{
+ int r = 0;
+
+ down_write(&b->conf_sem);
+
+ /* did we somehow race with another reader which enabled stats? */
+ if (b->stats)
+ goto out;
+
+ b->stats = kzalloc(sizeof(*b->stats), GFP_KERNEL);
+
+ if (!b->stats) {
+ /* allocation failed */
+ r = -ENOMEM;
+ goto out;
+ }
+ static_key_enable(&balloon_stat_enabled.key);
+out:
+ up_write(&b->conf_sem);
+ return r;
+}
+
+/**
+ * vmballoon_debug_show - shows statistics of balloon operations.
+ * @f: pointer to the &struct seq_file.
+ * @offset: ignored.
+ *
+ * Provides the statistics that can be accessed in vmmemctl in the debugfs.
+ * To avoid the overhead - mainly that of memory - of collecting the statistics,
+ * we only collect statistics after the first time the counters are read.
+ *
+ * Return: zero on success or an error code.
+ */
static int vmballoon_debug_show(struct seq_file *f, void *offset)
{
struct vmballoon *b = f->private;
- struct vmballoon_stats *stats = &b->stats;
- int i;
+ int i, j;
+
+ /* enables stats if they are disabled */
+ if (!b->stats) {
+ int r = vmballoon_enable_stats(b);
+
+ if (r)
+ return r;
+ }
/* format capabilities info */
- seq_printf(f,
- "balloon capabilities: %#4x\n"
- "used capabilities: %#4lx\n"
- "is resetting: %c\n",
- VMW_BALLOON_CAPABILITIES, b->capabilities,
- b->reset_required ? 'y' : 'n');
+ seq_printf(f, "%-22s: %#4x\n", "balloon capabilities",
+ VMW_BALLOON_CAPABILITIES);
+ seq_printf(f, "%-22s: %#4lx\n", "used capabilities",
+ b->capabilities);
+ seq_printf(f, "%-22s: %16s\n", "is resetting",
+ b->reset_required ? "y" : "n");
/* format size info */
- seq_printf(f,
- "target: %8d pages\n"
- "current: %8d pages\n",
- b->target, b->size);
+ seq_printf(f, "%-22s: %16u\n", "target", b->target);
+ seq_printf(f, "%-22s: %16u\n", "current", b->size);
for (i = 0; i < VMW_BALLOON_CMD_NUM; i++) {
if (vmballoon_cmd_names[i] == NULL)
continue;
- seq_printf(f, "%-22s: %16lu (%lu failed)\n",
- vmballoon_cmd_names[i], stats->ops[i],
- stats->ops_fail[i]);
+ seq_printf(f, "%-22s: %16llu (%llu failed)\n",
+ vmballoon_cmd_names[i],
+ atomic64_read(&b->stats->ops[i][VMW_BALLOON_OP_STAT]),
+ atomic64_read(&b->stats->ops[i][VMW_BALLOON_OP_FAIL_STAT]));
}
- seq_printf(f,
- "\n"
- "timer: %8u\n"
- "doorbell: %8u\n"
- "prim2mAlloc: %8u (%4u failed)\n"
- "prim4kAlloc: %8u (%4u failed)\n"
- "prim2mFree: %8u\n"
- "primFree: %8u\n"
- "err2mAlloc: %8u\n"
- "errAlloc: %8u\n"
- "err2mFree: %8u\n"
- "errFree: %8u\n",
- stats->timer,
- stats->doorbell,
- stats->alloc[true], stats->alloc_fail[true],
- stats->alloc[false], stats->alloc_fail[false],
- stats->free[true],
- stats->free[false],
- stats->refused_alloc[true], stats->refused_alloc[false],
- stats->refused_free[true], stats->refused_free[false]);
+ for (i = 0; i < VMW_BALLOON_STAT_NUM; i++)
+ seq_printf(f, "%-22s: %16llu\n",
+ vmballoon_stat_names[i],
+ atomic64_read(&b->stats->general_stat[i]));
+
+ for (i = 0; i < VMW_BALLOON_PAGE_STAT_NUM; i++) {
+ for (j = 0; j < VMW_BALLOON_NUM_PAGE_SIZES; j++)
+ seq_printf(f, "%-18s(%s): %16llu\n",
+ vmballoon_stat_page_names[i],
+ vmballoon_page_size_names[j],
+ atomic64_read(&b->stats->page_stat[i][j]));
+ }
return 0;
}
@@ -1064,7 +1238,10 @@ static int __init vmballoon_debugfs_init(struct vmballoon *b)
static void __exit vmballoon_debugfs_exit(struct vmballoon *b)
{
+ static_key_disable(&balloon_stat_enabled.key);
debugfs_remove(b->dbg_entry);
+ kfree(b->stats);
+ b->stats = NULL;
}
#else
@@ -1103,6 +1280,7 @@ static int __init vmballoon_init(void)
if (error)
return error;
+ init_rwsem(&balloon.conf_sem);
balloon.vmci_doorbell = VMCI_INVALID_HANDLE;
balloon.batch_page = NULL;
balloon.page = NULL;
--
2.17.1
^ permalink raw reply related [flat|nested] 24+ messages in thread* [PATCH v2 12/20] vmw_balloon: rework the inflate and deflate loops
2018-09-20 17:30 [PATCH v2 00/20] vmw_balloon: compaction, shrinker, 64-bit, etc Nadav Amit
` (10 preceding siblings ...)
2018-09-20 17:30 ` [PATCH v2 11/20] vmw_balloon: stats rework Nadav Amit
@ 2018-09-20 17:30 ` Nadav Amit
2018-09-20 17:30 ` [PATCH v2 13/20] vmw_balloon: general style cleanup Nadav Amit
` (8 subsequent siblings)
20 siblings, 0 replies; 24+ messages in thread
From: Nadav Amit @ 2018-09-20 17:30 UTC (permalink / raw)
To: Greg Kroah-Hartman, Arnd Bergmann
Cc: linux-kernel, Xavier Deguillard, Nadav Amit
In preparation for supporting compaction and OOM notification, this
patch reworks the inflate/deflate loops. The main idea is to separate
the allocation, communication with the hypervisor, and the handling of
errors from each other. Doing will allow us to perform concurrent
inflation and deflation, excluding the actual communication with the
hypervisor.
To do so, we need to get rid of the remaining global state that is kept
in the balloon struct, specifically the refuse_list. When the VM
communicates with the hypervisor, it does not free or put back pages
to the balloon list and instead only moves the pages whose status
indicated failure into a refuse_list on the stack. Once the operation
completes, the inflation or deflation functions handle the list
appropriately.
As we do that, we can consolidate the communication with the hypervisor
for both the lock and unlock operations into a single function. We also
reuse the deflation function for popping the balloon.
As a preparation for preventing races, we hold a spinlock when the
communication actually takes place, and use atomic operations for
updating the balloon size. The balloon page list is still racy and will
be handled in the next patch.
Reviewed-by: Xavier Deguillard <xdeguillard@vmware.com>
Signed-off-by: Nadav Amit <namit@vmware.com>
---
drivers/misc/vmw_balloon.c | 829 ++++++++++++++++++++++++-------------
1 file changed, 541 insertions(+), 288 deletions(-)
diff --git a/drivers/misc/vmw_balloon.c b/drivers/misc/vmw_balloon.c
index 0a2bdaf5773b..cb363db34b4a 100644
--- a/drivers/misc/vmw_balloon.c
+++ b/drivers/misc/vmw_balloon.c
@@ -27,6 +27,7 @@
#include <linux/seq_file.h>
#include <linux/rwsem.h>
#include <linux/slab.h>
+#include <linux/spinlock.h>
#include <linux/vmw_vmci_defs.h>
#include <linux/vmw_vmci_api.h>
#include <asm/hypervisor.h>
@@ -89,6 +90,16 @@ enum vmballoon_page_size_type {
#define VMW_BALLOON_NUM_PAGE_SIZES (VMW_BALLOON_LAST_SIZE + 1)
+static const char * const vmballoon_page_size_names[] = {
+ [VMW_BALLOON_4K_PAGE] = "4k",
+ [VMW_BALLOON_2M_PAGE] = "2M"
+};
+
+enum vmballoon_op {
+ VMW_BALLOON_INFLATE,
+ VMW_BALLOON_DEFLATE
+};
+
enum vmballoon_op_stat_type {
VMW_BALLOON_OP_STAT,
VMW_BALLOON_OP_FAIL_STAT
@@ -216,13 +227,18 @@ enum vmballoon_stat_general {
static DEFINE_STATIC_KEY_TRUE(vmw_balloon_batching);
static DEFINE_STATIC_KEY_FALSE(balloon_stat_enabled);
-struct vmballoon_page_size {
- /* list of reserved physical pages */
+struct vmballoon_ctl {
struct list_head pages;
-
- /* transient list of non-balloonable pages */
struct list_head refused_pages;
unsigned int n_refused_pages;
+ unsigned int n_pages;
+ enum vmballoon_page_size_type page_size;
+ enum vmballoon_op op;
+};
+
+struct vmballoon_page_size {
+ /* list of reserved physical pages */
+ struct list_head pages;
};
/**
@@ -241,16 +257,47 @@ struct vmballoon_batch_entry {
struct vmballoon {
struct vmballoon_page_size page_sizes[VMW_BALLOON_NUM_PAGE_SIZES];
- /* supported page sizes. 1 == 4k pages only, 2 == 4k and 2m pages */
- unsigned supported_page_sizes;
+ /**
+ * @max_page_size: maximum supported page size for ballooning.
+ *
+ * Protected by @conf_sem
+ */
+ enum vmballoon_page_size_type max_page_size;
+
+ /**
+ * @size: balloon actual size in basic page size (frames).
+ *
+ * While we currently do not support size which is bigger than 32-bit,
+ * in preparation for future support, use 64-bits.
+ */
+ atomic64_t size;
- /* balloon size in pages */
- unsigned int size;
- unsigned int target;
+ /**
+ * @target: balloon target size in basic page size (frames).
+ *
+ * We do not protect the target under the assumption that setting the
+ * value is always done through a single write. If this assumption ever
+ * breaks, we would have to use X_ONCE for accesses, and suffer the less
+ * optimized code. Although we may read stale target value if multiple
+ * accesses happen at once, the performance impact should be minor.
+ */
+ unsigned long target;
- /* reset flag */
+ /**
+ * @reset_required: reset flag
+ *
+ * Setting this flag may introduce races, but the code is expected to
+ * handle them gracefully. In the worst case, another operation will
+ * fail as reset did not take place. Clearing the flag is done while
+ * holding @conf_sem for write.
+ */
bool reset_required;
+ /**
+ * @capabilities: hypervisor balloon capabilities.
+ *
+ * Protected by @conf_sem.
+ */
unsigned long capabilities;
/**
@@ -261,7 +308,25 @@ struct vmballoon {
*/
struct vmballoon_batch_entry *batch_page;
+ /**
+ * @batch_max_pages: maximum pages that can be locked/unlocked.
+ *
+ * Indicates the number of pages that the hypervisor can lock or unlock
+ * at once, according to whether batching is enabled. If batching is
+ * disabled, only a single page can be locked/unlock on each operation.
+ *
+ * Protected by @conf_sem.
+ */
unsigned int batch_max_pages;
+
+ /**
+ * @page: page to be locked/unlocked by the hypervisor
+ *
+ * @page is only used when batching is disabled and a single page is
+ * reclaimed on each iteration.
+ *
+ * Protected by @comm_lock.
+ */
struct page *page;
/* statistics */
@@ -274,12 +339,24 @@ struct vmballoon {
struct delayed_work dwork;
+ /**
+ * @vmci_doorbell.
+ *
+ * Protected by @conf_sem.
+ */
struct vmci_handle vmci_doorbell;
/**
* @conf_sem: semaphore to protect the configuration and the statistics.
*/
struct rw_semaphore conf_sem;
+
+ /**
+ * @comm_lock: lock to protect the communication with the host.
+ *
+ * Lock ordering: @conf_sem -> @comm_lock .
+ */
+ spinlock_t comm_lock;
};
static struct vmballoon balloon;
@@ -326,10 +403,19 @@ static inline void vmballoon_stats_gen_add(struct vmballoon *b,
static inline void vmballoon_stats_page_inc(struct vmballoon *b,
enum vmballoon_stat_page stat,
- bool is_2m_page)
+ enum vmballoon_page_size_type size)
{
if (is_vmballoon_stats_on())
- atomic64_inc(&b->stats->page_stat[stat][is_2m_page]);
+ atomic64_inc(&b->stats->page_stat[stat][size]);
+}
+
+static inline void vmballoon_stats_page_add(struct vmballoon *b,
+ enum vmballoon_stat_page stat,
+ enum vmballoon_page_size_type size,
+ unsigned int val)
+{
+ if (is_vmballoon_stats_on())
+ atomic64_add(val, &b->stats->page_stat[stat][size]);
}
static inline unsigned long
@@ -361,7 +447,7 @@ __vmballoon_cmd(struct vmballoon *b, unsigned long cmd, unsigned long arg1,
/* update target when applicable */
if (status == VMW_BALLOON_SUCCESS &&
((1ul << cmd) & VMW_BALLOON_CMD_WITH_TARGET_MASK))
- b->target = local_result;
+ WRITE_ONCE(b->target, local_result);
if (status != VMW_BALLOON_SUCCESS &&
status != VMW_BALLOON_SUCCESS_WITH_CAPABILITIES) {
@@ -417,11 +503,11 @@ static bool vmballoon_send_start(struct vmballoon *b, unsigned long req_caps)
* reason disabled, do not use 2MB pages, since otherwise the legacy
* mechanism is used with 2MB pages, causing a failure.
*/
+ b->max_page_size = VMW_BALLOON_4K_PAGE;
if ((b->capabilities & VMW_BALLOON_BATCHED_2M_CMDS) &&
(b->capabilities & VMW_BALLOON_BATCHED_CMDS))
- b->supported_page_sizes = 2;
- else
- b->supported_page_sizes = 1;
+ b->max_page_size = VMW_BALLOON_2M_PAGE;
+
return success;
}
@@ -445,12 +531,28 @@ static bool vmballoon_send_guest_id(struct vmballoon *b)
return false;
}
-static u16 vmballoon_page_size(bool is_2m_page)
+/**
+ * vmballoon_page_order() - return the order of the page
+ * @page_size: the size of the page.
+ *
+ * Return: the allocation order.
+ */
+static inline
+unsigned int vmballoon_page_order(enum vmballoon_page_size_type page_size)
{
- if (is_2m_page)
- return 1 << VMW_BALLOON_2M_ORDER;
+ return page_size == VMW_BALLOON_2M_PAGE ? VMW_BALLOON_2M_ORDER : 0;
+}
- return 1;
+/**
+ * vmballoon_page_in_frames() - returns the number of frames in a page.
+ * @page_size: the size of the page.
+ *
+ * Return: the number of 4k frames.
+ */
+static inline unsigned int
+vmballoon_page_in_frames(enum vmballoon_page_size_type page_size)
+{
+ return 1 << vmballoon_page_order(page_size);
}
/**
@@ -478,53 +580,78 @@ static int vmballoon_send_get_target(struct vmballoon *b)
return status == VMW_BALLOON_SUCCESS ? 0 : -EIO;
}
-static struct page *vmballoon_alloc_page(bool is_2m_page)
+/**
+ * vmballoon_alloc_page_list - allocates a list of pages.
+ *
+ * @b: pointer to the balloon.
+ * @ctl: pointer for the %struct vmballoon_ctl, which defines the operation.
+ * @req_n_pages: the number of requested pages.
+ *
+ * Tries to allocate @req_n_pages. Add them to the list of balloon pages in
+ * @ctl.pages and updates @ctl.n_pages to reflect the number of pages.
+ *
+ * Return: zero on success or error code otherwise.
+ */
+static int vmballoon_alloc_page_list(struct vmballoon *b,
+ struct vmballoon_ctl *ctl,
+ unsigned int req_n_pages)
{
- if (is_2m_page)
- return alloc_pages(VMW_HUGE_PAGE_ALLOC_FLAGS,
- VMW_BALLOON_2M_ORDER);
+ struct page *page;
+ unsigned int i;
- return alloc_page(VMW_PAGE_ALLOC_FLAGS);
-}
+ for (i = 0; i < req_n_pages; i++) {
+ if (ctl->page_size == VMW_BALLOON_2M_PAGE)
+ page = alloc_pages(VMW_HUGE_PAGE_ALLOC_FLAGS,
+ VMW_BALLOON_2M_ORDER);
+ else
+ page = alloc_page(VMW_PAGE_ALLOC_FLAGS);
-static void vmballoon_free_page(struct page *page, bool is_2m_page)
-{
- if (is_2m_page)
- __free_pages(page, VMW_BALLOON_2M_ORDER);
- else
- __free_page(page);
+ /* Update statistics */
+ vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_ALLOC,
+ ctl->page_size);
+
+ if (page) {
+ /* Success. Add the page to the list and continue. */
+ list_add(&page->lru, &ctl->pages);
+ continue;
+ }
+
+ /* Allocation failed. Update statistics and stop. */
+ vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_ALLOC_FAIL,
+ ctl->page_size);
+ break;
+ }
+
+ ctl->n_pages = i;
+
+ return req_n_pages == ctl->n_pages ? 0 : -ENOMEM;
}
-/*
- * Quickly release all pages allocated for the balloon. This function is
- * called when host decides to "reset" balloon for one reason or another.
- * Unlike normal "deflate" we do not (shall not) notify host of the pages
- * being released.
+/**
+ * vmballoon_handle_one_result - Handle lock/unlock result for a single page.
+ *
+ * @b: pointer for %struct vmballoon.
+ * @page: pointer for the page whose result should be handled.
+ * @page_size: size of the page.
+ * @status: status of the operation as provided by the hypervisor.
*/
-static void vmballoon_pop(struct vmballoon *b)
+static int vmballoon_handle_one_result(struct vmballoon *b, struct page *page,
+ enum vmballoon_page_size_type page_size,
+ unsigned long status)
{
- struct page *page, *next;
- unsigned is_2m_pages;
-
- for (is_2m_pages = 0; is_2m_pages < VMW_BALLOON_NUM_PAGE_SIZES;
- is_2m_pages++) {
- struct vmballoon_page_size *page_size =
- &b->page_sizes[is_2m_pages];
- u16 size_per_page = vmballoon_page_size(is_2m_pages);
-
- list_for_each_entry_safe(page, next, &page_size->pages, lru) {
- list_del(&page->lru);
- vmballoon_free_page(page, is_2m_pages);
- vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_FREE,
- is_2m_pages);
- b->size -= size_per_page;
- cond_resched();
- }
- }
+ /* On success do nothing. The page is already on the balloon list. */
+ if (likely(status == VMW_BALLOON_SUCCESS))
+ return 0;
- /* Clearing the batch_page unconditionally has no adverse effect */
- free_page((unsigned long)b->batch_page);
- b->batch_page = NULL;
+ pr_debug("%s: failed comm pfn %lx status %lu page_size %s\n", __func__,
+ page_to_pfn(page), status,
+ vmballoon_page_size_names[page_size]);
+
+ /* Error occurred */
+ vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_REFUSED_ALLOC,
+ page_size);
+
+ return -EIO;
}
/**
@@ -565,8 +692,8 @@ static unsigned long vmballoon_status_page(struct vmballoon *b, int idx,
* vmballoon_lock_op - notifies the host about inflated/deflated pages.
* @b: pointer to the balloon.
* @num_pages: number of inflated/deflated pages.
- * @is_2m_pages: whether the page(s) are 2M (or 4k).
- * @lock: whether the operation is lock (or unlock).
+ * @page_size: size of the page.
+ * @op: the type of operation (lock or unlock).
*
* Notify the host about page(s) that were ballooned (or removed from the
* balloon) so that host can use it without fear that guest will need it (or
@@ -578,21 +705,27 @@ static unsigned long vmballoon_status_page(struct vmballoon *b, int idx,
*/
static unsigned long vmballoon_lock_op(struct vmballoon *b,
unsigned int num_pages,
- bool is_2m_pages, bool lock)
+ enum vmballoon_page_size_type page_size,
+ enum vmballoon_op op)
{
unsigned long cmd, pfn;
+ lockdep_assert_held(&b->comm_lock);
+
if (static_branch_likely(&vmw_balloon_batching)) {
- if (lock)
- cmd = is_2m_pages ? VMW_BALLOON_CMD_BATCHED_2M_LOCK :
- VMW_BALLOON_CMD_BATCHED_LOCK;
+ if (op == VMW_BALLOON_INFLATE)
+ cmd = page_size == VMW_BALLOON_2M_PAGE ?
+ VMW_BALLOON_CMD_BATCHED_2M_LOCK :
+ VMW_BALLOON_CMD_BATCHED_LOCK;
else
- cmd = is_2m_pages ? VMW_BALLOON_CMD_BATCHED_2M_UNLOCK :
- VMW_BALLOON_CMD_BATCHED_UNLOCK;
+ cmd = page_size == VMW_BALLOON_2M_PAGE ?
+ VMW_BALLOON_CMD_BATCHED_2M_UNLOCK :
+ VMW_BALLOON_CMD_BATCHED_UNLOCK;
pfn = PHYS_PFN(virt_to_phys(b->batch_page));
} else {
- cmd = lock ? VMW_BALLOON_CMD_LOCK : VMW_BALLOON_CMD_UNLOCK;
+ cmd = op == VMW_BALLOON_INFLATE ? VMW_BALLOON_CMD_LOCK :
+ VMW_BALLOON_CMD_UNLOCK;
pfn = page_to_pfn(b->page);
/* In non-batching mode, PFNs must fit in 32-bit */
@@ -603,22 +736,75 @@ static unsigned long vmballoon_lock_op(struct vmballoon *b,
return vmballoon_cmd(b, cmd, pfn, num_pages);
}
-static int vmballoon_lock(struct vmballoon *b, unsigned int num_pages,
- bool is_2m_pages)
+/**
+ * vmballoon_add_page - adds a page towards lock/unlock operation.
+ *
+ * @b: pointer to the balloon.
+ * @idx: index of the page to be ballooned in this batch.
+ * @p: pointer to the page that is about to be ballooned.
+ *
+ * Adds the page to be ballooned. Must be called while holding @comm_lock.
+ */
+static void vmballoon_add_page(struct vmballoon *b, unsigned int idx,
+ struct page *p)
+{
+ lockdep_assert_held(&b->comm_lock);
+
+ if (static_branch_likely(&vmw_balloon_batching))
+ b->batch_page[idx] = (struct vmballoon_batch_entry)
+ { .pfn = page_to_pfn(p) };
+ else
+ b->page = p;
+}
+
+/**
+ * vmballoon_lock - lock or unlock a batch of pages.
+ *
+ * @b: pointer to the balloon.
+ * @ctl: pointer for the %struct vmballoon_ctl, which defines the operation.
+ *
+ * Notifies the host of about ballooned pages (after inflation or deflation,
+ * according to @ctl). If the host rejects the page put it on the
+ * @ctl refuse list. These refused page are then released when moving to the
+ * next size of pages.
+ *
+ * Note that we neither free any @page here nor put them back on the ballooned
+ * pages list. Instead we queue it for later processing. We do that for several
+ * reasons. First, we do not want to free the page under the lock. Second, it
+ * allows us to unify the handling of lock and unlock. In the inflate case, the
+ * caller will check if there are too many refused pages and release them.
+ * Although it is not identical to the past behavior, it should not affect
+ * performance.
+ */
+static int vmballoon_lock(struct vmballoon *b, struct vmballoon_ctl *ctl)
{
unsigned long batch_status;
- int i;
- u16 size_per_page = vmballoon_page_size(is_2m_pages);
+ struct page *page;
+ unsigned int i, num_pages;
+
+ num_pages = ctl->n_pages;
+ if (num_pages == 0)
+ return 0;
- batch_status = vmballoon_lock_op(b, num_pages, is_2m_pages, true);
+ /* communication with the host is done under the communication lock */
+ spin_lock(&b->comm_lock);
+ i = 0;
+ list_for_each_entry(page, &ctl->pages, lru)
+ vmballoon_add_page(b, i++, page);
+
+ batch_status = vmballoon_lock_op(b, ctl->n_pages, ctl->page_size,
+ ctl->op);
+
+ /*
+ * Iterate over the pages in the provided list. Since we are changing
+ * @ctl->n_pages we are saving the original value in @num_pages and
+ * use this value to bound the loop.
+ */
for (i = 0; i < num_pages; i++) {
unsigned long status;
- struct page *p;
- struct vmballoon_page_size *page_size =
- &b->page_sizes[is_2m_pages];
- status = vmballoon_status_page(b, i, &p);
+ status = vmballoon_status_page(b, i, &page);
/*
* Failure of the whole batch overrides a single operation
@@ -627,109 +813,61 @@ static int vmballoon_lock(struct vmballoon *b, unsigned int num_pages,
if (batch_status != VMW_BALLOON_SUCCESS)
status = batch_status;
- if (status == VMW_BALLOON_SUCCESS) {
- /* track allocated page */
- list_add(&p->lru, &page_size->pages);
-
- /* update balloon size */
- b->size += size_per_page;
+ /* Continue if no error happened */
+ if (!vmballoon_handle_one_result(b, page, ctl->page_size,
+ status))
continue;
- }
-
- /* Error occurred */
- vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_REFUSED_ALLOC,
- is_2m_pages);
/*
- * Place page on the list of non-balloonable pages
- * and retry allocation, unless we already accumulated
- * too many of them, in which case take a breather.
+ * Error happened. Move the pages to the refused list and update
+ * the pages number.
*/
- list_add(&p->lru, &page_size->refused_pages);
- page_size->n_refused_pages++;
+ list_move(&page->lru, &ctl->refused_pages);
+ ctl->n_pages--;
+ ctl->n_refused_pages++;
}
+ spin_unlock(&b->comm_lock);
+
return batch_status == VMW_BALLOON_SUCCESS ? 0 : -EIO;
}
-/*
- * Release the page allocated for the balloon. Note that we first notify
- * the host so it can make sure the page will be available for the guest
- * to use, if needed.
+/**
+ * vmballoon_release_page_list() - Releases a page list
+ *
+ * @page_list: list of pages to release.
+ * @n_pages: pointer to the number of pages.
+ * @page_size: whether the pages in the list are 2MB (or else 4KB).
+ *
+ * Releases the list of pages and zeros the number of pages.
*/
-static int vmballoon_unlock(struct vmballoon *b, unsigned int num_pages,
- bool is_2m_pages)
+static void vmballoon_release_page_list(struct list_head *page_list,
+ int *n_pages,
+ enum vmballoon_page_size_type page_size)
{
- int i;
- unsigned long batch_status;
- u16 size_per_page = vmballoon_page_size(is_2m_pages);
-
- batch_status = vmballoon_lock_op(b, num_pages, is_2m_pages, false);
-
- for (i = 0; i < num_pages; i++) {
- struct vmballoon_page_size *page_size;
- unsigned long status;
- struct page *p;
+ struct page *page, *tmp;
- status = vmballoon_status_page(b, i, &p);
- page_size = &b->page_sizes[is_2m_pages];
-
- /*
- * Failure of the whole batch overrides a single operation
- * results.
- */
- if (batch_status != VMW_BALLOON_SUCCESS)
- status = batch_status;
-
- if (status != VMW_BALLOON_SUCCESS) {
- /*
- * That page wasn't successfully unlocked by the
- * hypervisor, re-add it to the list of pages owned by
- * the balloon driver.
- */
- list_add(&p->lru, &page_size->pages);
- } else {
- /* deallocate page */
- vmballoon_free_page(p, is_2m_pages);
- vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_FREE,
- is_2m_pages);
-
- /* update balloon size */
- b->size -= size_per_page;
- }
+ list_for_each_entry_safe(page, tmp, page_list, lru) {
+ list_del(&page->lru);
+ __free_pages(page, vmballoon_page_order(page_size));
}
- return batch_status == VMW_BALLOON_SUCCESS ? 0 : -EIO;
+ *n_pages = 0;
}
+
/*
* Release pages that were allocated while attempting to inflate the
* balloon but were refused by the host for one reason or another.
*/
static void vmballoon_release_refused_pages(struct vmballoon *b,
- bool is_2m_pages)
+ struct vmballoon_ctl *ctl)
{
- struct page *page, *next;
- struct vmballoon_page_size *page_size =
- &b->page_sizes[is_2m_pages];
+ vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_REFUSED_FREE,
+ ctl->page_size);
- list_for_each_entry_safe(page, next, &page_size->refused_pages, lru) {
- list_del(&page->lru);
- vmballoon_free_page(page, is_2m_pages);
- vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_REFUSED_FREE,
- is_2m_pages);
- }
-
- page_size->n_refused_pages = 0;
-}
-
-static void vmballoon_add_page(struct vmballoon *b, int idx, struct page *p)
-{
- if (static_branch_likely(&vmw_balloon_batching))
- b->batch_page[idx] = (struct vmballoon_batch_entry)
- { .pfn = page_to_pfn(p) };
- else
- b->page = p;
+ vmballoon_release_page_list(&ctl->refused_pages, &ctl->n_refused_pages,
+ ctl->page_size);
}
/**
@@ -744,8 +882,8 @@ static int64_t vmballoon_change(struct vmballoon *b)
{
int64_t size, target;
- size = b->size;
- target = b->target;
+ size = atomic64_read(&b->size);
+ target = READ_ONCE(b->target);
/*
* We must cast first because of int sizes
@@ -756,154 +894,257 @@ static int64_t vmballoon_change(struct vmballoon *b)
return 0;
/* consider a 2MB slack on deflate, unless the balloon is emptied */
- if (target < size && size - target < vmballoon_page_size(true) &&
- target != 0)
+ if (target < size && target != 0 &&
+ size - target < vmballoon_page_in_frames(VMW_BALLOON_2M_PAGE))
return 0;
return target - size;
}
-/*
- * Inflate the balloon towards its target size. Note that we try to limit
- * the rate of allocation to make sure we are not choking the rest of the
- * system.
+/**
+ * vmballoon_enqueue_page_list() - Enqueues list of pages after inflation.
+ *
+ * @b: pointer to balloon.
+ * @pages: list of pages to enqueue.
+ * @n_pages: pointer to number of pages in list. The value is zeroed.
+ * @page_size: whether the pages are 2MB or 4KB pages.
+ *
+ * Enqueues the provides list of pages in the ballooned page list, clears the
+ * list and zeroes the number of pages that was provided.
+ */
+static void vmballoon_enqueue_page_list(struct vmballoon *b,
+ struct list_head *pages,
+ unsigned int *n_pages,
+ enum vmballoon_page_size_type page_size)
+{
+ struct vmballoon_page_size *page_size_info = &b->page_sizes[page_size];
+
+ list_splice_init(pages, &page_size_info->pages);
+ *n_pages = 0;
+}
+
+/**
+ * vmballoon_dequeue_page_list() - Dequeues page lists for deflation.
+ *
+ * @b: pointer to balloon.
+ * @pages: list of pages to enqueue.
+ * @n_pages: pointer to number of pages in list. The value is zeroed.
+ * @page_size: whether the pages are 2MB or 4KB pages.
+ * @n_req_pages: the number of requested pages.
+ *
+ * Dequeues the number of requested pages from the balloon for deflation. The
+ * number of dequeued pages may be lower, if not enough pages in the requested
+ * size are available.
+ */
+static void vmballoon_dequeue_page_list(struct vmballoon *b,
+ struct list_head *pages,
+ unsigned int *n_pages,
+ enum vmballoon_page_size_type page_size,
+ unsigned int n_req_pages)
+{
+ struct vmballoon_page_size *page_size_info = &b->page_sizes[page_size];
+ struct page *page, *tmp;
+ unsigned int i = 0;
+
+ list_for_each_entry_safe(page, tmp, &page_size_info->pages, lru) {
+ list_move(&page->lru, pages);
+ if (++i == n_req_pages)
+ break;
+ }
+ *n_pages = i;
+}
+
+/**
+ * vmballoon_inflate() - Inflate the balloon towards its target size.
+ *
+ * @b: pointer to the balloon.
*/
static void vmballoon_inflate(struct vmballoon *b)
{
- unsigned int num_pages = 0;
- int error = 0;
- bool is_2m_pages;
+ int64_t to_inflate_frames;
+ struct vmballoon_ctl ctl = {
+ .pages = LIST_HEAD_INIT(ctl.pages),
+ .refused_pages = LIST_HEAD_INIT(ctl.refused_pages),
+ .page_size = b->max_page_size,
+ .op = VMW_BALLOON_INFLATE
+ };
- /*
- * First try NOSLEEP page allocations to inflate balloon.
- *
- * If we do not throttle nosleep allocations, we can drain all
- * free pages in the guest quickly (if the balloon target is high).
- * As a side-effect, draining free pages helps to inform (force)
- * the guest to start swapping if balloon target is not met yet,
- * which is a desired behavior. However, balloon driver can consume
- * all available CPU cycles if too many pages are allocated in a
- * second. Therefore, we throttle nosleep allocations even when
- * the guest is not under memory pressure. OTOH, if we have already
- * predicted that the guest is under memory pressure, then we
- * slowdown page allocations considerably.
- */
+ while ((to_inflate_frames = vmballoon_change(b)) > 0) {
+ unsigned int to_inflate_pages, page_in_frames;
+ int alloc_error, lock_error = 0;
- /*
- * Start with no sleep allocation rate which may be higher
- * than sleeping allocation rate.
- */
- is_2m_pages = b->supported_page_sizes == VMW_BALLOON_NUM_PAGE_SIZES;
+ VM_BUG_ON(!list_empty(&ctl.pages));
+ VM_BUG_ON(ctl.n_pages != 0);
- while ((int64_t)(num_pages * vmballoon_page_size(is_2m_pages)) <
- vmballoon_change(b)) {
- struct page *page;
+ page_in_frames = vmballoon_page_in_frames(ctl.page_size);
- vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_ALLOC,
- is_2m_pages);
-
- page = vmballoon_alloc_page(is_2m_pages);
- if (!page) {
- vmballoon_stats_page_inc(b,
- VMW_BALLOON_PAGE_STAT_ALLOC_FAIL, is_2m_pages);
-
- if (is_2m_pages) {
- vmballoon_lock(b, num_pages, true);
-
- /*
- * ignore errors from locking as we now switch
- * to 4k pages and we might get different
- * errors.
- */
-
- num_pages = 0;
- is_2m_pages = false;
- continue;
- }
+ to_inflate_pages = min_t(unsigned long, b->batch_max_pages,
+ DIV_ROUND_UP_ULL(to_inflate_frames,
+ page_in_frames));
+
+ /* Start by allocating */
+ alloc_error = vmballoon_alloc_page_list(b, &ctl,
+ to_inflate_pages);
+
+ /* Actually lock the pages by telling the hypervisor */
+ lock_error = vmballoon_lock(b, &ctl);
+
+ /*
+ * If an error indicates that something serious went wrong,
+ * stop the inflation.
+ */
+ if (lock_error)
break;
- }
- vmballoon_add_page(b, num_pages++, page);
- if (num_pages == b->batch_max_pages) {
- struct vmballoon_page_size *page_size =
- &b->page_sizes[is_2m_pages];
+ /* Update the balloon size */
+ atomic64_add(ctl.n_pages * page_in_frames, &b->size);
- error = vmballoon_lock(b, num_pages, is_2m_pages);
+ vmballoon_enqueue_page_list(b, &ctl.pages, &ctl.n_pages,
+ ctl.page_size);
- num_pages = 0;
+ /*
+ * If allocation failed or the number of refused pages exceeds
+ * the maximum allowed, move to the next page size.
+ */
+ if (alloc_error ||
+ ctl.n_refused_pages >= VMW_BALLOON_MAX_REFUSED) {
+ if (ctl.page_size == VMW_BALLOON_4K_PAGE)
+ break;
/*
- * Stop allocating this page size if we already
- * accumulated too many pages that the hypervisor
- * refused.
+ * Ignore errors from locking as we now switch to 4k
+ * pages and we might get different errors.
*/
- if (page_size->n_refused_pages >=
- VMW_BALLOON_MAX_REFUSED) {
- if (!is_2m_pages)
- break;
-
- /*
- * Release the refused pages as we move to 4k
- * pages.
- */
- vmballoon_release_refused_pages(b, true);
- is_2m_pages = true;
- }
-
- if (error)
- break;
+ vmballoon_release_refused_pages(b, &ctl);
+ ctl.page_size--;
}
cond_resched();
}
- if (num_pages > 0)
- vmballoon_lock(b, num_pages, is_2m_pages);
-
- vmballoon_release_refused_pages(b, true);
- vmballoon_release_refused_pages(b, false);
+ /*
+ * Release pages that were allocated while attempting to inflate the
+ * balloon but were refused by the host for one reason or another,
+ * and update the statistics.
+ */
+ if (ctl.n_refused_pages != 0)
+ vmballoon_release_refused_pages(b, &ctl);
}
-/*
+/**
+ * vmballoon_deflate() - Decrease the size of the balloon.
+ *
+ * @b: pointer to the balloon
+ * @n_frames: the number of frames to deflate. If zero, automatically
+ * calculated according to the target size.
+ * @coordinated: whether to coordinate with the host
+ *
* Decrease the size of the balloon allowing guest to use more memory.
+ *
+ * Return: The number of deflated frames (i.e., basic page size units)
*/
-static void vmballoon_deflate(struct vmballoon *b)
+static unsigned long vmballoon_deflate(struct vmballoon *b, uint64_t n_frames,
+ bool coordinated)
{
- unsigned is_2m_pages;
+ unsigned long deflated_frames = 0;
+ unsigned long tried_frames = 0;
+ struct vmballoon_ctl ctl = {
+ .pages = LIST_HEAD_INIT(ctl.pages),
+ .refused_pages = LIST_HEAD_INIT(ctl.refused_pages),
+ .page_size = VMW_BALLOON_4K_PAGE,
+ .op = VMW_BALLOON_DEFLATE
+ };
/* free pages to reach target */
- for (is_2m_pages = 0; is_2m_pages < b->supported_page_sizes;
- is_2m_pages++) {
- struct page *page, *next;
- unsigned int num_pages = 0;
- struct vmballoon_page_size *page_size =
- &b->page_sizes[is_2m_pages];
-
- list_for_each_entry_safe(page, next, &page_size->pages, lru) {
- if ((int64_t)(num_pages *
- vmballoon_page_size(is_2m_pages)) >=
- -vmballoon_change(b))
- break;
+ while (true) {
+ unsigned int to_deflate_pages, n_unlocked_frames;
+ unsigned int page_in_frames;
+ int64_t to_deflate_frames;
+ bool deflated_all;
+
+ page_in_frames = vmballoon_page_in_frames(ctl.page_size);
+
+ VM_BUG_ON(!list_empty(&ctl.pages));
+ VM_BUG_ON(ctl.n_pages);
+ VM_BUG_ON(!list_empty(&ctl.refused_pages));
+ VM_BUG_ON(ctl.n_refused_pages);
+
+ /*
+ * If we were requested a specific number of frames, we try to
+ * deflate this number of frames. Otherwise, deflation is
+ * performed according to the target and balloon size.
+ */
+ to_deflate_frames = n_frames ? n_frames - tried_frames :
+ -vmballoon_change(b);
+
+ /* break if no work to do */
+ if (to_deflate_frames <= 0)
+ break;
+
+ /*
+ * Calculate the number of frames based on current page size,
+ * but limit the deflated frames to a single chunk
+ */
+ to_deflate_pages = min_t(unsigned long, b->batch_max_pages,
+ DIV_ROUND_UP_ULL(to_deflate_frames,
+ page_in_frames));
+
+ /* First take the pages from the balloon pages. */
+ vmballoon_dequeue_page_list(b, &ctl.pages, &ctl.n_pages,
+ ctl.page_size, to_deflate_pages);
- list_del(&page->lru);
- vmballoon_add_page(b, num_pages++, page);
+ /*
+ * Before pages are moving to the refused list, count their
+ * frames as frames that we tried to deflate.
+ */
+ tried_frames += ctl.n_pages * page_in_frames;
+
+ /*
+ * Unlock the pages by communicating with the hypervisor if the
+ * communication is coordinated (i.e., not pop). We ignore the
+ * return code. Instead we check if all the pages we manage to
+ * unlock all the pages. If we failed, we will move to the next
+ * page size, and would eventually try again later.
+ */
+ if (coordinated)
+ vmballoon_lock(b, &ctl);
+
+ /*
+ * Check if we deflated enough. We will move to the next page
+ * size if we did not manage to do so. This calculation takes
+ * place now, as once the pages are released, the number of
+ * pages is zeroed.
+ */
+ deflated_all = (ctl.n_pages == to_deflate_pages);
- if (num_pages == b->batch_max_pages) {
- int error;
+ /* Update local and global counters */
+ n_unlocked_frames = ctl.n_pages * page_in_frames;
+ atomic64_sub(n_unlocked_frames, &b->size);
+ deflated_frames += n_unlocked_frames;
- error = vmballoon_unlock(b, num_pages,
- is_2m_pages);
- num_pages = 0;
- if (error)
- return;
- }
+ vmballoon_stats_page_add(b, VMW_BALLOON_PAGE_STAT_FREE,
+ ctl.page_size, ctl.n_pages);
- cond_resched();
+ /* free the ballooned pages */
+ vmballoon_release_page_list(&ctl.pages, &ctl.n_pages,
+ ctl.page_size);
+
+ /* Return the refused pages to the ballooned list. */
+ vmballoon_enqueue_page_list(b, &ctl.refused_pages,
+ &ctl.n_refused_pages,
+ ctl.page_size);
+
+ /* If we failed to unlock all the pages, move to next size. */
+ if (!deflated_all) {
+ if (ctl.page_size == b->max_page_size)
+ break;
+ ctl.page_size++;
}
- if (num_pages > 0)
- vmballoon_unlock(b, num_pages, is_2m_pages);
+ cond_resched();
}
+
+ return deflated_frames;
}
/**
@@ -1004,6 +1245,23 @@ static int vmballoon_vmci_init(struct vmballoon *b)
return -EIO;
}
+/**
+ * vmballoon_pop - Quickly release all pages allocate for the balloon.
+ *
+ * @b: pointer to the balloon.
+ *
+ * This function is called when host decides to "reset" balloon for one reason
+ * or another. Unlike normal "deflate" we do not (shall not) notify host of the
+ * pages being released.
+ */
+static void vmballoon_pop(struct vmballoon *b)
+{
+ unsigned long size;
+
+ while ((size = atomic64_read(&b->size)))
+ vmballoon_deflate(b, size, false);
+}
+
/*
* Perform standard reset sequence by popping the balloon (in case it
* is not empty) and then restarting protocol. This operation normally
@@ -1080,13 +1338,13 @@ static void vmballoon_work(struct work_struct *work)
change = vmballoon_change(b);
if (change != 0) {
- pr_debug("%s - size: %u, target %u", __func__,
- b->size, b->target);
+ pr_debug("%s - size: %llu, target %lu\n", __func__,
+ atomic64_read(&b->size), READ_ONCE(b->target));
if (change > 0)
vmballoon_inflate(b);
else /* (change < 0) */
- vmballoon_deflate(b);
+ vmballoon_deflate(b, 0, true);
}
up_read(&b->conf_sem);
@@ -1118,11 +1376,6 @@ static const char * const vmballoon_stat_names[] = {
[VMW_BALLOON_STAT_DOORBELL] = "doorbell"
};
-static const char * const vmballoon_page_size_names[] = {
- [VMW_BALLOON_4K_PAGE] = "4k",
- [VMW_BALLOON_2M_PAGE] = "2M"
-};
-
static int vmballoon_enable_stats(struct vmballoon *b)
{
int r = 0;
@@ -1171,16 +1424,15 @@ static int vmballoon_debug_show(struct seq_file *f, void *offset)
}
/* format capabilities info */
- seq_printf(f, "%-22s: %#4x\n", "balloon capabilities",
+ seq_printf(f, "%-22s: %#16x\n", "balloon capabilities",
VMW_BALLOON_CAPABILITIES);
- seq_printf(f, "%-22s: %#4lx\n", "used capabilities",
- b->capabilities);
+ seq_printf(f, "%-22s: %#16lx\n", "used capabilities", b->capabilities);
seq_printf(f, "%-22s: %16s\n", "is resetting",
b->reset_required ? "y" : "n");
/* format size info */
- seq_printf(f, "%-22s: %16u\n", "target", b->target);
- seq_printf(f, "%-22s: %16u\n", "current", b->size);
+ seq_printf(f, "%-22s: %16lu\n", "target", READ_ONCE(b->target));
+ seq_printf(f, "%-22s: %16llu\n", "current", atomic64_read(&b->size));
for (i = 0; i < VMW_BALLOON_CMD_NUM; i++) {
if (vmballoon_cmd_names[i] == NULL)
@@ -1259,8 +1511,9 @@ static inline void vmballoon_debugfs_exit(struct vmballoon *b)
static int __init vmballoon_init(void)
{
+ enum vmballoon_page_size_type page_size;
int error;
- unsigned is_2m_pages;
+
/*
* Check if we are running on VMware's hypervisor and bail out
* if we are not.
@@ -1268,11 +1521,10 @@ static int __init vmballoon_init(void)
if (x86_hyper_type != X86_HYPER_VMWARE)
return -ENODEV;
- for (is_2m_pages = 0; is_2m_pages < VMW_BALLOON_NUM_PAGE_SIZES;
- is_2m_pages++) {
- INIT_LIST_HEAD(&balloon.page_sizes[is_2m_pages].pages);
- INIT_LIST_HEAD(&balloon.page_sizes[is_2m_pages].refused_pages);
- }
+ for (page_size = VMW_BALLOON_4K_PAGE;
+ page_size <= VMW_BALLOON_LAST_SIZE; page_size++)
+ INIT_LIST_HEAD(&balloon.page_sizes[page_size].pages);
+
INIT_DELAYED_WORK(&balloon.dwork, vmballoon_work);
@@ -1280,6 +1532,7 @@ static int __init vmballoon_init(void)
if (error)
return error;
+ spin_lock_init(&balloon.comm_lock);
init_rwsem(&balloon.conf_sem);
balloon.vmci_doorbell = VMCI_INVALID_HANDLE;
balloon.batch_page = NULL;
--
2.17.1
^ permalink raw reply related [flat|nested] 24+ messages in thread* [PATCH v2 13/20] vmw_balloon: general style cleanup
2018-09-20 17:30 [PATCH v2 00/20] vmw_balloon: compaction, shrinker, 64-bit, etc Nadav Amit
` (11 preceding siblings ...)
2018-09-20 17:30 ` [PATCH v2 12/20] vmw_balloon: rework the inflate and deflate loops Nadav Amit
@ 2018-09-20 17:30 ` Nadav Amit
2018-09-20 17:30 ` [PATCH v2 14/20] vmw_balloon: add reset stat Nadav Amit
` (7 subsequent siblings)
20 siblings, 0 replies; 24+ messages in thread
From: Nadav Amit @ 2018-09-20 17:30 UTC (permalink / raw)
To: Greg Kroah-Hartman, Arnd Bergmann
Cc: linux-kernel, Xavier Deguillard, Nadav Amit
Change all the remaining return values to int to avoid mistakes. Reduce
indentation when possible.
Reviewed-by: Xavier Deguillard <xdeguillard@vmware.com>
Signed-off-by: Nadav Amit <namit@vmware.com>
---
drivers/misc/vmw_balloon.c | 39 ++++++++++++++++++++++----------------
1 file changed, 23 insertions(+), 16 deletions(-)
diff --git a/drivers/misc/vmw_balloon.c b/drivers/misc/vmw_balloon.c
index cb363db34b4a..99a831ea82c0 100644
--- a/drivers/misc/vmw_balloon.c
+++ b/drivers/misc/vmw_balloon.c
@@ -477,10 +477,9 @@ vmballoon_cmd(struct vmballoon *b, unsigned long cmd, unsigned long arg1,
* Send "start" command to the host, communicating supported version
* of the protocol.
*/
-static bool vmballoon_send_start(struct vmballoon *b, unsigned long req_caps)
+static int vmballoon_send_start(struct vmballoon *b, unsigned long req_caps)
{
unsigned long status, capabilities;
- bool success;
status = __vmballoon_cmd(b, VMW_BALLOON_CMD_START, req_caps, 0,
&capabilities);
@@ -488,14 +487,12 @@ static bool vmballoon_send_start(struct vmballoon *b, unsigned long req_caps)
switch (status) {
case VMW_BALLOON_SUCCESS_WITH_CAPABILITIES:
b->capabilities = capabilities;
- success = true;
break;
case VMW_BALLOON_SUCCESS:
b->capabilities = VMW_BALLOON_BASIC_CMDS;
- success = true;
break;
default:
- success = false;
+ return -EIO;
}
/*
@@ -509,26 +506,29 @@ static bool vmballoon_send_start(struct vmballoon *b, unsigned long req_caps)
b->max_page_size = VMW_BALLOON_2M_PAGE;
- return success;
+ return 0;
}
-/*
+/**
+ * vmballoon_send_guest_id - communicate guest type to the host.
+ *
+ * @b: pointer to the balloon.
+ *
* Communicate guest type to the host so that it can adjust ballooning
* algorithm to the one most appropriate for the guest. This command
* is normally issued after sending "start" command and is part of
* standard reset sequence.
+ *
+ * Return: zero on success or appropriate error code.
*/
-static bool vmballoon_send_guest_id(struct vmballoon *b)
+static int vmballoon_send_guest_id(struct vmballoon *b)
{
unsigned long status;
status = vmballoon_cmd(b, VMW_BALLOON_CMD_GUEST_ID,
VMW_BALLOON_GUEST_ID, 0);
- if (status == VMW_BALLOON_SUCCESS)
- return true;
-
- return false;
+ return status == VMW_BALLOON_SUCCESS ? 0 : -EIO;
}
/**
@@ -1215,8 +1215,15 @@ static void vmballoon_vmci_cleanup(struct vmballoon *b)
}
}
-/*
- * Initialize vmci doorbell, to get notified as soon as balloon changes
+/**
+ * vmballoon_vmci_init - Initialize vmci doorbell.
+ *
+ * @b: pointer to the balloon.
+ *
+ * Return: zero on success or when wakeup command not supported. Error-code
+ * otherwise.
+ *
+ * Initialize vmci doorbell, to get notified as soon as balloon changes.
*/
static int vmballoon_vmci_init(struct vmballoon *b)
{
@@ -1278,7 +1285,7 @@ static void vmballoon_reset(struct vmballoon *b)
/* free all pages, skipping monitor unlock */
vmballoon_pop(b);
- if (!vmballoon_send_start(b, VMW_BALLOON_CAPABILITIES))
+ if (vmballoon_send_start(b, VMW_BALLOON_CAPABILITIES))
return;
if ((b->capabilities & VMW_BALLOON_BATCHED_CMDS) != 0) {
@@ -1302,7 +1309,7 @@ static void vmballoon_reset(struct vmballoon *b)
if (error)
pr_err("failed to initialize vmci doorbell\n");
- if (!vmballoon_send_guest_id(b))
+ if (vmballoon_send_guest_id(b))
pr_err("failed to send guest ID to the host\n");
up_write(&b->conf_sem);
--
2.17.1
^ permalink raw reply related [flat|nested] 24+ messages in thread* [PATCH v2 14/20] vmw_balloon: add reset stat
2018-09-20 17:30 [PATCH v2 00/20] vmw_balloon: compaction, shrinker, 64-bit, etc Nadav Amit
` (12 preceding siblings ...)
2018-09-20 17:30 ` [PATCH v2 13/20] vmw_balloon: general style cleanup Nadav Amit
@ 2018-09-20 17:30 ` Nadav Amit
2018-09-20 17:30 ` [PATCH v2 15/20] mm/balloon_compaction: suppress allocation warnings Nadav Amit
` (6 subsequent siblings)
20 siblings, 0 replies; 24+ messages in thread
From: Nadav Amit @ 2018-09-20 17:30 UTC (permalink / raw)
To: Greg Kroah-Hartman, Arnd Bergmann
Cc: linux-kernel, Xavier Deguillard, Nadav Amit
It is useful to expose how many times the balloon resets. If it happens
more than very rarely - this is an indication for a problem.
Reviewed-by: Xavier Deguillard <xdeguillard@vmware.com>
Signed-off-by: Nadav Amit <namit@vmware.com>
---
drivers/misc/vmw_balloon.c | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/drivers/misc/vmw_balloon.c b/drivers/misc/vmw_balloon.c
index 99a831ea82c0..9b0b3fa4f836 100644
--- a/drivers/misc/vmw_balloon.c
+++ b/drivers/misc/vmw_balloon.c
@@ -218,7 +218,8 @@ enum vmballoon_stat_page {
enum vmballoon_stat_general {
VMW_BALLOON_STAT_TIMER,
VMW_BALLOON_STAT_DOORBELL,
- VMW_BALLOON_STAT_LAST = VMW_BALLOON_STAT_DOORBELL
+ VMW_BALLOON_STAT_RESET,
+ VMW_BALLOON_STAT_LAST = VMW_BALLOON_STAT_RESET
};
#define VMW_BALLOON_STAT_NUM (VMW_BALLOON_STAT_LAST + 1)
@@ -1303,6 +1304,7 @@ static void vmballoon_reset(struct vmballoon *b)
vmballoon_deinit_batching(b);
}
+ vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_RESET);
b->reset_required = false;
error = vmballoon_vmci_init(b);
@@ -1380,7 +1382,8 @@ static const char * const vmballoon_stat_page_names[] = {
static const char * const vmballoon_stat_names[] = {
[VMW_BALLOON_STAT_TIMER] = "timer",
- [VMW_BALLOON_STAT_DOORBELL] = "doorbell"
+ [VMW_BALLOON_STAT_DOORBELL] = "doorbell",
+ [VMW_BALLOON_STAT_RESET] = "reset",
};
static int vmballoon_enable_stats(struct vmballoon *b)
--
2.17.1
^ permalink raw reply related [flat|nested] 24+ messages in thread* [PATCH v2 15/20] mm/balloon_compaction: suppress allocation warnings
2018-09-20 17:30 [PATCH v2 00/20] vmw_balloon: compaction, shrinker, 64-bit, etc Nadav Amit
` (13 preceding siblings ...)
2018-09-20 17:30 ` [PATCH v2 14/20] vmw_balloon: add reset stat Nadav Amit
@ 2018-09-20 17:30 ` Nadav Amit
2018-09-20 17:30 ` [PATCH v2 16/20] mm/balloon_compaction: list interfaces Nadav Amit
` (5 subsequent siblings)
20 siblings, 0 replies; 24+ messages in thread
From: Nadav Amit @ 2018-09-20 17:30 UTC (permalink / raw)
To: Greg Kroah-Hartman, Arnd Bergmann
Cc: linux-kernel, Xavier Deguillard, Nadav Amit, Michael S. Tsirkin,
Jason Wang, virtualization, linux-mm
There is no reason to print warnings when balloon page allocation fails.
Cc: "Michael S. Tsirkin" <mst@redhat.com>
Cc: Jason Wang <jasowang@redhat.com>
Cc: virtualization@lists.linux-foundation.org
Cc: linux-mm@kvack.org
Reviewed-by: Xavier Deguillard <xdeguillard@vmware.com>
Signed-off-by: Nadav Amit <namit@vmware.com>
---
mm/balloon_compaction.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/mm/balloon_compaction.c b/mm/balloon_compaction.c
index ef858d547e2d..a6c0efb3544f 100644
--- a/mm/balloon_compaction.c
+++ b/mm/balloon_compaction.c
@@ -22,7 +22,8 @@
struct page *balloon_page_alloc(void)
{
struct page *page = alloc_page(balloon_mapping_gfp_mask() |
- __GFP_NOMEMALLOC | __GFP_NORETRY);
+ __GFP_NOMEMALLOC | __GFP_NORETRY |
+ __GFP_NOWARN);
return page;
}
EXPORT_SYMBOL_GPL(balloon_page_alloc);
--
2.17.1
^ permalink raw reply related [flat|nested] 24+ messages in thread* [PATCH v2 16/20] mm/balloon_compaction: list interfaces
2018-09-20 17:30 [PATCH v2 00/20] vmw_balloon: compaction, shrinker, 64-bit, etc Nadav Amit
` (14 preceding siblings ...)
2018-09-20 17:30 ` [PATCH v2 15/20] mm/balloon_compaction: suppress allocation warnings Nadav Amit
@ 2018-09-20 17:30 ` Nadav Amit
2018-09-20 17:30 ` [PATCH v2 17/20] vmw_balloon: compaction support Nadav Amit
` (4 subsequent siblings)
20 siblings, 0 replies; 24+ messages in thread
From: Nadav Amit @ 2018-09-20 17:30 UTC (permalink / raw)
To: Greg Kroah-Hartman, Arnd Bergmann
Cc: linux-kernel, Xavier Deguillard, Nadav Amit, Michael S. Tsirkin,
Jason Wang, linux-mm, virtualization
Introduce interfaces for ballooning enqueueing and dequeueing of a list
of pages. These interfaces reduce the overhead of storing and restoring
IRQs by batching the operations. In addition they do not panic if the
list of pages is empty.
Cc: "Michael S. Tsirkin" <mst@redhat.com>
Cc: Jason Wang <jasowang@redhat.com>
Cc: linux-mm@kvack.org
Cc: virtualization@lists.linux-foundation.org
Reviewed-by: Xavier Deguillard <xdeguillard@vmware.com>
Signed-off-by: Nadav Amit <namit@vmware.com>
---
include/linux/balloon_compaction.h | 4 +
mm/balloon_compaction.c | 139 +++++++++++++++++++++--------
2 files changed, 105 insertions(+), 38 deletions(-)
diff --git a/include/linux/balloon_compaction.h b/include/linux/balloon_compaction.h
index 53051f3d8f25..2c5a8e09e413 100644
--- a/include/linux/balloon_compaction.h
+++ b/include/linux/balloon_compaction.h
@@ -72,6 +72,10 @@ extern struct page *balloon_page_alloc(void);
extern void balloon_page_enqueue(struct balloon_dev_info *b_dev_info,
struct page *page);
extern struct page *balloon_page_dequeue(struct balloon_dev_info *b_dev_info);
+extern void balloon_page_list_enqueue(struct balloon_dev_info *b_dev_info,
+ struct list_head *pages);
+extern int balloon_page_list_dequeue(struct balloon_dev_info *b_dev_info,
+ struct list_head *pages, int n_req_pages);
static inline void balloon_devinfo_init(struct balloon_dev_info *balloon)
{
diff --git a/mm/balloon_compaction.c b/mm/balloon_compaction.c
index a6c0efb3544f..b920c2a10d6f 100644
--- a/mm/balloon_compaction.c
+++ b/mm/balloon_compaction.c
@@ -10,6 +10,100 @@
#include <linux/export.h>
#include <linux/balloon_compaction.h>
+static int balloon_page_enqueue_one(struct balloon_dev_info *b_dev_info,
+ struct page *page)
+{
+ /*
+ * Block others from accessing the 'page' when we get around to
+ * establishing additional references. We should be the only one
+ * holding a reference to the 'page' at this point.
+ */
+ if (!trylock_page(page)) {
+ WARN_ONCE(1, "balloon inflation failed to enqueue page\n");
+ return -EFAULT;
+ }
+ list_del(&page->lru);
+ balloon_page_insert(b_dev_info, page);
+ unlock_page(page);
+ __count_vm_event(BALLOON_INFLATE);
+ return 0;
+}
+
+/**
+ * balloon_page_list_enqueue() - inserts a list of pages into the balloon page
+ * list.
+ * @b_dev_info: balloon device descriptor where we will insert a new page to
+ * @pages: pages to enqueue - allocated using balloon_page_alloc.
+ *
+ * Driver must call it to properly enqueue a balloon pages before definitively
+ * removing it from the guest system.
+ */
+void balloon_page_list_enqueue(struct balloon_dev_info *b_dev_info,
+ struct list_head *pages)
+{
+ struct page *page, *tmp;
+ unsigned long flags;
+
+ spin_lock_irqsave(&b_dev_info->pages_lock, flags);
+ list_for_each_entry_safe(page, tmp, pages, lru)
+ balloon_page_enqueue_one(b_dev_info, page);
+ spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
+}
+EXPORT_SYMBOL_GPL(balloon_page_list_enqueue);
+
+/**
+ * balloon_page_list_dequeue() - removes pages from balloon's page list and
+ * returns a list of the pages.
+ * @b_dev_info: balloon device decriptor where we will grab a page from.
+ * @pages: pointer to the list of pages that would be returned to the caller.
+ * @n_req_pages: number of requested pages.
+ *
+ * Driver must call it to properly de-allocate a previous enlisted balloon pages
+ * before definetively releasing it back to the guest system. This function
+ * tries to remove @n_req_pages from the ballooned pages and return it to the
+ * caller in the @pages list.
+ *
+ * Note that this function may fail to dequeue some pages temporarily empty due
+ * to compaction isolated pages.
+ *
+ * Return: number of pages that were added to the @pages list.
+ */
+int balloon_page_list_dequeue(struct balloon_dev_info *b_dev_info,
+ struct list_head *pages, int n_req_pages)
+{
+ struct page *page, *tmp;
+ unsigned long flags;
+ int n_pages = 0;
+
+ spin_lock_irqsave(&b_dev_info->pages_lock, flags);
+ list_for_each_entry_safe(page, tmp, &b_dev_info->pages, lru) {
+ /*
+ * Block others from accessing the 'page' while we get around
+ * establishing additional references and preparing the 'page'
+ * to be released by the balloon driver.
+ */
+ if (!trylock_page(page))
+ continue;
+
+ if (IS_ENABLED(CONFIG_BALLOON_COMPACTION) &&
+ PageIsolated(page)) {
+ /* raced with isolation */
+ unlock_page(page);
+ continue;
+ }
+ balloon_page_delete(page);
+ __count_vm_event(BALLOON_DEFLATE);
+ unlock_page(page);
+ list_add(&page->lru, pages);
+ if (++n_pages >= n_req_pages)
+ break;
+ }
+ spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
+
+ return n_pages;
+}
+EXPORT_SYMBOL_GPL(balloon_page_list_dequeue);
+
/*
* balloon_page_alloc - allocates a new page for insertion into the balloon
* page list.
@@ -44,17 +138,9 @@ void balloon_page_enqueue(struct balloon_dev_info *b_dev_info,
{
unsigned long flags;
- /*
- * Block others from accessing the 'page' when we get around to
- * establishing additional references. We should be the only one
- * holding a reference to the 'page' at this point.
- */
- BUG_ON(!trylock_page(page));
spin_lock_irqsave(&b_dev_info->pages_lock, flags);
- balloon_page_insert(b_dev_info, page);
- __count_vm_event(BALLOON_INFLATE);
+ balloon_page_enqueue_one(b_dev_info, page);
spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
- unlock_page(page);
}
EXPORT_SYMBOL_GPL(balloon_page_enqueue);
@@ -71,36 +157,13 @@ EXPORT_SYMBOL_GPL(balloon_page_enqueue);
*/
struct page *balloon_page_dequeue(struct balloon_dev_info *b_dev_info)
{
- struct page *page, *tmp;
unsigned long flags;
- bool dequeued_page;
+ LIST_HEAD(pages);
+ int n_pages;
- dequeued_page = false;
- spin_lock_irqsave(&b_dev_info->pages_lock, flags);
- list_for_each_entry_safe(page, tmp, &b_dev_info->pages, lru) {
- /*
- * Block others from accessing the 'page' while we get around
- * establishing additional references and preparing the 'page'
- * to be released by the balloon driver.
- */
- if (trylock_page(page)) {
-#ifdef CONFIG_BALLOON_COMPACTION
- if (PageIsolated(page)) {
- /* raced with isolation */
- unlock_page(page);
- continue;
- }
-#endif
- balloon_page_delete(page);
- __count_vm_event(BALLOON_DEFLATE);
- unlock_page(page);
- dequeued_page = true;
- break;
- }
- }
- spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
+ n_pages = balloon_page_list_dequeue(b_dev_info, &pages, 1);
- if (!dequeued_page) {
+ if (n_pages != 1) {
/*
* If we are unable to dequeue a balloon page because the page
* list is empty and there is no isolated pages, then something
@@ -113,9 +176,9 @@ struct page *balloon_page_dequeue(struct balloon_dev_info *b_dev_info)
!b_dev_info->isolated_pages))
BUG();
spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
- page = NULL;
+ return NULL;
}
- return page;
+ return list_first_entry(&pages, struct page, lru);
}
EXPORT_SYMBOL_GPL(balloon_page_dequeue);
--
2.17.1
^ permalink raw reply related [flat|nested] 24+ messages in thread* [PATCH v2 17/20] vmw_balloon: compaction support
2018-09-20 17:30 [PATCH v2 00/20] vmw_balloon: compaction, shrinker, 64-bit, etc Nadav Amit
` (15 preceding siblings ...)
2018-09-20 17:30 ` [PATCH v2 16/20] mm/balloon_compaction: list interfaces Nadav Amit
@ 2018-09-20 17:30 ` Nadav Amit
2018-09-25 18:15 ` Greg Kroah-Hartman
2018-09-20 17:30 ` [PATCH v2 18/20] vmw_balloon: support 64-bit memory limit Nadav Amit
` (3 subsequent siblings)
20 siblings, 1 reply; 24+ messages in thread
From: Nadav Amit @ 2018-09-20 17:30 UTC (permalink / raw)
To: Greg Kroah-Hartman, Arnd Bergmann
Cc: linux-kernel, Xavier Deguillard, Nadav Amit
Add support for compaction for VMware balloon. Since unlike the virtio
balloon, we also support huge-pages, which are not going through
compaction, we keep these pages in vmballoon and handle this list
separately. We use the same lock to protect both lists, as this lock is
not supposed to be contended.
Doing so also eliminates the need for the page_size lists. We update the
accounting as needed to reflect inflation, deflation and migration to be
reflected in vmstat.
Since VMware balloon now provides statistics for inflation, deflation
and migration in vmstat, select MEMORY_BALLOON in Kconfig.
Reviewed-by: Xavier Deguillard <xdeguillard@vmware.com>
Signed-off-by: Nadav Amit <namit@vmware.com>
---
drivers/misc/Kconfig | 1 +
drivers/misc/vmw_balloon.c | 299 ++++++++++++++++++++++++++++++++-----
include/uapi/linux/magic.h | 1 +
3 files changed, 263 insertions(+), 38 deletions(-)
diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig
index 3726eacdf65d..98e684961ece 100644
--- a/drivers/misc/Kconfig
+++ b/drivers/misc/Kconfig
@@ -409,6 +409,7 @@ config SPEAR13XX_PCIE_GADGET
config VMWARE_BALLOON
tristate "VMware Balloon Driver"
depends on VMWARE_VMCI && X86 && HYPERVISOR_GUEST
+ select MEMORY_BALLOON
help
This is VMware physical memory management driver which acts
like a "balloon" that can be inflated to reclaim physical pages
diff --git a/drivers/misc/vmw_balloon.c b/drivers/misc/vmw_balloon.c
index 9b0b3fa4f836..cace823bc68c 100644
--- a/drivers/misc/vmw_balloon.c
+++ b/drivers/misc/vmw_balloon.c
@@ -28,6 +28,9 @@
#include <linux/rwsem.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
+#include <linux/mount.h>
+#include <linux/magic.h>
+#include <linux/balloon_compaction.h>
#include <linux/vmw_vmci_defs.h>
#include <linux/vmw_vmci_api.h>
#include <asm/hypervisor.h>
@@ -39,23 +42,6 @@ MODULE_ALIAS("dmi:*:svnVMware*:*");
MODULE_ALIAS("vmware_vmmemctl");
MODULE_LICENSE("GPL");
-/*
- * Use __GFP_HIGHMEM to allow pages from HIGHMEM zone. We don't allow wait
- * (__GFP_RECLAIM) for huge page allocations. Use __GFP_NOWARN, to suppress page
- * allocation failure warnings. Disallow access to emergency low-memory pools.
- */
-#define VMW_HUGE_PAGE_ALLOC_FLAGS (__GFP_HIGHMEM|__GFP_NOWARN| \
- __GFP_NOMEMALLOC)
-
-/*
- * Use __GFP_HIGHMEM to allow pages from HIGHMEM zone. We allow lightweight
- * reclamation (__GFP_NORETRY). Use __GFP_NOWARN, to suppress page allocation
- * failure warnings. Disallow access to emergency low-memory pools.
- */
-#define VMW_PAGE_ALLOC_FLAGS (__GFP_HIGHMEM|__GFP_NOWARN| \
- __GFP_NOMEMALLOC|__GFP_NORETRY)
-
-/* Maximum number of refused pages we accumulate during inflation cycle */
#define VMW_BALLOON_MAX_REFUSED 16
/*
@@ -237,11 +223,6 @@ struct vmballoon_ctl {
enum vmballoon_op op;
};
-struct vmballoon_page_size {
- /* list of reserved physical pages */
- struct list_head pages;
-};
-
/**
* struct vmballoon_batch_entry - a batch entry for lock or unlock.
*
@@ -256,8 +237,6 @@ struct vmballoon_batch_entry {
} __packed;
struct vmballoon {
- struct vmballoon_page_size page_sizes[VMW_BALLOON_NUM_PAGE_SIZES];
-
/**
* @max_page_size: maximum supported page size for ballooning.
*
@@ -338,8 +317,20 @@ struct vmballoon {
struct dentry *dbg_entry;
#endif
+ /**
+ * @b_dev_info: balloon device information descriptor.
+ */
+ struct balloon_dev_info b_dev_info;
+
struct delayed_work dwork;
+ /**
+ * @huge_pages - list of the inflated 2MB pages.
+ *
+ * Protected by @b_dev_info.pages_lock .
+ */
+ struct list_head huge_pages;
+
/**
* @vmci_doorbell.
*
@@ -602,10 +593,10 @@ static int vmballoon_alloc_page_list(struct vmballoon *b,
for (i = 0; i < req_n_pages; i++) {
if (ctl->page_size == VMW_BALLOON_2M_PAGE)
- page = alloc_pages(VMW_HUGE_PAGE_ALLOC_FLAGS,
- VMW_BALLOON_2M_ORDER);
+ page = alloc_pages(__GFP_HIGHMEM|__GFP_NOWARN|
+ __GFP_NOMEMALLOC, VMW_BALLOON_2M_ORDER);
else
- page = alloc_page(VMW_PAGE_ALLOC_FLAGS);
+ page = balloon_page_alloc();
/* Update statistics */
vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_ALLOC,
@@ -918,9 +909,22 @@ static void vmballoon_enqueue_page_list(struct vmballoon *b,
unsigned int *n_pages,
enum vmballoon_page_size_type page_size)
{
- struct vmballoon_page_size *page_size_info = &b->page_sizes[page_size];
+ unsigned long flags;
+
+ if (page_size == VMW_BALLOON_4K_PAGE) {
+ balloon_page_list_enqueue(&b->b_dev_info, pages);
+ } else {
+ /*
+ * Keep the huge pages in a local list which is not available
+ * for the balloon compaction mechanism.
+ */
+ spin_lock_irqsave(&b->b_dev_info.pages_lock, flags);
+ list_splice_init(pages, &b->huge_pages);
+ __count_vm_events(BALLOON_INFLATE, *n_pages *
+ vmballoon_page_in_frames(VMW_BALLOON_2M_PAGE));
+ spin_unlock_irqrestore(&b->b_dev_info.pages_lock, flags);
+ }
- list_splice_init(pages, &page_size_info->pages);
*n_pages = 0;
}
@@ -943,15 +947,28 @@ static void vmballoon_dequeue_page_list(struct vmballoon *b,
enum vmballoon_page_size_type page_size,
unsigned int n_req_pages)
{
- struct vmballoon_page_size *page_size_info = &b->page_sizes[page_size];
struct page *page, *tmp;
unsigned int i = 0;
+ unsigned long flags;
- list_for_each_entry_safe(page, tmp, &page_size_info->pages, lru) {
+ /* In the case of 4k pages, use the compaction infrastructure */
+ if (page_size == VMW_BALLOON_4K_PAGE) {
+ *n_pages = balloon_page_list_dequeue(&b->b_dev_info, pages,
+ n_req_pages);
+ return;
+ }
+
+ /* 2MB pages */
+ spin_lock_irqsave(&b->b_dev_info.pages_lock, flags);
+ list_for_each_entry_safe(page, tmp, &b->huge_pages, lru) {
list_move(&page->lru, pages);
if (++i == n_req_pages)
break;
}
+
+ __count_vm_events(BALLOON_DEFLATE,
+ i * vmballoon_page_in_frames(VMW_BALLOON_2M_PAGE));
+ spin_unlock_irqrestore(&b->b_dev_info.pages_lock, flags);
*n_pages = i;
}
@@ -1519,9 +1536,204 @@ static inline void vmballoon_debugfs_exit(struct vmballoon *b)
#endif /* CONFIG_DEBUG_FS */
+
+#ifdef CONFIG_BALLOON_COMPACTION
+
+static struct dentry *vmballoon_mount(struct file_system_type *fs_type,
+ int flags, const char *dev_name,
+ void *data)
+{
+ static const struct dentry_operations ops = {
+ .d_dname = simple_dname,
+ };
+
+ return mount_pseudo(fs_type, "balloon-vmware:", NULL, &ops,
+ BALLOON_VMW_MAGIC);
+}
+
+static struct file_system_type vmballoon_fs = {
+ .name = "balloon-vmware",
+ .mount = vmballoon_mount,
+ .kill_sb = kill_anon_super,
+};
+
+static struct vfsmount *vmballoon_mnt;
+
+/**
+ * vmballoon_migratepage() - migrates a balloon page.
+ * @b_dev_info: balloon device information descriptor.
+ * @newpage: the page to which @page should be migrated.
+ * @page: a ballooned page that should be migrated.
+ * @mode: migration mode, ignored.
+ *
+ * This function is really open-coded, but that is according to the interface
+ * that balloon_compaction provides.
+ *
+ * Return: zero on success, -EAGAIN when migration cannot be performed
+ * momentarily, and -EBUSY if migration failed and should be retried
+ * with that specific page.
+ */
+static int vmballoon_migratepage(struct balloon_dev_info *b_dev_info,
+ struct page *newpage, struct page *page,
+ enum migrate_mode mode)
+{
+ unsigned long status, flags;
+ struct vmballoon *b;
+ int ret;
+
+ b = container_of(b_dev_info, struct vmballoon, b_dev_info);
+
+ /*
+ * If the semaphore is taken, there is ongoing configuration change
+ * (i.e., balloon reset), so try again.
+ */
+ if (!down_read_trylock(&b->conf_sem))
+ return -EAGAIN;
+
+ spin_lock(&b->comm_lock);
+ /*
+ * We must start by deflating and not inflating, as otherwise the
+ * hypervisor may tell us that it has enough memory and the new page is
+ * not needed. Since the old page is isolated, we cannot use the list
+ * interface to unlock it, as the LRU field is used for isolation.
+ * Instead, we use the native interface directly.
+ */
+ vmballoon_add_page(b, 0, page);
+ status = vmballoon_lock_op(b, 1, VMW_BALLOON_4K_PAGE,
+ VMW_BALLOON_DEFLATE);
+
+ if (status == VMW_BALLOON_SUCCESS)
+ status = vmballoon_status_page(b, 0, &page);
+
+ /*
+ * If a failure happened, let the migration mechanism know that it
+ * should not retry.
+ */
+ if (status != VMW_BALLOON_SUCCESS) {
+ spin_unlock(&b->comm_lock);
+ ret = -EBUSY;
+ goto out_unlock;
+ }
+
+ /*
+ * The page is isolated, so it is safe to delete it without holding
+ * @pages_lock . We keep holding @comm_lock since we will need it in a
+ * second.
+ */
+ balloon_page_delete(page);
+
+ put_page(page);
+
+ /* Inflate */
+ vmballoon_add_page(b, 0, newpage);
+ status = vmballoon_lock_op(b, 1, VMW_BALLOON_4K_PAGE,
+ VMW_BALLOON_INFLATE);
+
+ if (status == VMW_BALLOON_SUCCESS)
+ status = vmballoon_status_page(b, 0, &newpage);
+
+ spin_unlock(&b->comm_lock);
+
+ if (status != VMW_BALLOON_SUCCESS) {
+ /*
+ * A failure happened. While we can deflate the page we just
+ * inflated, this deflation can also encounter an error. Instead
+ * we will decrease the size of the balloon to reflect the
+ * change and report failure.
+ */
+ atomic64_dec(&b->size);
+ ret = -EBUSY;
+ } else {
+ /*
+ * Success. Take a reference for the page, and we will add it to
+ * the list after acquiring the lock.
+ */
+ get_page(newpage);
+ ret = MIGRATEPAGE_SUCCESS;
+ }
+
+ /* Update the balloon list under the @pages_lock */
+ spin_lock_irqsave(&b->b_dev_info.pages_lock, flags);
+
+ /*
+ * On inflation success, we already took a reference for the @newpage.
+ * If we succeed just insert it to the list and update the statistics
+ * under the lock.
+ */
+ if (ret == MIGRATEPAGE_SUCCESS) {
+ balloon_page_insert(&b->b_dev_info, newpage);
+ __count_vm_event(BALLOON_MIGRATE);
+ }
+
+ /*
+ * We deflated successfully, so regardless to the inflation success, we
+ * need to reduce the number of isolated_pages.
+ */
+ b->b_dev_info.isolated_pages--;
+ spin_unlock_irqrestore(&b->b_dev_info.pages_lock, flags);
+
+out_unlock:
+ up_read(&b->conf_sem);
+ return ret;
+}
+
+/**
+ * vmballoon_compaction_deinit() - removes compaction related data.
+ *
+ * @b: pointer to the balloon.
+ */
+static void vmballoon_compaction_deinit(struct vmballoon *b)
+{
+ if (!IS_ERR(b->b_dev_info.inode))
+ iput(b->b_dev_info.inode);
+
+ b->b_dev_info.inode = NULL;
+ kern_unmount(vmballoon_mnt);
+ vmballoon_mnt = NULL;
+}
+
+/**
+ * vmballoon_compaction_init() - initialized compaction for the balloon.
+ *
+ * @b: pointer to the balloon.
+ *
+ * If during the initialization a failure occurred, this function does not
+ * perform cleanup. The caller must call vmballoon_compaction_deinit() in this
+ * case.
+ *
+ * Return: zero on success or error code on failure.
+ */
+static __init int vmballoon_compaction_init(struct vmballoon *b)
+{
+ vmballoon_mnt = kern_mount(&vmballoon_fs);
+ if (IS_ERR(vmballoon_mnt))
+ return PTR_ERR(vmballoon_mnt);
+
+ b->b_dev_info.migratepage = vmballoon_migratepage;
+ b->b_dev_info.inode = alloc_anon_inode(vmballoon_mnt->mnt_sb);
+
+ if (IS_ERR(b->b_dev_info.inode))
+ return PTR_ERR(b->b_dev_info.inode);
+
+ b->b_dev_info.inode->i_mapping->a_ops = &balloon_aops;
+ return 0;
+}
+
+#else /* CONFIG_BALLOON_COMPACTION */
+
+static void vmballoon_compaction_deinit(struct vmballoon *b)
+{
+}
+
+static int vmballoon_compaction_init(struct vmballoon *b)
+{
+ return 0;
+}
+
+#endif /* CONFIG_BALLOON_COMPACTION */
+
static int __init vmballoon_init(void)
{
- enum vmballoon_page_size_type page_size;
int error;
/*
@@ -1531,17 +1743,22 @@ static int __init vmballoon_init(void)
if (x86_hyper_type != X86_HYPER_VMWARE)
return -ENODEV;
- for (page_size = VMW_BALLOON_4K_PAGE;
- page_size <= VMW_BALLOON_LAST_SIZE; page_size++)
- INIT_LIST_HEAD(&balloon.page_sizes[page_size].pages);
-
-
INIT_DELAYED_WORK(&balloon.dwork, vmballoon_work);
error = vmballoon_debugfs_init(&balloon);
if (error)
- return error;
+ goto fail;
+ /*
+ * Initialization of compaction must be done after the call to
+ * balloon_devinfo_init() .
+ */
+ balloon_devinfo_init(&balloon.b_dev_info);
+ error = vmballoon_compaction_init(&balloon);
+ if (error)
+ goto fail;
+
+ INIT_LIST_HEAD(&balloon.huge_pages);
spin_lock_init(&balloon.comm_lock);
init_rwsem(&balloon.conf_sem);
balloon.vmci_doorbell = VMCI_INVALID_HANDLE;
@@ -1552,6 +1769,9 @@ static int __init vmballoon_init(void)
queue_delayed_work(system_freezable_wq, &balloon.dwork, 0);
return 0;
+fail:
+ vmballoon_compaction_deinit(&balloon);
+ return error;
}
/*
@@ -1576,5 +1796,8 @@ static void __exit vmballoon_exit(void)
*/
vmballoon_send_start(&balloon, 0);
vmballoon_pop(&balloon);
+
+ /* Only once we popped the balloon, compaction can be deinit */
+ vmballoon_compaction_deinit(&balloon);
}
module_exit(vmballoon_exit);
diff --git a/include/uapi/linux/magic.h b/include/uapi/linux/magic.h
index 1a6fee974116..d4e432295b3f 100644
--- a/include/uapi/linux/magic.h
+++ b/include/uapi/linux/magic.h
@@ -89,5 +89,6 @@
#define UDF_SUPER_MAGIC 0x15013346
#define BALLOON_KVM_MAGIC 0x13661366
#define ZSMALLOC_MAGIC 0x58295829
+#define BALLOON_VMW_MAGIC 0xba11007
#endif /* __LINUX_MAGIC_H__ */
--
2.17.1
^ permalink raw reply related [flat|nested] 24+ messages in thread* Re: [PATCH v2 17/20] vmw_balloon: compaction support
2018-09-20 17:30 ` [PATCH v2 17/20] vmw_balloon: compaction support Nadav Amit
@ 2018-09-25 18:15 ` Greg Kroah-Hartman
0 siblings, 0 replies; 24+ messages in thread
From: Greg Kroah-Hartman @ 2018-09-25 18:15 UTC (permalink / raw)
To: Nadav Amit; +Cc: Arnd Bergmann, linux-kernel, Xavier Deguillard
On Thu, Sep 20, 2018 at 10:30:23AM -0700, Nadav Amit wrote:
> --- a/include/uapi/linux/magic.h
> +++ b/include/uapi/linux/magic.h
> @@ -89,5 +89,6 @@
> #define UDF_SUPER_MAGIC 0x15013346
> #define BALLOON_KVM_MAGIC 0x13661366
> #define ZSMALLOC_MAGIC 0x58295829
> +#define BALLOON_VMW_MAGIC 0xba11007
Any specific reason why you didn't put the leading 0 on your magic
number to make everything align properly?
And there is no need to put this in magic.h, nothing outside of your
code sees this value. It is not a user api, so please do not make it
one.
thanks,
greg k-h
^ permalink raw reply [flat|nested] 24+ messages in thread
* [PATCH v2 18/20] vmw_balloon: support 64-bit memory limit
2018-09-20 17:30 [PATCH v2 00/20] vmw_balloon: compaction, shrinker, 64-bit, etc Nadav Amit
` (16 preceding siblings ...)
2018-09-20 17:30 ` [PATCH v2 17/20] vmw_balloon: compaction support Nadav Amit
@ 2018-09-20 17:30 ` Nadav Amit
2018-09-20 17:30 ` [PATCH v2 19/20] vmw_balloon: memory shrinker Nadav Amit
` (2 subsequent siblings)
20 siblings, 0 replies; 24+ messages in thread
From: Nadav Amit @ 2018-09-20 17:30 UTC (permalink / raw)
To: Greg Kroah-Hartman, Arnd Bergmann
Cc: linux-kernel, Xavier Deguillard, Nadav Amit
From: Xavier Deguillard <xdeguillard@vmware.com>
Currently, the balloon driver would fail to run if memory is greater
than 16TB of vRAM. Previous patches have already converted the balloon
target and size to 64-bit, so all that is left to do add is to avoid
asserting memory is smaller than 16TB if the hypervisor supports 64-bits
target.
The driver advertises a new capability VMW_BALLOON_64_BITS_TARGET.
Hypervisors that support 16TB of memory or more will report that this
capability is enabled.
Signed-off-by: Xavier Deguillard <xdeguillard@vmware.com>
Signed-off-by: Nadav Amit <namit@vmware.com>
---
drivers/misc/vmw_balloon.c | 18 +++++++++++++++---
1 file changed, 15 insertions(+), 3 deletions(-)
diff --git a/drivers/misc/vmw_balloon.c b/drivers/misc/vmw_balloon.c
index cace823bc68c..9da8b76342fd 100644
--- a/drivers/misc/vmw_balloon.c
+++ b/drivers/misc/vmw_balloon.c
@@ -59,15 +59,26 @@ enum vmwballoon_capabilities {
VMW_BALLOON_BATCHED_CMDS = (1 << 2),
VMW_BALLOON_BATCHED_2M_CMDS = (1 << 3),
VMW_BALLOON_SIGNALLED_WAKEUP_CMD = (1 << 4),
+ VMW_BALLOON_64_BIT_TARGET = (1 << 5)
};
-#define VMW_BALLOON_CAPABILITIES (VMW_BALLOON_BASIC_CMDS \
+#define VMW_BALLOON_CAPABILITIES_COMMON (VMW_BALLOON_BASIC_CMDS \
| VMW_BALLOON_BATCHED_CMDS \
| VMW_BALLOON_BATCHED_2M_CMDS \
| VMW_BALLOON_SIGNALLED_WAKEUP_CMD)
#define VMW_BALLOON_2M_ORDER (PMD_SHIFT - PAGE_SHIFT)
+/*
+ * 64-bit targets are only supported in 64-bit
+ */
+#ifdef CONFIG_64BIT
+#define VMW_BALLOON_CAPABILITIES (VMW_BALLOON_CAPABILITIES_COMMON \
+ | VMW_BALLOON_64_BIT_TARGET)
+#else
+#define VMW_BALLOON_CAPABILITIES VMW_BALLOON_CAPABILITIES_COMMON
+#endif
+
enum vmballoon_page_size_type {
VMW_BALLOON_4K_PAGE,
VMW_BALLOON_2M_PAGE,
@@ -563,8 +574,9 @@ static int vmballoon_send_get_target(struct vmballoon *b)
limit = totalram_pages;
- /* Ensure limit fits in 32-bits */
- if (limit != (u32)limit)
+ /* Ensure limit fits in 32-bits if 64-bit targets are not supported */
+ if (!(b->capabilities & VMW_BALLOON_64_BIT_TARGET) &&
+ limit != (u32)limit)
return -EINVAL;
status = vmballoon_cmd(b, VMW_BALLOON_CMD_GET_TARGET, limit, 0);
--
2.17.1
^ permalink raw reply related [flat|nested] 24+ messages in thread* [PATCH v2 19/20] vmw_balloon: memory shrinker
2018-09-20 17:30 [PATCH v2 00/20] vmw_balloon: compaction, shrinker, 64-bit, etc Nadav Amit
` (17 preceding siblings ...)
2018-09-20 17:30 ` [PATCH v2 18/20] vmw_balloon: support 64-bit memory limit Nadav Amit
@ 2018-09-20 17:30 ` Nadav Amit
2018-09-20 17:30 ` [PATCH v2 20/20] vmw_balloon: split refused pages Nadav Amit
2018-09-25 18:15 ` [PATCH v2 00/20] vmw_balloon: compaction, shrinker, 64-bit, etc Greg Kroah-Hartman
20 siblings, 0 replies; 24+ messages in thread
From: Nadav Amit @ 2018-09-20 17:30 UTC (permalink / raw)
To: Greg Kroah-Hartman, Arnd Bergmann
Cc: linux-kernel, Xavier Deguillard, Nadav Amit
Adding a shrinker to the VMware balloon to prevent out-of-memory events.
We reuse the deflate logic for this matter. Deadlocks should not happen,
as no memory allocation is performed while the locks of the
communication (batch/page) and page-list are taken. In the unlikely
event in which the configuration semaphore is taken for write we bail
out and fail gracefully (causing processes to be killed).
Once the shrinker is called, inflation is postponed for few seconds.
The timeout is updated without any lock, but this should not cause any
races, as it is written and read atomically.
Reviewed-by: Xavier Deguillard <xdeguillard@vmware.com>
Signed-off-by: Nadav Amit <namit@vmware.com>
---
drivers/misc/vmw_balloon.c | 124 ++++++++++++++++++++++++++++++++++++-
1 file changed, 122 insertions(+), 2 deletions(-)
diff --git a/drivers/misc/vmw_balloon.c b/drivers/misc/vmw_balloon.c
index 9da8b76342fd..75407be8ddbc 100644
--- a/drivers/misc/vmw_balloon.c
+++ b/drivers/misc/vmw_balloon.c
@@ -42,6 +42,10 @@ MODULE_ALIAS("dmi:*:svnVMware*:*");
MODULE_ALIAS("vmware_vmmemctl");
MODULE_LICENSE("GPL");
+/* Delay in seconds after shrink before inflation. */
+#define VMBALLOON_SHRINK_DELAY (5)
+
+/* Maximum number of refused pages we accumulate during inflation cycle */
#define VMW_BALLOON_MAX_REFUSED 16
/*
@@ -216,12 +220,13 @@ enum vmballoon_stat_general {
VMW_BALLOON_STAT_TIMER,
VMW_BALLOON_STAT_DOORBELL,
VMW_BALLOON_STAT_RESET,
- VMW_BALLOON_STAT_LAST = VMW_BALLOON_STAT_RESET
+ VMW_BALLOON_STAT_SHRINK,
+ VMW_BALLOON_STAT_SHRINK_FREE,
+ VMW_BALLOON_STAT_LAST = VMW_BALLOON_STAT_SHRINK_FREE
};
#define VMW_BALLOON_STAT_NUM (VMW_BALLOON_STAT_LAST + 1)
-
static DEFINE_STATIC_KEY_TRUE(vmw_balloon_batching);
static DEFINE_STATIC_KEY_FALSE(balloon_stat_enabled);
@@ -320,6 +325,15 @@ struct vmballoon {
*/
struct page *page;
+ /**
+ * @shrink_timeout: timeout until the next inflation.
+ *
+ * After an shrink event, indicates the time in jiffies after which
+ * inflation is allowed again. Can be written concurrently with reads,
+ * so must use READ_ONCE/WRITE_ONCE when accessing.
+ */
+ unsigned long shrink_timeout;
+
/* statistics */
struct vmballoon_stats *stats;
@@ -360,6 +374,20 @@ struct vmballoon {
* Lock ordering: @conf_sem -> @comm_lock .
*/
spinlock_t comm_lock;
+
+ /**
+ * @shrinker: shrinker interface that is used to avoid over-inflation.
+ */
+ struct shrinker shrinker;
+
+ /**
+ * @shrinker_registered: whether the shrinker was registered.
+ *
+ * The shrinker interface does not handle gracefully the removal of
+ * shrinker that was not registered before. This indication allows to
+ * simplify the unregistration process.
+ */
+ bool shrinker_registered;
};
static struct vmballoon balloon;
@@ -902,6 +930,10 @@ static int64_t vmballoon_change(struct vmballoon *b)
size - target < vmballoon_page_in_frames(VMW_BALLOON_2M_PAGE))
return 0;
+ /* If an out-of-memory recently occurred, inflation is disallowed. */
+ if (target > size && time_before(jiffies, READ_ONCE(b->shrink_timeout)))
+ return 0;
+
return target - size;
}
@@ -1396,6 +1428,86 @@ static void vmballoon_work(struct work_struct *work)
}
+/**
+ * vmballoon_shrinker_scan() - deflate the balloon due to memory pressure.
+ * @shrinker: pointer to the balloon shrinker.
+ * @sc: page reclaim information.
+ *
+ * Returns: number of pages that were freed during deflation.
+ */
+static unsigned long vmballoon_shrinker_scan(struct shrinker *shrinker,
+ struct shrink_control *sc)
+{
+ struct vmballoon *b = &balloon;
+ unsigned long deflated_frames;
+
+ pr_debug("%s - size: %llu", __func__, atomic64_read(&b->size));
+
+ vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_SHRINK);
+
+ /*
+ * If the lock is also contended for read, we cannot easily reclaim and
+ * we bail out.
+ */
+ if (!down_read_trylock(&b->conf_sem))
+ return 0;
+
+ deflated_frames = vmballoon_deflate(b, sc->nr_to_scan, true);
+
+ vmballoon_stats_gen_add(b, VMW_BALLOON_STAT_SHRINK_FREE,
+ deflated_frames);
+
+ /*
+ * Delay future inflation for some time to mitigate the situations in
+ * which balloon continuously grows and shrinks. Use WRITE_ONCE() since
+ * the access is asynchronous.
+ */
+ WRITE_ONCE(b->shrink_timeout, jiffies + HZ * VMBALLOON_SHRINK_DELAY);
+
+ up_read(&b->conf_sem);
+
+ return deflated_frames;
+}
+
+/**
+ * vmballoon_shrinker_count() - return the number of ballooned pages.
+ * @shrinker: pointer to the balloon shrinker.
+ * @sc: page reclaim information.
+ *
+ * Returns: number of 4k pages that are allocated for the balloon and can
+ * therefore be reclaimed under pressure.
+ */
+static unsigned long vmballoon_shrinker_count(struct shrinker *shrinker,
+ struct shrink_control *sc)
+{
+ struct vmballoon *b = &balloon;
+
+ return atomic64_read(&b->size);
+}
+
+static void vmballoon_unregister_shrinker(struct vmballoon *b)
+{
+ if (b->shrinker_registered)
+ unregister_shrinker(&b->shrinker);
+ b->shrinker_registered = false;
+}
+
+static int vmballoon_register_shrinker(struct vmballoon *b)
+{
+ int r;
+
+ b->shrinker.scan_objects = vmballoon_shrinker_scan;
+ b->shrinker.count_objects = vmballoon_shrinker_count;
+ b->shrinker.seeks = DEFAULT_SEEKS;
+
+ r = register_shrinker(&b->shrinker);
+
+ if (r == 0)
+ b->shrinker_registered = true;
+
+ return r;
+}
+
/*
* DEBUGFS Interface
*/
@@ -1413,6 +1525,8 @@ static const char * const vmballoon_stat_names[] = {
[VMW_BALLOON_STAT_TIMER] = "timer",
[VMW_BALLOON_STAT_DOORBELL] = "doorbell",
[VMW_BALLOON_STAT_RESET] = "reset",
+ [VMW_BALLOON_STAT_SHRINK] = "shrink",
+ [VMW_BALLOON_STAT_SHRINK_FREE] = "shrinkFree"
};
static int vmballoon_enable_stats(struct vmballoon *b)
@@ -1757,6 +1871,10 @@ static int __init vmballoon_init(void)
INIT_DELAYED_WORK(&balloon.dwork, vmballoon_work);
+ error = vmballoon_register_shrinker(&balloon);
+ if (error)
+ goto fail;
+
error = vmballoon_debugfs_init(&balloon);
if (error)
goto fail;
@@ -1782,6 +1900,7 @@ static int __init vmballoon_init(void)
return 0;
fail:
+ vmballoon_unregister_shrinker(&balloon);
vmballoon_compaction_deinit(&balloon);
return error;
}
@@ -1796,6 +1915,7 @@ late_initcall(vmballoon_init);
static void __exit vmballoon_exit(void)
{
+ vmballoon_unregister_shrinker(&balloon);
vmballoon_vmci_cleanup(&balloon);
cancel_delayed_work_sync(&balloon.dwork);
--
2.17.1
^ permalink raw reply related [flat|nested] 24+ messages in thread* [PATCH v2 20/20] vmw_balloon: split refused pages
2018-09-20 17:30 [PATCH v2 00/20] vmw_balloon: compaction, shrinker, 64-bit, etc Nadav Amit
` (18 preceding siblings ...)
2018-09-20 17:30 ` [PATCH v2 19/20] vmw_balloon: memory shrinker Nadav Amit
@ 2018-09-20 17:30 ` Nadav Amit
2018-09-25 18:15 ` [PATCH v2 00/20] vmw_balloon: compaction, shrinker, 64-bit, etc Greg Kroah-Hartman
20 siblings, 0 replies; 24+ messages in thread
From: Nadav Amit @ 2018-09-20 17:30 UTC (permalink / raw)
To: Greg Kroah-Hartman, Arnd Bergmann
Cc: linux-kernel, Xavier Deguillard, Nadav Amit
The hypervisor might refuse to inflate pages. While the balloon driver
handles this scenario correctly, a refusal to inflate a 2MB pages might
cause the same page to be allocated again later just for its inflation
to be refused again. This wastes energy and time.
To avoid this situation we split the 2MB page to 4KB pages, and then try
to inflate each one individually. Most of the 4KB pages out of the 2MB
should be inflated successfully, and we are likely to prevent the
scenario of repeated refused inflation.
Reviewed-by: Xavier Deguillard <xdeguillard@vmware.com>
Signed-off-by: Nadav Amit <namit@vmware.com>
---
drivers/misc/vmw_balloon.c | 63 +++++++++++++++++++++++++++++++-------
1 file changed, 52 insertions(+), 11 deletions(-)
diff --git a/drivers/misc/vmw_balloon.c b/drivers/misc/vmw_balloon.c
index 75407be8ddbc..34fceb82fe85 100644
--- a/drivers/misc/vmw_balloon.c
+++ b/drivers/misc/vmw_balloon.c
@@ -233,6 +233,7 @@ static DEFINE_STATIC_KEY_FALSE(balloon_stat_enabled);
struct vmballoon_ctl {
struct list_head pages;
struct list_head refused_pages;
+ struct list_head prealloc_pages;
unsigned int n_refused_pages;
unsigned int n_pages;
enum vmballoon_page_size_type page_size;
@@ -632,15 +633,25 @@ static int vmballoon_alloc_page_list(struct vmballoon *b,
unsigned int i;
for (i = 0; i < req_n_pages; i++) {
- if (ctl->page_size == VMW_BALLOON_2M_PAGE)
- page = alloc_pages(__GFP_HIGHMEM|__GFP_NOWARN|
+ /*
+ * First check if we happen to have pages that were allocated
+ * before. This happens when 2MB page rejected during inflation
+ * by the hypervisor, and then split into 4KB pages.
+ */
+ if (!list_empty(&ctl->prealloc_pages)) {
+ page = list_first_entry(&ctl->prealloc_pages,
+ struct page, lru);
+ list_del(&page->lru);
+ } else {
+ if (ctl->page_size == VMW_BALLOON_2M_PAGE)
+ page = alloc_pages(__GFP_HIGHMEM|__GFP_NOWARN|
__GFP_NOMEMALLOC, VMW_BALLOON_2M_ORDER);
- else
- page = balloon_page_alloc();
+ else
+ page = balloon_page_alloc();
- /* Update statistics */
- vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_ALLOC,
- ctl->page_size);
+ vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_ALLOC,
+ ctl->page_size);
+ }
if (page) {
/* Success. Add the page to the list and continue. */
@@ -884,7 +895,8 @@ static void vmballoon_release_page_list(struct list_head *page_list,
__free_pages(page, vmballoon_page_order(page_size));
}
- *n_pages = 0;
+ if (n_pages)
+ *n_pages = 0;
}
@@ -1016,6 +1028,32 @@ static void vmballoon_dequeue_page_list(struct vmballoon *b,
*n_pages = i;
}
+/**
+ * vmballoon_split_refused_pages() - Split the 2MB refused pages to 4k.
+ *
+ * If inflation of 2MB pages was denied by the hypervisor, it is likely to be
+ * due to one or few 4KB pages. These 2MB pages may keep being allocated and
+ * then being refused. To prevent this case, this function splits the refused
+ * pages into 4KB pages and adds them into @prealloc_pages list.
+ *
+ * @ctl: pointer for the %struct vmballoon_ctl, which defines the operation.
+ */
+static void vmballoon_split_refused_pages(struct vmballoon_ctl *ctl)
+{
+ struct page *page, *tmp;
+ unsigned int i, order;
+
+ order = vmballoon_page_order(ctl->page_size);
+
+ list_for_each_entry_safe(page, tmp, &ctl->refused_pages, lru) {
+ list_del(&page->lru);
+ split_page(page, order);
+ for (i = 0; i < (1 << order); i++)
+ list_add(&page[i].lru, &ctl->prealloc_pages);
+ }
+ ctl->n_refused_pages = 0;
+}
+
/**
* vmballoon_inflate() - Inflate the balloon towards its target size.
*
@@ -1027,6 +1065,7 @@ static void vmballoon_inflate(struct vmballoon *b)
struct vmballoon_ctl ctl = {
.pages = LIST_HEAD_INIT(ctl.pages),
.refused_pages = LIST_HEAD_INIT(ctl.refused_pages),
+ .prealloc_pages = LIST_HEAD_INIT(ctl.prealloc_pages),
.page_size = b->max_page_size,
.op = VMW_BALLOON_INFLATE
};
@@ -1074,10 +1113,10 @@ static void vmballoon_inflate(struct vmballoon *b)
break;
/*
- * Ignore errors from locking as we now switch to 4k
- * pages and we might get different errors.
+ * Split the refused pages to 4k. This will also empty
+ * the refused pages list.
*/
- vmballoon_release_refused_pages(b, &ctl);
+ vmballoon_split_refused_pages(&ctl);
ctl.page_size--;
}
@@ -1091,6 +1130,8 @@ static void vmballoon_inflate(struct vmballoon *b)
*/
if (ctl.n_refused_pages != 0)
vmballoon_release_refused_pages(b, &ctl);
+
+ vmballoon_release_page_list(&ctl.prealloc_pages, NULL, ctl.page_size);
}
/**
--
2.17.1
^ permalink raw reply related [flat|nested] 24+ messages in thread* Re: [PATCH v2 00/20] vmw_balloon: compaction, shrinker, 64-bit, etc.
2018-09-20 17:30 [PATCH v2 00/20] vmw_balloon: compaction, shrinker, 64-bit, etc Nadav Amit
` (19 preceding siblings ...)
2018-09-20 17:30 ` [PATCH v2 20/20] vmw_balloon: split refused pages Nadav Amit
@ 2018-09-25 18:15 ` Greg Kroah-Hartman
2018-09-25 19:55 ` Nadav Amit
20 siblings, 1 reply; 24+ messages in thread
From: Greg Kroah-Hartman @ 2018-09-25 18:15 UTC (permalink / raw)
To: Nadav Amit
Cc: Arnd Bergmann, linux-kernel, Xavier Deguillard,
Michael S. Tsirkin, Jason Wang, linux-mm, virtualization
On Thu, Sep 20, 2018 at 10:30:06AM -0700, Nadav Amit wrote:
> This patch-set adds the following enhancements to the VMware balloon
> driver:
>
> 1. Balloon compaction support.
> 2. Report the number of inflated/deflated ballooned pages through vmstat.
> 3. Memory shrinker to avoid balloon over-inflation (and OOM).
> 4. Support VMs with memory limit that is greater than 16TB.
> 5. Faster and more aggressive inflation.
>
> To support compaction we wish to use the existing infrastructure.
> However, we need to make slight adaptions for it. We add a new list
> interface to balloon-compaction, which is more generic and efficient,
> since it does not require as many IRQ save/restore operations. We leave
> the old interface that is used by the virtio balloon.
>
> Big parts of this patch-set are cleanup and documentation. Patches 1-13
> simplify the balloon code, document its behavior and allow the balloon
> code to run concurrently. The support for concurrency is required for
> compaction and the shrinker interface.
>
> For documentation we use the kernel-doc format. We are aware that the
> balloon interface is not public, but following the kernel-doc format may
> be useful one day.
I applied the first 16 patches. Please fix up 17 and resend.
thanks,
greg k-h
^ permalink raw reply [flat|nested] 24+ messages in thread* Re: [PATCH v2 00/20] vmw_balloon: compaction, shrinker, 64-bit, etc.
2018-09-25 18:15 ` [PATCH v2 00/20] vmw_balloon: compaction, shrinker, 64-bit, etc Greg Kroah-Hartman
@ 2018-09-25 19:55 ` Nadav Amit
0 siblings, 0 replies; 24+ messages in thread
From: Nadav Amit @ 2018-09-25 19:55 UTC (permalink / raw)
To: Greg Kroah-Hartman
Cc: Arnd Bergmann, linux-kernel, Xavier Deguillard,
Michael S. Tsirkin, Jason Wang, linux-mm, virtualization
at 11:15 AM, Greg Kroah-Hartman <gregkh@linuxfoundation.org> wrote:
> On Thu, Sep 20, 2018 at 10:30:06AM -0700, Nadav Amit wrote:
>> This patch-set adds the following enhancements to the VMware balloon
>> driver:
>>
>> 1. Balloon compaction support.
>> 2. Report the number of inflated/deflated ballooned pages through vmstat.
>> 3. Memory shrinker to avoid balloon over-inflation (and OOM).
>> 4. Support VMs with memory limit that is greater than 16TB.
>> 5. Faster and more aggressive inflation.
>>
>> To support compaction we wish to use the existing infrastructure.
>> However, we need to make slight adaptions for it. We add a new list
>> interface to balloon-compaction, which is more generic and efficient,
>> since it does not require as many IRQ save/restore operations. We leave
>> the old interface that is used by the virtio balloon.
>>
>> Big parts of this patch-set are cleanup and documentation. Patches 1-13
>> simplify the balloon code, document its behavior and allow the balloon
>> code to run concurrently. The support for concurrency is required for
>> compaction and the shrinker interface.
>>
>> For documentation we use the kernel-doc format. We are aware that the
>> balloon interface is not public, but following the kernel-doc format may
>> be useful one day.
>
> I applied the first 16 patches. Please fix up 17 and resend.
Thanks. I will update it and resend later today.
Regards,
Nadav
^ permalink raw reply [flat|nested] 24+ messages in thread