[PATCH v2] x86, vdso32: fix out of memory handling setup vDSO

[PATCH v2] x86, vdso32: fix out of memory handling setup vDSO

[Stefani Seibold]

This patch add a correct out of memory handling for setup a 32 bit vDSO.

The patch is against tip commit 4e40112c4ff6a577dd06d92b2a54cdf06265bf74

Signed-off-by: Stefani Seibold <stefani@seibold.net>
---
 arch/x86/vdso/vdso32-setup.c | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/arch/x86/vdso/vdso32-setup.c b/arch/x86/vdso/vdso32-setup.c
index 0bc363a..b77fdb2 100644
--- a/arch/x86/vdso/vdso32-setup.c
+++ b/arch/x86/vdso/vdso32-setup.c
@@ -134,8 +134,14 @@ int __init sysenter_setup(void)
 	}
 
 	vdso32_size = (vdso_len + PAGE_SIZE - 1) / PAGE_SIZE;
-	vdso32_pages = kmalloc(sizeof(*vdso32_pages) * vdso32_size, GFP_ATOMIC); 
-	vdso_pages = kmalloc(VDSO_OFFSET(vdso32_size), GFP_ATOMIC);
+
+	vdso32_pages = kmalloc(sizeof(*vdso32_pages) * vdso32_size, GFP_KERNEL);
+	if (!vdso32_pages)
+		goto fail;
+
+	vdso_pages = kmalloc(VDSO_OFFSET(vdso32_size), GFP_KERNEL);
+	if (!vdso_pages)
+		goto fail;
 
 	for(i = 0; i != vdso32_size; ++i)
 		vdso32_pages[i] = virt_to_page(vdso_pages + VDSO_OFFSET(i));
@@ -144,6 +150,12 @@ int __init sysenter_setup(void)
 	patch_vdso32(vdso_pages, vdso_len);
 
 	return 0;
+fail:
+	kfree(vdso32_pages);
+	kfree(vdso_pages);
+	vdso32_size = 0;
+
+	return -ENOMEM;
 }
 
 /* Setup a VMA at program startup for the vsyscall page */
@@ -162,6 +174,9 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
 	if (vdso_enabled != 1)  /* Other values all mean "disabled" */
 		return 0;
 
+	if (!vdso32_size)
+		return 0;
+
 	down_write(&mm->mmap_sem);
 
 	addr = get_unmapped_area(NULL, 0, PAGE_SIZE, 0, 0);
-- 
1.9.0