From: Alexander Potapenko <glider@google.com>
To: glider@google.com
Cc: Alexander Viro <viro@zeniv.linux.org.uk>,
Andrew Morton <akpm@linux-foundation.org>,
Andrey Konovalov <andreyknvl@google.com>,
Andy Lutomirski <luto@kernel.org>, Arnd Bergmann <arnd@arndb.de>,
Borislav Petkov <bp@alien8.de>, Christoph Hellwig <hch@lst.de>,
Christoph Lameter <cl@linux.com>,
David Rientjes <rientjes@google.com>,
Dmitry Vyukov <dvyukov@google.com>,
Eric Dumazet <edumazet@google.com>,
Greg Kroah-Hartman <gregkh@linuxfoundation.org>,
Herbert Xu <herbert@gondor.apana.org.au>,
Ilya Leoshkevich <iii@linux.ibm.com>,
Ingo Molnar <mingo@redhat.com>, Jens Axboe <axboe@kernel.dk>,
Joonsoo Kim <iamjoonsoo.kim@lge.com>,
Kees Cook <keescook@chromium.org>,
Marco Elver <elver@google.com>,
Mark Rutland <mark.rutland@arm.com>,
Matthew Wilcox <willy@infradead.org>,
"Michael S. Tsirkin" <mst@redhat.com>,
Pekka Enberg <penberg@kernel.org>,
Peter Zijlstra <peterz@infradead.org>,
Petr Mladek <pmladek@suse.com>,
Steven Rostedt <rostedt@goodmis.org>,
Thomas Gleixner <tglx@linutronix.de>,
Vasily Gorbik <gor@linux.ibm.com>,
Vegard Nossum <vegard.nossum@oracle.com>,
Vlastimil Babka <vbabka@suse.cz>,
linux-mm@kvack.org, linux-arch@vger.kernel.org,
linux-kernel@vger.kernel.org
Subject: [PATCH v2 18/48] kmsan: mm: call KMSAN hooks from SLUB code
Date: Tue, 29 Mar 2022 14:39:47 +0200 [thread overview]
Message-ID: <20220329124017.737571-19-glider@google.com> (raw)
In-Reply-To: <20220329124017.737571-1-glider@google.com>
In order to report uninitialized memory coming from heap allocations
KMSAN has to poison them unless they're created with __GFP_ZERO.
It's handy that we need KMSAN hooks in the places where
init_on_alloc/init_on_free initialization is performed.
Signed-off-by: Alexander Potapenko <glider@google.com>
---
v2:
-- move the implementation of SLUB hooks here
Link: https://linux-review.googlesource.com/id/I6954b386c5c5d7f99f48bb6cbcc74b75136ce86e
---
include/linux/kmsan.h | 57 ++++++++++++++++++++++++++++++
mm/kmsan/hooks.c | 80 +++++++++++++++++++++++++++++++++++++++++++
mm/slab.h | 1 +
mm/slub.c | 21 ++++++++++--
4 files changed, 157 insertions(+), 2 deletions(-)
diff --git a/include/linux/kmsan.h b/include/linux/kmsan.h
index da41850b46cbd..ed3630068e2ef 100644
--- a/include/linux/kmsan.h
+++ b/include/linux/kmsan.h
@@ -16,6 +16,7 @@
#include <linux/vmalloc.h>
struct page;
+struct kmem_cache;
#ifdef CONFIG_KMSAN
@@ -73,6 +74,44 @@ void kmsan_free_page(struct page *page, unsigned int order);
*/
void kmsan_copy_page_meta(struct page *dst, struct page *src);
+/**
+ * kmsan_slab_alloc() - Notify KMSAN about a slab allocation.
+ * @s: slab cache the object belongs to.
+ * @object: object pointer.
+ * @flags: GFP flags passed to the allocator.
+ *
+ * Depending on cache flags and GFP flags, KMSAN sets up the metadata of the
+ * newly created object, marking it as initialized or uninitialized.
+ */
+void kmsan_slab_alloc(struct kmem_cache *s, void *object, gfp_t flags);
+
+/**
+ * kmsan_slab_free() - Notify KMSAN about a slab deallocation.
+ * @s: slab cache the object belongs to.
+ * @object: object pointer.
+ *
+ * KMSAN marks the freed object as uninitialized.
+ */
+void kmsan_slab_free(struct kmem_cache *s, void *object);
+
+/**
+ * kmsan_kmalloc_large() - Notify KMSAN about a large slab allocation.
+ * @ptr: object pointer.
+ * @size: object size.
+ * @flags: GFP flags passed to the allocator.
+ *
+ * Similar to kmsan_slab_alloc(), but for large allocations.
+ */
+void kmsan_kmalloc_large(const void *ptr, size_t size, gfp_t flags);
+
+/**
+ * kmsan_kfree_large() - Notify KMSAN about a large slab deallocation.
+ * @ptr: object pointer.
+ *
+ * Similar to kmsan_slab_free(), but for large allocations.
+ */
+void kmsan_kfree_large(const void *ptr);
+
/**
* kmsan_map_kernel_range_noflush() - Notify KMSAN about a vmap.
* @start: start of vmapped range.
@@ -139,6 +178,24 @@ static inline void kmsan_copy_page_meta(struct page *dst, struct page *src)
{
}
+static inline void kmsan_slab_alloc(struct kmem_cache *s, void *object,
+ gfp_t flags)
+{
+}
+
+static inline void kmsan_slab_free(struct kmem_cache *s, void *object)
+{
+}
+
+static inline void kmsan_kmalloc_large(const void *ptr, size_t size,
+ gfp_t flags)
+{
+}
+
+static inline void kmsan_kfree_large(const void *ptr)
+{
+}
+
static inline void kmsan_vmap_pages_range_noflush(unsigned long start,
unsigned long end,
pgprot_t prot,
diff --git a/mm/kmsan/hooks.c b/mm/kmsan/hooks.c
index 5d886df57adca..e7c3ff48ed5cd 100644
--- a/mm/kmsan/hooks.c
+++ b/mm/kmsan/hooks.c
@@ -26,6 +26,86 @@
* skipping effects of functions like memset() inside instrumented code.
*/
+void kmsan_slab_alloc(struct kmem_cache *s, void *object, gfp_t flags)
+{
+ if (unlikely(object == NULL))
+ return;
+ if (!kmsan_enabled || kmsan_in_runtime())
+ return;
+ /*
+ * There's a ctor or this is an RCU cache - do nothing. The memory
+ * status hasn't changed since last use.
+ */
+ if (s->ctor || (s->flags & SLAB_TYPESAFE_BY_RCU))
+ return;
+
+ kmsan_enter_runtime();
+ if (flags & __GFP_ZERO)
+ kmsan_internal_unpoison_memory(object, s->object_size,
+ KMSAN_POISON_CHECK);
+ else
+ kmsan_internal_poison_memory(object, s->object_size, flags,
+ KMSAN_POISON_CHECK);
+ kmsan_leave_runtime();
+}
+EXPORT_SYMBOL(kmsan_slab_alloc);
+
+void kmsan_slab_free(struct kmem_cache *s, void *object)
+{
+ if (!kmsan_enabled || kmsan_in_runtime())
+ return;
+
+ /* RCU slabs could be legally used after free within the RCU period */
+ if (unlikely(s->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON)))
+ return;
+ /*
+ * If there's a constructor, freed memory must remain in the same state
+ * until the next allocation. We cannot save its state to detect
+ * use-after-free bugs, instead we just keep it unpoisoned.
+ */
+ if (s->ctor)
+ return;
+ kmsan_enter_runtime();
+ kmsan_internal_poison_memory(object, s->object_size, GFP_KERNEL,
+ KMSAN_POISON_CHECK | KMSAN_POISON_FREE);
+ kmsan_leave_runtime();
+}
+EXPORT_SYMBOL(kmsan_slab_free);
+
+void kmsan_kmalloc_large(const void *ptr, size_t size, gfp_t flags)
+{
+ if (unlikely(ptr == NULL))
+ return;
+ if (!kmsan_enabled || kmsan_in_runtime())
+ return;
+ kmsan_enter_runtime();
+ if (flags & __GFP_ZERO)
+ kmsan_internal_unpoison_memory((void *)ptr, size,
+ /*checked*/ true);
+ else
+ kmsan_internal_poison_memory((void *)ptr, size, flags,
+ KMSAN_POISON_CHECK);
+ kmsan_leave_runtime();
+}
+EXPORT_SYMBOL(kmsan_kmalloc_large);
+
+void kmsan_kfree_large(const void *ptr)
+{
+ struct page *page;
+
+ if (!kmsan_enabled || kmsan_in_runtime())
+ return;
+ kmsan_enter_runtime();
+ page = virt_to_head_page((void *)ptr);
+ KMSAN_WARN_ON(ptr != page_address(page));
+ kmsan_internal_poison_memory((void *)ptr,
+ PAGE_SIZE << compound_order(page),
+ GFP_KERNEL,
+ KMSAN_POISON_CHECK | KMSAN_POISON_FREE);
+ kmsan_leave_runtime();
+}
+EXPORT_SYMBOL(kmsan_kfree_large);
+
static unsigned long vmalloc_shadow(unsigned long addr)
{
return (unsigned long)kmsan_get_metadata((void *)addr,
diff --git a/mm/slab.h b/mm/slab.h
index c7f2abc2b154c..c2538d856ec45 100644
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -734,6 +734,7 @@ static inline void slab_post_alloc_hook(struct kmem_cache *s,
memset(p[i], 0, s->object_size);
kmemleak_alloc_recursive(p[i], s->object_size, 1,
s->flags, flags);
+ kmsan_slab_alloc(s, p[i], flags);
}
memcg_slab_post_alloc_hook(s, objcg, flags, size, p);
diff --git a/mm/slub.c b/mm/slub.c
index 261474092e43e..9b266f6b384b9 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -22,6 +22,7 @@
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <linux/kasan.h>
+#include <linux/kmsan.h>
#include <linux/cpu.h>
#include <linux/cpuset.h>
#include <linux/mempolicy.h>
@@ -357,18 +358,28 @@ static void prefetch_freepointer(const struct kmem_cache *s, void *object)
prefetchw(object + s->offset);
}
+/*
+ * When running under KMSAN, get_freepointer_safe() may return an uninitialized
+ * pointer value in the case the current thread loses the race for the next
+ * memory chunk in the freelist. In that case this_cpu_cmpxchg_double() in
+ * slab_alloc_node() will fail, so the uninitialized value won't be used, but
+ * KMSAN will still check all arguments of cmpxchg because of imperfect
+ * handling of inline assembly.
+ * To work around this problem, use kmsan_init() to force initialize the
+ * return value of get_freepointer_safe().
+ */
static inline void *get_freepointer_safe(struct kmem_cache *s, void *object)
{
unsigned long freepointer_addr;
void *p;
if (!debug_pagealloc_enabled_static())
- return get_freepointer(s, object);
+ return kmsan_init(get_freepointer(s, object));
object = kasan_reset_tag(object);
freepointer_addr = (unsigned long)object + s->offset;
copy_from_kernel_nofault(&p, (void **)freepointer_addr, sizeof(p));
- return freelist_ptr(s, p, freepointer_addr);
+ return kmsan_init(freelist_ptr(s, p, freepointer_addr));
}
static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
@@ -1683,6 +1694,7 @@ static inline void *kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags)
ptr = kasan_kmalloc_large(ptr, size, flags);
/* As ptr might get tagged, call kmemleak hook after KASAN. */
kmemleak_alloc(ptr, size, 1, flags);
+ kmsan_kmalloc_large(ptr, size, flags);
return ptr;
}
@@ -1690,12 +1702,14 @@ static __always_inline void kfree_hook(void *x)
{
kmemleak_free(x);
kasan_kfree_large(x);
+ kmsan_kfree_large(x);
}
static __always_inline bool slab_free_hook(struct kmem_cache *s,
void *x, bool init)
{
kmemleak_free_recursive(x, s->flags);
+ kmsan_slab_free(s, x);
debug_check_no_locks_freed(x, s->object_size);
@@ -3729,6 +3743,7 @@ int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
*/
slab_post_alloc_hook(s, objcg, flags, size, p,
slab_want_init_on_alloc(flags, s));
+
return i;
error:
slub_put_cpu_ptr(s->cpu_slab);
@@ -5910,6 +5925,7 @@ static char *create_unique_id(struct kmem_cache *s)
p += sprintf(p, "%07u", s->size);
BUG_ON(p > name + ID_STR_LENGTH - 1);
+ kmsan_unpoison_memory(name, p - name);
return name;
}
@@ -6011,6 +6027,7 @@ static int sysfs_slab_alias(struct kmem_cache *s, const char *name)
al->name = name;
al->next = alias_list;
alias_list = al;
+ kmsan_unpoison_memory(al, sizeof(struct saved_alias));
return 0;
}
--
2.35.1.1021.g381101b075-goog
next prev parent reply other threads:[~2022-03-29 12:41 UTC|newest]
Thread overview: 57+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-03-29 12:39 [PATCH v2 00/48] Add KernelMemorySanitizer infrastructure Alexander Potapenko
2022-03-29 12:39 ` [PATCH v2 01/48] x86: add missing include to sparsemem.h Alexander Potapenko
2022-03-29 12:39 ` [PATCH v2 02/48] stackdepot: reserve 5 extra bits in depot_stack_handle_t Alexander Potapenko
2022-03-29 12:39 ` [PATCH v2 03/48] kasan: common: adapt to the new prototype of __stack_depot_save() Alexander Potapenko
2022-03-29 12:39 ` [PATCH v2 04/48] instrumented.h: allow instrumenting both sides of copy_from_user() Alexander Potapenko
2022-03-29 12:39 ` [PATCH v2 05/48] x86: asm: instrument usercopy in get_user() and __put_user_size() Alexander Potapenko
2022-03-29 12:39 ` [PATCH v2 06/48] asm-generic: instrument usercopy in cacheflush.h Alexander Potapenko
2022-03-29 12:39 ` [PATCH v2 07/48] kmsan: add ReST documentation Alexander Potapenko
2022-03-29 12:39 ` [PATCH v2 08/48] kmsan: introduce __no_sanitize_memory and __no_kmsan_checks Alexander Potapenko
2022-03-29 12:39 ` [PATCH v2 09/48] kmsan: mark noinstr as __no_sanitize_memory Alexander Potapenko
2022-03-29 12:39 ` [PATCH v2 10/48] x86: kmsan: pgtable: reduce vmalloc space Alexander Potapenko
2022-03-29 12:39 ` [PATCH v2 11/48] libnvdimm/pfn_dev: increase MAX_STRUCT_PAGE_SIZE Alexander Potapenko
2022-03-29 12:39 ` [PATCH v2 12/48] kcsan: clang: retire CONFIG_KCSAN_KCOV_BROKEN Alexander Potapenko
2022-03-30 6:00 ` Marco Elver
2022-03-29 12:39 ` [PATCH v2 13/48] kmsan: add KMSAN runtime core Alexander Potapenko
2022-03-30 8:58 ` Peter Zijlstra
2022-03-30 9:41 ` Peter Zijlstra
2022-04-04 14:39 ` Alexander Potapenko
2022-03-29 12:39 ` [PATCH v2 14/48] kmsan: implement kmsan_init(), initialize READ_ONCE_NOCHECK() Alexander Potapenko
2022-03-29 12:39 ` [PATCH v2 15/48] kmsan: disable instrumentation of unsupported common kernel code Alexander Potapenko
2022-03-29 12:39 ` [PATCH v2 16/48] MAINTAINERS: add entry for KMSAN Alexander Potapenko
2022-03-29 12:39 ` [PATCH v2 17/48] kmsan: mm: maintain KMSAN metadata for page operations Alexander Potapenko
2022-03-29 12:39 ` Alexander Potapenko [this message]
2022-03-29 12:39 ` [PATCH v2 19/48] kmsan: handle task creation and exiting Alexander Potapenko
2022-03-29 12:39 ` [PATCH v2 20/48] kmsan: init: call KMSAN initialization routines Alexander Potapenko
2022-03-29 12:39 ` [PATCH v2 21/48] instrumented.h: add KMSAN support Alexander Potapenko
2022-03-29 12:39 ` [PATCH v2 22/48] kmsan: unpoison @tlb in arch_tlb_gather_mmu() Alexander Potapenko
2022-03-29 12:39 ` [PATCH v2 23/48] kmsan: add iomap support Alexander Potapenko
2022-03-29 12:39 ` [PATCH v2 24/48] Input: libps2: mark data received in __ps2_command() as initialized Alexander Potapenko
2022-03-29 12:39 ` [PATCH v2 25/48] kmsan: dma: unpoison DMA mappings Alexander Potapenko
2022-03-29 12:39 ` [PATCH v2 26/48] kmsan: virtio: check/unpoison scatterlist in vring_map_one_sg() Alexander Potapenko
2022-03-29 12:39 ` [PATCH v2 27/48] kmsan: handle memory sent to/from USB Alexander Potapenko
2022-03-29 12:39 ` [PATCH v2 28/48] kmsan: instrumentation.h: add instrumentation_begin_with_regs() Alexander Potapenko
2022-03-29 12:39 ` [PATCH v2 29/48] kmsan: entry: handle register passing from uninstrumented code Alexander Potapenko
2022-03-29 12:39 ` [PATCH v2 30/48] kmsan: add tests for KMSAN Alexander Potapenko
2022-03-29 12:40 ` [PATCH v2 31/48] kernel: kmsan: don't instrument stacktrace.c Alexander Potapenko
2022-04-04 14:56 ` Alexander Potapenko
2022-03-29 12:40 ` [PATCH v2 32/48] kmsan: disable strscpy() optimization under KMSAN Alexander Potapenko
2022-03-29 12:40 ` [PATCH v2 33/48] crypto: kmsan: disable accelerated configs " Alexander Potapenko
2022-03-29 12:40 ` [PATCH v2 34/48] kmsan: disable physical page merging in biovec Alexander Potapenko
2022-03-29 12:40 ` [PATCH v2 35/48] kmsan: block: skip bio block merging logic for KMSAN Alexander Potapenko
2022-03-29 12:40 ` [PATCH v2 36/48] kmsan: kcov: unpoison area->list in kcov_remote_area_put() Alexander Potapenko
2022-03-29 12:40 ` [PATCH v2 37/48] security: kmsan: fix interoperability with auto-initialization Alexander Potapenko
2022-03-29 12:40 ` [PATCH v2 38/48] objtool: kmsan: list KMSAN API functions as uaccess-safe Alexander Potapenko
2022-03-30 8:46 ` Peter Zijlstra
2022-04-14 15:30 ` Alexander Potapenko
2022-04-14 15:38 ` Peter Zijlstra
2022-03-29 12:40 ` [PATCH v2 39/48] x86: kmsan: make READ_ONCE_TASK_STACK() return initialized values Alexander Potapenko
2022-03-29 12:40 ` [PATCH v2 40/48] x86: kmsan: disable instrumentation of unsupported code Alexander Potapenko
2022-03-29 12:40 ` [PATCH v2 41/48] x86: kmsan: skip shadow checks in __switch_to() Alexander Potapenko
2022-03-29 12:40 ` [PATCH v2 42/48] x86: kmsan: handle open-coded assembly in lib/iomem.c Alexander Potapenko
2022-03-29 12:40 ` [PATCH v2 43/48] x86: kmsan: use __msan_ string functions where possible Alexander Potapenko
2022-03-29 12:40 ` [PATCH v2 44/48] x86: kmsan: sync metadata pages on page fault Alexander Potapenko
2022-03-29 12:40 ` [PATCH v2 45/48] x86: kasan: kmsan: support CONFIG_GENERIC_CSUM on x86, enable it for KASAN/KMSAN Alexander Potapenko
2022-03-29 12:40 ` [PATCH v2 46/48] x86: fs: kmsan: disable CONFIG_DCACHE_WORD_ACCESS Alexander Potapenko
2022-03-29 12:40 ` [PATCH v2 47/48] x86: kmsan: handle register passing from uninstrumented code Alexander Potapenko
2022-03-29 12:40 ` [PATCH v2 48/48] x86: kmsan: enable KMSAN builds for x86 Alexander Potapenko
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20220329124017.737571-19-glider@google.com \
--to=glider@google.com \
--cc=akpm@linux-foundation.org \
--cc=andreyknvl@google.com \
--cc=arnd@arndb.de \
--cc=axboe@kernel.dk \
--cc=bp@alien8.de \
--cc=cl@linux.com \
--cc=dvyukov@google.com \
--cc=edumazet@google.com \
--cc=elver@google.com \
--cc=gor@linux.ibm.com \
--cc=gregkh@linuxfoundation.org \
--cc=hch@lst.de \
--cc=herbert@gondor.apana.org.au \
--cc=iamjoonsoo.kim@lge.com \
--cc=iii@linux.ibm.com \
--cc=keescook@chromium.org \
--cc=linux-arch@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=luto@kernel.org \
--cc=mark.rutland@arm.com \
--cc=mingo@redhat.com \
--cc=mst@redhat.com \
--cc=penberg@kernel.org \
--cc=peterz@infradead.org \
--cc=pmladek@suse.com \
--cc=rientjes@google.com \
--cc=rostedt@goodmis.org \
--cc=tglx@linutronix.de \
--cc=vbabka@suse.cz \
--cc=vegard.nossum@oracle.com \
--cc=viro@zeniv.linux.org.uk \
--cc=willy@infradead.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).