[PATCH RFC v4 09/42] kmsan: add KMSAN runtime core

[glider@google.com]

This patch adds the core parts of KMSAN runtime and associated files:

  - include/linux/kmsan-checks.h: user API to poison/unpoison/check memory
  - include/linux/kmsan.h: declarations of KMSAN memory hooks to be
    referenced outside KMSAN runtime
  - lib/Kconfig.kmsan: declarations for CONFIG_KMSAN and CONFIG_TEST_KMSAN
  - mm/kmsan/Makefile: boilerplate Makefile
  - mm/kmsan/kmsan.h: internal KMSAN declarations
  - mm/kmsan/kmsan.c: core functions that operate with shadow and
    origin memory and perform checks, utility functions
  - mm/kmsan/kmsan_init.c: KMSAN initialization routines
  - scripts/Makefile.kmsan: CFLAGS_KMSAN

The patch also adds the necessary bookkeeping bits to struct page and
struct task_struct:
 - each struct page now contains pointers to two struct pages holding
   KMSAN metadata (shadow and origins) for the original struct page;
 - each task_struct contains a struct kmsan_task_state used to track
   the metadata of function parameters and return values for that task.

Signed-off-by: Alexander Potapenko <glider@google.com>
To: Alexander Potapenko <glider@google.com>
Cc: Jens Axboe <axboe@kernel.dk>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Wolfram Sang <wsa@the-dreams.de>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Vegard Nossum <vegard.nossum@oracle.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Marco Elver <elver@google.com>
Cc: Andrey Konovalov <andreyknvl@google.com>
Cc: linux-mm@kvack.org

---
v2:
 - dropped kmsan_handle_vprintk()
 - use locking for single kmsan_pr_err() calls
 - don't try to understand we're inside printk()
v3:
 - fix an endless loop in __msan_poison_alloca()
 - implement kmsan_handle_dma()
 - dropped kmsan_handle_i2c_transfer()
 - fixed compilation with UNWINDER_ORC
 - dropped assembly hooks for system calls
v4:
 - splitted away some runtime parts to ease the review process
 - fix a lot of comments by Marco Elver and Andrey Konovalov:
 -- clean up headers and #defines, remove debugging code
 -- dropped kmsan_pr_* macros, fixed reporting code
 -- removed TODOs
 -- simplified kmsan_get_shadow_origin_ptr()
 - actually filter out IRQ frames using filter_irq_stacks()
 - simplify kmsan_get_metadata()
 - include build_bug.h into kmsan-checks.h
 - don't instrument KMSAN files with stackprotector
 - squashed "kmsan: add KMSAN bits to struct page and struct
   task_struct" into this patch as requested by Marco Elver

Change-Id: I4b3a7aba6d5804afac4f5f7274cadf8675b6e119
---
 arch/x86/Kconfig             |   1 +
 include/linux/kmsan-checks.h | 127 ++++++++
 include/linux/kmsan.h        | 335 +++++++++++++++++++++
 include/linux/mm_types.h     |   9 +
 include/linux/sched.h        |   5 +
 lib/Kconfig.debug            |   2 +
 lib/Kconfig.kmsan            |  22 ++
 mm/kmsan/Makefile            |  11 +
 mm/kmsan/kmsan.c             | 547 +++++++++++++++++++++++++++++++++++
 mm/kmsan/kmsan.h             | 161 +++++++++++
 mm/kmsan/kmsan_init.c        |  79 +++++
 mm/kmsan/kmsan_report.c      | 143 +++++++++
 mm/kmsan/kmsan_shadow.c      | 456 +++++++++++++++++++++++++++++
 mm/kmsan/kmsan_shadow.h      |  30 ++
 scripts/Makefile.kmsan       |  12 +
 15 files changed, 1940 insertions(+)
 create mode 100644 include/linux/kmsan-checks.h
 create mode 100644 include/linux/kmsan.h
 create mode 100644 lib/Kconfig.kmsan
 create mode 100644 mm/kmsan/Makefile
 create mode 100644 mm/kmsan/kmsan.c
 create mode 100644 mm/kmsan/kmsan.h
 create mode 100644 mm/kmsan/kmsan_init.c
 create mode 100644 mm/kmsan/kmsan_report.c
 create mode 100644 mm/kmsan/kmsan_shadow.c
 create mode 100644 mm/kmsan/kmsan_shadow.h
 create mode 100644 scripts/Makefile.kmsan

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 5e8949953660..8081bc31d5f0 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -135,6 +135,7 @@ config X86
 	select HAVE_ARCH_JUMP_LABEL_RELATIVE
 	select HAVE_ARCH_KASAN			if X86_64
 	select HAVE_ARCH_KASAN_VMALLOC		if X86_64
+	select HAVE_ARCH_KMSAN			if X86_64
 	select HAVE_ARCH_KGDB
 	select HAVE_ARCH_MMAP_RND_BITS		if MMU
 	select HAVE_ARCH_MMAP_RND_COMPAT_BITS	if MMU && COMPAT
diff --git a/include/linux/kmsan-checks.h b/include/linux/kmsan-checks.h
new file mode 100644
index 000000000000..2e4b8001e8d9
--- /dev/null
+++ b/include/linux/kmsan-checks.h
@@ -0,0 +1,127 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * KMSAN checks to be used for one-off annotations in subsystems.
+ *
+ * Copyright (C) 2017-2019 Google LLC
+ * Author: Alexander Potapenko <glider@google.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+#ifndef _LINUX_KMSAN_CHECKS_H
+#define _LINUX_KMSAN_CHECKS_H
+
+#include <linux/build_bug.h>
+#include <linux/types.h>
+
+#ifdef CONFIG_KMSAN
+
+/*
+ * Helper functions that mark the return value initialized.
+ * Note that Clang ignores the inline attribute in the cases when a no_sanitize
+ * function is called from an instrumented one. For the same reason these
+ * functions may not be declared __always_inline - in that case they dissolve in
+ * the callers and KMSAN won't be able to notice they should not be
+ * instrumented.
+ */
+
+__no_sanitize_memory
+static inline u8 KMSAN_INIT_1(u8 value)
+{
+	return value;
+}
+
+__no_sanitize_memory
+static inline u16 KMSAN_INIT_2(u16 value)
+{
+	return value;
+}
+
+__no_sanitize_memory
+static inline u32 KMSAN_INIT_4(u32 value)
+{
+	return value;
+}
+
+__no_sanitize_memory
+static inline u64 KMSAN_INIT_8(u64 value)
+{
+	return value;
+}
+
+/**
+ * KMSAN_INIT_VALUE - Make the value initialized.
+ * @val: 1-, 2-, 4- or 8-byte integer that may be treated as uninitialized by
+ *       KMSAN's.
+ *
+ * Return: value of @val that KMSAN treats as initialized.
+ */
+#define KMSAN_INIT_VALUE(val)		\
+	({				\
+		typeof(val) __ret;	\
+		switch (sizeof(val)) {	\
+		case 1:						\
+			*(u8 *)&__ret = KMSAN_INIT_1((u8)val);	\
+			break;					\
+		case 2:						\
+			*(u16 *)&__ret = KMSAN_INIT_2((u16)val);\
+			break;					\
+		case 4:						\
+			*(u32 *)&__ret = KMSAN_INIT_4((u32)val);\
+			break;					\
+		case 8:						\
+			*(u64 *)&__ret = KMSAN_INIT_8((u64)val);\
+			break;					\
+		default:					\
+			BUILD_BUG_ON(1);			\
+		}						\
+		__ret;						\
+	}) /**/
+
+/**
+ * kmsan_poison_shadow() - Mark the memory range as uninitialized.
+ * @address: address to start with.
+ * @size:    size of buffer to poison.
+ * @flags:   GFP flags for allocations done by this function.
+ *
+ * Until other data is written to this range, KMSAN will treat it as
+ * uninitialized. Error reports for this memory will reference the call site of
+ * kmsan_poison_shadow() as origin.
+ */
+void kmsan_poison_shadow(const void *address, size_t size, gfp_t flags);
+
+/**
+ * kmsan_unpoison_shadow() -  Mark the memory range as initialized.
+ * @address: address to start with.
+ * @size:    size of buffer to unpoison.
+ *
+ * Until other data is written to this range, KMSAN will treat it as
+ * initialized.
+ */
+void kmsan_unpoison_shadow(const void *address, size_t size);
+
+/**
+ * kmsan_check_memory() - Check the memory range for being initialized.
+ * @address: address to start with.
+ * @size:    size of buffer to check.
+ *
+ * If any piece of the given range is marked as uninitialized, KMSAN will report
+ * an error.
+ */
+void kmsan_check_memory(const void *address, size_t size);
+
+#else
+
+#define KMSAN_INIT_VALUE(value) (value)
+
+static inline void kmsan_poison_shadow(const void *address, size_t size,
+				       gfp_t flags) {}
+static inline void kmsan_unpoison_shadow(const void *address, size_t size) {}
+static inline void kmsan_check_memory(const void *address, size_t size) {}
+
+#endif
+
+#endif /* _LINUX_KMSAN_CHECKS_H */
diff --git a/include/linux/kmsan.h b/include/linux/kmsan.h
new file mode 100644
index 000000000000..d90676679103
--- /dev/null
+++ b/include/linux/kmsan.h
@@ -0,0 +1,335 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * KMSAN API for subsystems.
+ *
+ * Copyright (C) 2017-2019 Google LLC
+ * Author: Alexander Potapenko <glider@google.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+#ifndef LINUX_KMSAN_H
+#define LINUX_KMSAN_H
+
+#include <linux/dma-direction.h>
+#include <linux/gfp.h>
+#include <linux/stackdepot.h>
+#include <linux/types.h>
+#include <linux/vmalloc.h>
+
+struct page;
+struct kmem_cache;
+struct task_struct;
+struct sk_buff;
+struct urb;
+
+#ifdef CONFIG_KMSAN
+
+/* These constants are defined in the MSan LLVM instrumentation pass. */
+#define KMSAN_RETVAL_SIZE 800
+#define KMSAN_PARAM_SIZE 800
+#define KMSAN_PARAM_ARRAY_SIZE (KMSAN_PARAM_SIZE / sizeof(depot_stack_handle_t))
+
+struct kmsan_context_state {
+	char param_tls[KMSAN_PARAM_SIZE];
+	char retval_tls[KMSAN_RETVAL_SIZE];
+	char va_arg_tls[KMSAN_PARAM_SIZE];
+	char va_arg_origin_tls[KMSAN_PARAM_SIZE];
+	u64 va_arg_overflow_size_tls;
+	depot_stack_handle_t param_origin_tls[KMSAN_PARAM_ARRAY_SIZE];
+	depot_stack_handle_t retval_origin_tls;
+	depot_stack_handle_t origin_tls;
+};
+
+#undef KMSAN_PARAM_ARRAY_SIZE
+#undef KMSAN_PARAM_SIZE
+#undef KMSAN_RETVAL_SIZE
+
+struct kmsan_task_state {
+	bool allow_reporting;
+	struct kmsan_context_state cstate;
+};
+
+/**
+ * kmsan_initialize_shadow() - Initialize KMSAN shadow at boot time.
+ *
+ * Allocate and initialize KMSAN metadata for early allocations.
+ */
+void __init kmsan_initialize_shadow(void);
+
+/**
+ * kmsan_initialize() - Initialize KMSAN state and enable KMSAN.
+ */
+void __init kmsan_initialize(void);
+
+/**
+ * kmsan_task_create() - Initialize KMSAN state for the task.
+ * @task: task to initialize.
+ */
+void kmsan_task_create(struct task_struct *task);
+
+/**
+ * kmsan_task_exit() - Notify KMSAN that a task has exited.
+ * @task: task about to finish.
+ */
+void kmsan_task_exit(struct task_struct *task);
+
+/**
+ * kmsan_alloc_page() - Notify KMSAN about an alloc_pages() call.
+ * @page:  struct page pointer returned by alloc_pages().
+ * @order: order of allocated struct page.
+ * @flags: GFP flags used by alloc_pages()
+ *
+ * Return:
+ * * 0       - Ok
+ * * -ENOMEM - allocation failure
+ *
+ * KMSAN allocates metadata (shadow and origin pages) for @page and marks
+ * 1<<@order pages starting at @page as uninitialized, unless @flags contain
+ * __GFP_ZERO.
+ */
+int kmsan_alloc_page(struct page *page, unsigned int order, gfp_t flags);
+
+/**
+ * kmsan_free_page() - Notify KMSAN about a free_pages() call.
+ * @page:  struct page pointer passed to free_pages().
+ * @order: order of deallocated struct page.
+ *
+ * KMSAN deallocates the metadata pages for the given struct page.
+ */
+void kmsan_free_page(struct page *page, unsigned int order);
+
+/**
+ * kmsan_split_page() - Notify KMSAN about a split_page() call.
+ * @page:  struct page pointer passed to split_page().
+ * @order: order of split struct page.
+ *
+ * KMSAN splits the metadata pages for the given struct page, so that they
+ * can be deallocated separately.
+ */
+void kmsan_split_page(struct page *page, unsigned int order);
+
+/**
+ * kmsan_copy_page_meta() - Copy KMSAN metadata between two pages.
+ * @dst: destination page.
+ * @src: source page.
+ *
+ * KMSAN copies the contents of metadata pages for @src into the metadata pages
+ * for @dst. If @dst has no associated metadata pages, nothing happens.
+ * If @src has no associated metadata pages, @dst metadata pages are unpoisoned.
+ */
+void kmsan_copy_page_meta(struct page *dst, struct page *src);
+
+/**
+ * kmsan_gup_pgd_range() - Notify KMSAN about a gup_pgd_range() call.
+ * @pages: array of struct page pointers.
+ * @nr:    array size.
+ *
+ * gup_pgd_range() creates new pages, some of which may belong to the userspace
+ * memory. In that case these pages should be initialized.
+ */
+void kmsan_gup_pgd_range(struct page **pages, int nr);
+
+/**
+ * kmsan_slab_alloc() - Notify KMSAN about a slab allocation.
+ * @s:      slab cache the object belongs to.
+ * @object: object pointer.
+ * @flags:  GFP flags passed to the allocator.
+ *
+ * Depending on cache flags and GFP flags, KMSAN sets up the metadata of the
+ * newly created object, marking it initialized or uninitialized.
+ */
+void kmsan_slab_alloc(struct kmem_cache *s, void *object, gfp_t flags);
+
+/**
+ * kmsan_slab_free() - Notify KMSAN about a slab deallocation.
+ * @s:      slab cache the object belongs to.
+ * @object: object pointer.
+ *
+ * KMSAN marks the freed object as uninitialized.
+ */
+void kmsan_slab_free(struct kmem_cache *s, void *object);
+
+/**
+ * kmsan_kmalloc_large() - Notify KMSAN about a large slab allocation.
+ * @ptr:   object pointer.
+ * @size:  object size.
+ * @flags: GFP flags passed to the allocator.
+ *
+ * Similar to kmsan_slab_alloc(), but for large allocations.
+ */
+void kmsan_kmalloc_large(const void *ptr, size_t size, gfp_t flags);
+
+/**
+ * kmsan_kfree_large() - Notify KMSAN about a large slab deallocation.
+ * @ptr: object pointer.
+ *
+ * Similar to kmsan_slab_free(), but for large allocations.
+ */
+void kmsan_kfree_large(const void *ptr);
+
+/**
+ * kmsan_vmap_page_range_noflush() - Notify KMSAN about a vmap.
+ * @start: start address of vmapped range.
+ * @end:   end address of vmapped range.
+ * @prot:  page protection flags used for vmap.
+ * @pages: array of pages.
+ *
+ * KMSAN maps shadow and origin pages of @pages into contiguous ranges in
+ * vmalloc metadata address range.
+ */
+void kmsan_vmap_page_range_noflush(unsigned long start, unsigned long end,
+				   pgprot_t prot, struct page **pages);
+
+/**
+ * kmsan_vunmap_page_range() - Notify KMSAN about a vunmap.
+ * @addr: start address of vunmapped range.
+ * @end:  end address of vunmapped range.
+ *
+ * KMSAN unmaps the contiguous metadata ranges created by
+ * kmsan_vmap_page_range_noflush().
+ */
+void kmsan_vunmap_page_range(unsigned long addr, unsigned long end);
+
+/**
+ * kmsan_ioremap_page_range() - Notify KMSAN about a ioremap_page_range() call.
+ * @addr:      range start.
+ * @end:       range end.
+ * @phys_addr: physical range start.
+ * @prot:      page protection flags used for ioremap_page_range().
+ *
+ * KMSAN creates new metadata pages for the physical pages mapped into the
+ * virtual memory.
+ */
+void kmsan_ioremap_page_range(unsigned long addr, unsigned long end,
+			      phys_addr_t phys_addr, pgprot_t prot);
+
+/**
+ * kmsan_iounmap_page_range() - Notify KMSAN about a iounmap_page_range() call.
+ * @start: range start.
+ * @end:   range end.
+ *
+ * KMSAN unmaps the metadata pages for the given range and, unlike for
+ * vunmap_page_range(), also deallocates them.
+ */
+void kmsan_iounmap_page_range(unsigned long start, unsigned long end);
+
+/**
+ * kmsan_softirq_enter() - Notify KMSAN about a context entry.
+ *
+ * This function should be called whenever the kernel leaves the current task
+ * and enters an IRQ, softirq or NMI context. KMSAN will switch the task state
+ * to a per-thread storage.
+ */
+void kmsan_context_enter(void);
+
+/**
+ * kmsan_softirq_exit() - Notify KMSAN about a context exit.
+ *
+ * This function should be called when the kernel leaves the previously entered
+ * context.
+ */
+void kmsan_context_exit(void);
+
+/**
+ * kmsan_copy_to_user() - Notify KMSAN about a data transfer to userspace.
+ * @to:      destination address in the userspace.
+ * @from:    source address in the kernel.
+ * @to_copy: number of bytes to copy.
+ * @left:    number of bytes not copied.
+ *
+ * If this is a real userspace data transfer, KMSAN checks the bytes that were
+ * actually copied to ensure there was no information leak. If @to belongs to
+ * the kernel space (which is possible for compat syscalls), KMSAN just copies
+ * the metadata.
+ */
+void kmsan_copy_to_user(const void *to, const void *from, size_t to_copy,
+			size_t left);
+
+/**
+ * kmsan_check_skb() - Check an sk_buff for being initialized.
+ *
+ * KMSAN checks the memory belonging to a socket buffer and reports an error if
+ * contains uninitialized values.
+ */
+void kmsan_check_skb(const struct sk_buff *skb);
+
+/**
+ * kmsan_handle_dma() - Handle a DMA data transfer.
+ * @address:   buffer address.
+ * @size:      buffer size.
+ * @direction: one of possible dma_data_direction values.
+ *
+ * Depending on @direction, KMSAN:
+ * * checks the buffer, if it is copied to device;
+ * * initializes the buffer, if it is copied from device;
+ * * does both, if this is a DMA_BIDIRECTIONAL transfer.
+ */
+void kmsan_handle_dma(const void *address, size_t size,
+		      enum dma_data_direction direction);
+
+/**
+ * kmsan_handle_urb() - Handle a USB data transfer.
+ * @urb:    struct urb pointer.
+ * @is_out: data transfer direction (true means output to hardware)
+ *
+ * If @is_out is true, KMSAN checks the transfer buffer of @urb. Otherwise,
+ * KMSAN initializes the transfer buffer.
+ */
+void kmsan_handle_urb(const struct urb *urb, bool is_out);
+
+#else
+
+static inline void __init kmsan_initialize_shadow(void) { }
+static inline void __init kmsan_initialize(void) { }
+
+static inline void kmsan_task_create(struct task_struct *task) {}
+static inline void kmsan_task_exit(struct task_struct *task) {}
+
+static inline int kmsan_alloc_page(struct page *page, unsigned int order,
+				   gfp_t flags)
+{
+	return 0;
+}
+static inline void kmsan_free_page(struct page *page, unsigned int order) {}
+static inline void kmsan_split_page(struct page *page, unsigned int order) {}
+static inline void kmsan_copy_page_meta(struct page *dst, struct page *src) {}
+static inline void kmsan_gup_pgd_range(struct page **pages, int nr) {}
+
+static inline void kmsan_slab_alloc(struct kmem_cache *s, void *object,
+				    gfp_t flags) {}
+static inline void kmsan_slab_free(struct kmem_cache *s, void *object) {}
+static inline void kmsan_kmalloc_large(const void *ptr, size_t size,
+				       gfp_t flags) {}
+static inline void kmsan_kfree_large(const void *ptr) {}
+
+static inline void kmsan_vmap_page_range_noflush(unsigned long start,
+						 unsigned long end,
+						 pgprot_t prot,
+						 struct page **pages) {}
+static inline void kmsan_vunmap_page_range(unsigned long start,
+					   unsigned long end) {}
+
+static inline void kmsan_ioremap_page_range(unsigned long start,
+					    unsigned long end,
+					    phys_addr_t phys_addr,
+					    pgprot_t prot) {}
+static inline void kmsan_iounmap_page_range(unsigned long start,
+					    unsigned long end) {}
+
+static inline void kmsan_softirq_enter(void) {}
+static inline void kmsan_softirq_exit(void) {}
+
+static inline void kmsan_copy_to_user(
+	const void *to, const void *from, size_t to_copy, size_t left) {}
+
+static inline void kmsan_check_skb(const struct sk_buff *skb) {}
+static inline void kmsan_handle_dma(const void *address, size_t size,
+				    enum dma_data_direction direction) {}
+static inline void kmsan_handle_urb(const struct urb *urb, bool is_out) {}
+
+#endif
+
+#endif /* LINUX_KMSAN_H */
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 270aa8fd2800..37411d907ef4 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -216,6 +216,15 @@ struct page {
 					   not kmapped, ie. highmem) */
 #endif /* WANT_PAGE_VIRTUAL */
 
+#ifdef CONFIG_KMSAN
+	/*
+	 * Bits in struct page are scarce, so the LSB in *shadow is used to
+	 * indicate whether the page should be ignored by KMSAN or not.
+	 */
+	struct page *shadow;
+	struct page *origin;
+#endif
+
 #ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
 	int _last_cpupid;
 #endif
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 467d26046416..ba9d43192cad 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -15,6 +15,7 @@
 #include <linux/sem.h>
 #include <linux/shm.h>
 #include <linux/kcov.h>
+#include <linux/kmsan.h>
 #include <linux/mutex.h>
 #include <linux/plist.h>
 #include <linux/hrtimer.h>
@@ -1180,6 +1181,10 @@ struct task_struct {
 	unsigned int			kasan_depth;
 #endif
 
+#ifdef CONFIG_KMSAN
+	struct kmsan_task_state		kmsan;
+#endif
+
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 	/* Index of current stored address in ret_stack: */
 	int				curr_ret_stack;
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index d1842fe756d5..10c6ccdb7af6 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -783,6 +783,8 @@ config DEBUG_STACKOVERFLOW
 
 source "lib/Kconfig.kasan"
 
+source "lib/Kconfig.kmsan"
+
 endmenu # "Memory Debugging"
 
 config DEBUG_SHIRQ
diff --git a/lib/Kconfig.kmsan b/lib/Kconfig.kmsan
new file mode 100644
index 000000000000..187dddfcf220
--- /dev/null
+++ b/lib/Kconfig.kmsan
@@ -0,0 +1,22 @@
+config HAVE_ARCH_KMSAN
+	bool
+
+if HAVE_ARCH_KMSAN
+
+config KMSAN
+	bool "KMSAN: detector of uninitialized memory use"
+	depends on SLUB && !KASAN
+	select STACKDEPOT
+	help
+	  KMSAN is a dynamic detector of uses of uninitialized memory in the
+	  kernel. It is based on compiler instrumentation provided by Clang
+	  and thus requires Clang 10.0.0+ to build.
+
+config TEST_KMSAN
+	tristate "Module for testing KMSAN for bug detection"
+	depends on m && KMSAN
+	help
+	  Test module that can trigger various uses of uninitialized memory
+	  detectable by KMSAN.
+
+endif
diff --git a/mm/kmsan/Makefile b/mm/kmsan/Makefile
new file mode 100644
index 000000000000..a9778eb8a46a
--- /dev/null
+++ b/mm/kmsan/Makefile
@@ -0,0 +1,11 @@
+obj-y := kmsan.o kmsan_instr.o kmsan_init.o kmsan_entry.o kmsan_hooks.o kmsan_report.o kmsan_shadow.o
+
+# KMSAN runtime functions may enable UACCESS checks, so build them without
+# stackprotector to avoid objtool warnings.
+CFLAGS_kmsan_instr.o := $(call cc-option, -fno-stack-protector)
+CFLAGS_kmsan_shadow.o := $(call cc-option, -fno-stack-protector)
+CFLAGS_kmsan_hooks.o := $(call cc-option, -fno-stack-protector)
+
+KMSAN_SANITIZE := n
+KCOV_INSTRUMENT := n
+UBSAN_SANITIZE := n
diff --git a/mm/kmsan/kmsan.c b/mm/kmsan/kmsan.c
new file mode 100644
index 000000000000..037f8b5f33a5
--- /dev/null
+++ b/mm/kmsan/kmsan.c
@@ -0,0 +1,547 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * KMSAN runtime library.
+ *
+ * Copyright (C) 2017-2019 Google LLC
+ * Author: Alexander Potapenko <glider@google.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <asm/page.h>
+#include <linux/compiler.h>
+#include <linux/export.h>
+#include <linux/highmem.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/kmsan.h>
+#include <linux/memory.h>
+#include <linux/mm.h>
+#include <linux/mm_types.h>
+#include <linux/mmzone.h>
+#include <linux/percpu-defs.h>
+#include <linux/preempt.h>
+#include <linux/slab.h>
+#include <linux/stackdepot.h>
+#include <linux/stacktrace.h>
+#include <linux/types.h>
+#include <linux/vmalloc.h>
+
+#include "../slab.h"
+#include "kmsan.h"
+
+#define KMSAN_STACK_DEPTH 64
+#define MAX_CHAIN_DEPTH 7
+
+/*
+ * Some kernel asm() calls mention the non-existing |__force_order| variable
+ * in the asm constraints to preserve the order of accesses to control
+ * registers. KMSAN turns those mentions into actual memory accesses, therefore
+ * the variable is now required to link the kernel.
+ */
+unsigned long __force_order;
+EXPORT_SYMBOL(__force_order);
+
+bool kmsan_ready;
+/*
+ * According to Documentation/x86/kernel-stacks, kernel code can run on the
+ * following stacks:
+ * - regular task stack - when executing the task code
+ *  - interrupt stack - when handling external hardware interrupts and softirqs
+ *  - NMI stack
+ * 0 is for regular interrupts, 1 for softirqs, 2 for NMI.
+ * Because interrupts may nest, trying to use a new context for every new
+ * interrupt.
+ */
+/* [0] for dummy per-CPU context. */
+DEFINE_PER_CPU(struct kmsan_context_state[KMSAN_NESTED_CONTEXT_MAX],
+	       kmsan_percpu_cstate);
+/* 0 for task context, |i>0| for kmsan_context_state[i]. */
+DEFINE_PER_CPU(int, kmsan_context_level);
+DEFINE_PER_CPU(int, kmsan_in_runtime_cnt);
+
+struct kmsan_context_state *kmsan_task_context_state(void)
+{
+	int cpu = smp_processor_id();
+	int level = this_cpu_read(kmsan_context_level);
+	struct kmsan_context_state *ret;
+
+	if (!kmsan_ready || kmsan_in_runtime()) {
+		ret = &per_cpu(kmsan_percpu_cstate[0], cpu);
+		__memset(ret, 0, sizeof(struct kmsan_context_state));
+		return ret;
+	}
+
+	if (!level)
+		ret = &current->kmsan.cstate;
+	else
+		ret = &per_cpu(kmsan_percpu_cstate[level], cpu);
+	return ret;
+}
+
+void kmsan_internal_task_create(struct task_struct *task)
+{
+	struct kmsan_task_state *state = &task->kmsan;
+
+	__memset(state, 0, sizeof(struct kmsan_task_state));
+	state->allow_reporting = true;
+}
+
+void kmsan_internal_memset_shadow(void *addr, int b, size_t size,
+				  bool checked)
+{
+	void *shadow_start;
+	u64 page_offset, address = (u64)addr;
+	size_t to_fill;
+
+	BUG_ON(!metadata_is_contiguous(addr, size, META_SHADOW));
+	while (size) {
+		page_offset = address % PAGE_SIZE;
+		to_fill = min(PAGE_SIZE - page_offset, (u64)size);
+		shadow_start = kmsan_get_metadata((void *)address, to_fill,
+						  META_SHADOW);
+		if (!shadow_start) {
+			if (checked)
+				panic("%s: not memsetting %d bytes starting at %px, because the shadow is NULL\n",
+				      __func__, to_fill, address);
+			/* Otherwise just move on. */
+		} else {
+			__memset(shadow_start, b, to_fill);
+		}
+		address += to_fill;
+		size -= to_fill;
+	}
+}
+
+void kmsan_internal_poison_shadow(void *address, size_t size,
+				gfp_t flags, unsigned int poison_flags)
+{
+	bool checked = poison_flags & KMSAN_POISON_CHECK;
+	depot_stack_handle_t handle;
+	u32 extra_bits = kmsan_extra_bits(/*depth*/0,
+					  poison_flags & KMSAN_POISON_FREE);
+
+	kmsan_internal_memset_shadow(address, -1, size, checked);
+	handle = kmsan_save_stack_with_flags(flags, extra_bits);
+	kmsan_set_origin_checked(address, size, handle, checked);
+}
+
+void kmsan_internal_unpoison_shadow(void *address, size_t size, bool checked)
+{
+	kmsan_internal_memset_shadow(address, 0, size, checked);
+	kmsan_set_origin_checked(address, size, 0, checked);
+}
+
+depot_stack_handle_t kmsan_save_stack_with_flags(gfp_t flags,
+						 unsigned int reserved)
+{
+	depot_stack_handle_t handle;
+	unsigned long entries[KMSAN_STACK_DEPTH];
+	unsigned int nr_entries;
+
+	nr_entries = stack_trace_save(entries, KMSAN_STACK_DEPTH, 0);
+	nr_entries = filter_irq_stacks(entries, nr_entries);
+
+	/* Don't sleep (see might_sleep_if() in __alloc_pages_nodemask()). */
+	flags &= ~__GFP_DIRECT_RECLAIM;
+
+	handle = stack_depot_save(entries, nr_entries, flags);
+	return set_dsh_extra_bits(handle, reserved);
+}
+
+/*
+ * Depending on the value of is_memmove, this serves as both a memcpy and a
+ * memmove implementation.
+ *
+ * As with the regular memmove, do the following:
+ * - if src and dst don't overlap, use memcpy();
+ * - if src and dst overlap:
+ *   - if src > dst, use memcpy();
+ *   - if src < dst, use reverse-memcpy.
+ * Why this is correct:
+ * - problems may arise if for some part of the overlapping region we
+ *   overwrite its shadow with a new value before copying it somewhere.
+ *   But there's a 1:1 mapping between the kernel memory and its shadow,
+ *   therefore if this doesn't happen with the kernel memory it can't happen
+ *   with the shadow.
+ */
+static void kmsan_memcpy_memmove_metadata(void *dst, void *src, size_t n,
+					  bool is_memmove)
+{
+	void *shadow_src, *shadow_dst;
+	depot_stack_handle_t *origin_src, *origin_dst;
+	int src_slots, dst_slots, i, iter, step, skip_bits;
+	depot_stack_handle_t old_origin = 0, chain_origin, new_origin = 0;
+	u32 *align_shadow_src, shadow;
+	bool backwards;
+
+	shadow_dst = kmsan_get_metadata(dst, n, META_SHADOW);
+	if (!shadow_dst)
+		return;
+	BUG_ON(!metadata_is_contiguous(dst, n, META_SHADOW));
+
+	shadow_src = kmsan_get_metadata(src, n, META_SHADOW);
+	if (!shadow_src) {
+		/*
+		 * |src| is untracked: zero out destination shadow, ignore the
+		 * origins, we're done.
+		 */
+		__memset(shadow_dst, 0, n);
+		return;
+	}
+	BUG_ON(!metadata_is_contiguous(src, n, META_SHADOW));
+
+	if (is_memmove)
+		__memmove(shadow_dst, shadow_src, n);
+	else
+		__memcpy(shadow_dst, shadow_src, n);
+
+	origin_dst = kmsan_get_metadata(dst, n, META_ORIGIN);
+	origin_src = kmsan_get_metadata(src, n, META_ORIGIN);
+	BUG_ON(!origin_dst || !origin_src);
+	BUG_ON(!metadata_is_contiguous(dst, n, META_ORIGIN));
+	BUG_ON(!metadata_is_contiguous(src, n, META_ORIGIN));
+	src_slots = (ALIGN((u64)src + n, ORIGIN_SIZE) -
+		     ALIGN_DOWN((u64)src, ORIGIN_SIZE)) / ORIGIN_SIZE;
+	dst_slots = (ALIGN((u64)dst + n, ORIGIN_SIZE) -
+		     ALIGN_DOWN((u64)dst, ORIGIN_SIZE)) / ORIGIN_SIZE;
+	BUG_ON(!src_slots || !dst_slots);
+	BUG_ON((src_slots < 1) || (dst_slots < 1));
+	BUG_ON((src_slots - dst_slots > 1) || (dst_slots - src_slots < -1));
+
+	backwards = is_memmove && (dst > src);
+	i = backwards ? min(src_slots, dst_slots) - 1 : 0;
+	iter = backwards ? -1 : 1;
+
+	align_shadow_src = (u32 *)ALIGN_DOWN((u64)shadow_src, ORIGIN_SIZE);
+	for (step = 0; step < min(src_slots, dst_slots); step++, i += iter) {
+		BUG_ON(i < 0);
+		shadow = align_shadow_src[i];
+		if (i == 0) {
+			/*
+			 * If |src| isn't aligned on ORIGIN_SIZE, don't
+			 * look at the first |src % ORIGIN_SIZE| bytes
+			 * of the first shadow slot.
+			 */
+			skip_bits = ((u64)src % ORIGIN_SIZE) * 8;
+			shadow = (shadow << skip_bits) >> skip_bits;
+		}
+		if (i == src_slots - 1) {
+			/*
+			 * If |src + n| isn't aligned on
+			 * ORIGIN_SIZE, don't look at the last
+			 * |(src + n) % ORIGIN_SIZE| bytes of the
+			 * last shadow slot.
+			 */
+			skip_bits = (((u64)src + n) % ORIGIN_SIZE) * 8;
+			shadow = (shadow >> skip_bits) << skip_bits;
+		}
+		/*
+		 * Overwrite the origin only if the corresponding
+		 * shadow is nonempty.
+		 */
+		if (origin_src[i] && (origin_src[i] != old_origin) && shadow) {
+			old_origin = origin_src[i];
+			chain_origin = kmsan_internal_chain_origin(old_origin);
+			/*
+			 * kmsan_internal_chain_origin() may return
+			 * NULL, but we don't want to lose the previous
+			 * origin value.
+			 */
+			if (chain_origin)
+				new_origin = chain_origin;
+			else
+				new_origin = old_origin;
+		}
+		if (shadow)
+			origin_dst[i] = new_origin;
+		else
+			origin_dst[i] = 0;
+	}
+}
+
+void kmsan_memcpy_metadata(void *dst, void *src, size_t n)
+{
+	kmsan_memcpy_memmove_metadata(dst, src, n, /*is_memmove*/false);
+}
+
+void kmsan_memmove_metadata(void *dst, void *src, size_t n)
+{
+	kmsan_memcpy_memmove_metadata(dst, src, n, /*is_memmove*/true);
+}
+
+depot_stack_handle_t kmsan_internal_chain_origin(depot_stack_handle_t id)
+{
+	depot_stack_handle_t handle;
+	unsigned long entries[3];
+	u64 magic = KMSAN_CHAIN_MAGIC_ORIGIN_FULL;
+	int depth = 0;
+	static int skipped;
+	u32 extra_bits;
+	bool uaf;
+
+	if (!id)
+		return id;
+	/*
+	 * Make sure we have enough spare bits in |id| to hold the UAF bit and
+	 * the chain depth.
+	 */
+	BUILD_BUG_ON((1 << STACK_DEPOT_EXTRA_BITS) <= (MAX_CHAIN_DEPTH << 1));
+
+	extra_bits = get_dsh_extra_bits(id);
+	depth = kmsan_depth_from_eb(extra_bits);
+	uaf = kmsan_uaf_from_eb(extra_bits);
+
+	if (depth >= MAX_CHAIN_DEPTH) {
+		skipped++;
+		if (skipped % 10000 == 0) {
+			pr_warn("not chained %d origins\n", skipped);
+			dump_stack();
+			kmsan_print_origin(id);
+		}
+		return id;
+	}
+	depth++;
+	extra_bits = kmsan_extra_bits(depth, uaf);
+
+	entries[0] = magic + depth;
+	entries[1] = kmsan_save_stack_with_flags(GFP_ATOMIC, extra_bits);
+	entries[2] = id;
+	handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_ATOMIC);
+	return set_dsh_extra_bits(handle, extra_bits);
+}
+
+void kmsan_write_aligned_origin(void *var, size_t size, u32 origin)
+{
+	u32 *var_cast = (u32 *)var;
+	int i;
+
+	BUG_ON((u64)var_cast % ORIGIN_SIZE);
+	BUG_ON(size % ORIGIN_SIZE);
+	for (i = 0; i < size / ORIGIN_SIZE; i++)
+		var_cast[i] = origin;
+}
+
+void kmsan_internal_set_origin(void *addr, int size, u32 origin)
+{
+	void *origin_start;
+	u64 address = (u64)addr, page_offset;
+	size_t to_fill, pad = 0;
+
+	if (!IS_ALIGNED(address, ORIGIN_SIZE)) {
+		pad = address % ORIGIN_SIZE;
+		address -= pad;
+		size += pad;
+	}
+
+	while (size > 0) {
+		page_offset = address % PAGE_SIZE;
+		to_fill = min(PAGE_SIZE - page_offset, (u64)size);
+		/* write at least ORIGIN_SIZE bytes */
+		to_fill = ALIGN(to_fill, ORIGIN_SIZE);
+		BUG_ON(!to_fill);
+		origin_start = kmsan_get_metadata((void *)address, to_fill,
+						  META_ORIGIN);
+		address += to_fill;
+		size -= to_fill;
+		if (!origin_start)
+			/* Can happen e.g. if the memory is untracked. */
+			continue;
+		kmsan_write_aligned_origin(origin_start, to_fill, origin);
+	}
+}
+
+void kmsan_set_origin_checked(void *addr, int size, u32 origin, bool checked)
+{
+	if (checked && !metadata_is_contiguous(addr, size, META_ORIGIN))
+		panic("%s: WARNING: not setting origin for %d bytes starting at %px, because the metadata is incontiguous\n",
+		      __func__, size, addr);
+	kmsan_internal_set_origin(addr, size, origin);
+}
+
+struct page *vmalloc_to_page_or_null(void *vaddr)
+{
+	struct page *page;
+
+	if (!kmsan_internal_is_vmalloc_addr(vaddr) &&
+	    !kmsan_internal_is_module_addr(vaddr))
+		return NULL;
+	page = vmalloc_to_page(vaddr);
+	if (pfn_valid(page_to_pfn(page)))
+		return page;
+	else
+		return NULL;
+}
+
+void kmsan_internal_check_memory(void *addr, size_t size, const void *user_addr,
+				 int reason)
+{
+	unsigned long irq_flags;
+	unsigned long addr64 = (unsigned long)addr;
+	unsigned char *shadow = NULL;
+	depot_stack_handle_t *origin = NULL;
+	depot_stack_handle_t cur_origin = 0, new_origin = 0;
+	int cur_off_start = -1;
+	int i, chunk_size;
+	size_t pos = 0;
+
+	BUG_ON(!metadata_is_contiguous(addr, size, META_SHADOW));
+	if (size <= 0)
+		return;
+	while (pos < size) {
+		chunk_size = min(size - pos,
+				 PAGE_SIZE - ((addr64 + pos) % PAGE_SIZE));
+		shadow = kmsan_get_metadata((void *)(addr64 + pos), chunk_size,
+					    META_SHADOW);
+		if (!shadow) {
+			/*
+			 * This page is untracked. If there were uninitialized
+			 * bytes before, report them.
+			 */
+			if (cur_origin) {
+				irq_flags = kmsan_enter_runtime();
+				kmsan_report(cur_origin, addr, size,
+					     cur_off_start, pos - 1, user_addr,
+					     reason);
+				kmsan_leave_runtime(irq_flags);
+			}
+			cur_origin = 0;
+			cur_off_start = -1;
+			pos += chunk_size;
+			continue;
+		}
+		for (i = 0; i < chunk_size; i++) {
+			if (!shadow[i]) {
+				/*
+				 * This byte is unpoisoned. If there were
+				 * poisoned bytes before, report them.
+				 */
+				if (cur_origin) {
+					irq_flags = kmsan_enter_runtime();
+					kmsan_report(cur_origin, addr, size,
+						     cur_off_start, pos + i - 1,
+						     user_addr, reason);
+					kmsan_leave_runtime(irq_flags);
+				}
+				cur_origin = 0;
+				cur_off_start = -1;
+				continue;
+			}
+			origin = kmsan_get_metadata((void *)(addr64 + pos + i),
+						chunk_size - i, META_ORIGIN);
+			BUG_ON(!origin);
+			new_origin = *origin;
+			/*
+			 * Encountered new origin - report the previous
+			 * uninitialized range.
+			 */
+			if (cur_origin != new_origin) {
+				if (cur_origin) {
+					irq_flags = kmsan_enter_runtime();
+					kmsan_report(cur_origin, addr, size,
+						     cur_off_start, pos + i - 1,
+						     user_addr, reason);
+					kmsan_leave_runtime(irq_flags);
+				}
+				cur_origin = new_origin;
+				cur_off_start = pos + i;
+			}
+		}
+		pos += chunk_size;
+	}
+	BUG_ON(pos != size);
+	if (cur_origin) {
+		irq_flags = kmsan_enter_runtime();
+		kmsan_report(cur_origin, addr, size, cur_off_start, pos - 1,
+			     user_addr, reason);
+		kmsan_leave_runtime(irq_flags);
+	}
+}
+
+bool metadata_is_contiguous(void *addr, size_t size, bool is_origin)
+{
+	u64 cur_addr = (u64)addr, next_addr;
+	char *cur_meta = NULL, *next_meta = NULL;
+	depot_stack_handle_t *origin_p;
+	bool all_untracked = false;
+	const char *fname = is_origin ? "origin" : "shadow";
+
+	if (!size)
+		return true;
+
+	/* The whole range belongs to the same page. */
+	if (ALIGN_DOWN(cur_addr + size - 1, PAGE_SIZE) ==
+	    ALIGN_DOWN(cur_addr, PAGE_SIZE))
+		return true;
+	cur_meta = kmsan_get_metadata((void *)cur_addr, 1, is_origin);
+	if (!cur_meta)
+		all_untracked = true;
+	for (next_addr = cur_addr + PAGE_SIZE; next_addr < (u64)addr + size;
+		     cur_addr = next_addr,
+		     cur_meta = next_meta,
+		     next_addr += PAGE_SIZE) {
+		next_meta = kmsan_get_metadata((void *)next_addr, 1, is_origin);
+		if (!next_meta) {
+			if (!all_untracked)
+				goto report;
+			continue;
+		}
+		if ((u64)cur_meta == ((u64)next_meta - PAGE_SIZE))
+			continue;
+		goto report;
+	}
+	return true;
+
+report:
+	pr_err("BUG: attempting to access two shadow page ranges.\n");
+	dump_stack();
+	pr_err("\n");
+	pr_err("Access of size %d at %px.\n", size, addr);
+	pr_err("Addresses belonging to different ranges: %px and %px\n",
+	       cur_addr, next_addr);
+	pr_err("page[0].%s: %px, page[1].%s: %px\n",
+	       fname, cur_meta, fname, next_meta);
+	origin_p = kmsan_get_metadata(addr, 1, META_ORIGIN);
+	if (origin_p) {
+		pr_err("Origin: %08x\n", *origin_p);
+		kmsan_print_origin(*origin_p);
+	} else {
+		pr_err("Origin: unavailable\n");
+	}
+	return false;
+}
+
+/*
+ * Dummy replacement for __builtin_return_address() which may crash without
+ * frame pointers.
+ */
+void *kmsan_internal_return_address(int arg)
+{
+#ifdef CONFIG_UNWINDER_FRAME_POINTER
+	switch (arg) {
+	case 1:
+		return __builtin_return_address(1);
+	case 2:
+		return __builtin_return_address(2);
+	default:
+		BUG();
+	}
+#else
+	unsigned long entries[1];
+
+	stack_trace_save(entries, 1, arg);
+	return (void *)entries[0];
+#endif
+}
+
+bool kmsan_internal_is_module_addr(void *vaddr)
+{
+	return ((u64)vaddr >= MODULES_VADDR) && ((u64)vaddr < MODULES_END);
+}
+
+bool kmsan_internal_is_vmalloc_addr(void *addr)
+{
+	return ((u64)addr >= VMALLOC_START) && ((u64)addr < VMALLOC_END);
+}
diff --git a/mm/kmsan/kmsan.h b/mm/kmsan/kmsan.h
new file mode 100644
index 000000000000..9568b0005b5e
--- /dev/null
+++ b/mm/kmsan/kmsan.h
@@ -0,0 +1,161 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * KMSAN internal declarations.
+ *
+ * Copyright (C) 2017-2019 Google LLC
+ * Author: Alexander Potapenko <glider@google.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+#ifndef __MM_KMSAN_KMSAN_H
+#define __MM_KMSAN_KMSAN_H
+
+#include <asm/pgtable_64_types.h>
+#include <linux/irqflags.h>
+#include <linux/sched.h>
+#include <linux/stackdepot.h>
+#include <linux/stacktrace.h>
+#include <linux/nmi.h>
+#include <linux/mm.h>
+#include <linux/printk.h>
+
+#include "kmsan_shadow.h"
+
+#define KMSAN_MAGIC_MASK 0xffffffffff00
+#define KMSAN_ALLOCA_MAGIC_ORIGIN 0x4110c4071900
+#define KMSAN_CHAIN_MAGIC_ORIGIN_FULL 0xd419170cba00
+
+#define KMSAN_POISON_NOCHECK	0x0
+#define KMSAN_POISON_CHECK	0x1
+#define KMSAN_POISON_FREE	0x2
+
+#define ORIGIN_SIZE 4
+
+#define META_SHADOW	(false)
+#define META_ORIGIN	(true)
+
+#define KMSAN_NESTED_CONTEXT_MAX (8)
+/* [0] for dummy per-CPU context */
+DECLARE_PER_CPU(struct kmsan_context_state[KMSAN_NESTED_CONTEXT_MAX],
+		kmsan_percpu_cstate);
+/* 0 for task context, |i>0| for kmsan_context_state[i]. */
+DECLARE_PER_CPU(int, kmsan_context_level);
+
+extern spinlock_t report_lock;
+extern bool kmsan_ready;
+
+void kmsan_print_origin(depot_stack_handle_t origin);
+void kmsan_report(depot_stack_handle_t origin,
+		  void *address, int size, int off_first, int off_last,
+		  const void *user_addr, int reason);
+
+enum KMSAN_BUG_REASON {
+	REASON_ANY,
+	REASON_COPY_TO_USER,
+	REASON_USE_AFTER_FREE,
+	REASON_SUBMIT_URB,
+};
+
+/*
+ * When a compiler hook is invoked, it may make a call to instrumented code
+ * and eventually call itself recursively. To avoid that, we protect the
+ * runtime entry points with kmsan_enter_runtime()/kmsan_leave_runtime() and
+ * exit the hook if kmsan_in_runtime() is true. But when an interrupt occurs
+ * inside the runtime, the hooks won’t run either, which may lead to errors.
+ * Therefore we have to disable interrupts inside the runtime.
+ */
+DECLARE_PER_CPU(int, kmsan_in_runtime_cnt);
+
+static __always_inline bool kmsan_in_runtime(void)
+{
+	return this_cpu_read(kmsan_in_runtime_cnt);
+}
+
+static __always_inline unsigned long kmsan_enter_runtime(void)
+{
+	int level;
+	unsigned long irq_flags;
+
+	preempt_disable();
+	local_irq_save(irq_flags);
+	stop_nmi();
+	level = this_cpu_inc_return(kmsan_in_runtime_cnt);
+	BUG_ON(level > 1);
+	return irq_flags;
+}
+
+static __always_inline void kmsan_leave_runtime(unsigned long irq_flags)
+{
+	int level = this_cpu_dec_return(kmsan_in_runtime_cnt);
+
+	if (level)
+		panic("kmsan_in_runtime: %d\n", level);
+	restart_nmi();
+	local_irq_restore(irq_flags);
+	preempt_enable();
+}
+
+void kmsan_memcpy_metadata(void *dst, void *src, size_t n);
+void kmsan_memmove_metadata(void *dst, void *src, size_t n);
+
+depot_stack_handle_t kmsan_save_stack(void);
+depot_stack_handle_t kmsan_save_stack_with_flags(gfp_t flags,
+						 unsigned int extra_bits);
+
+/*
+ * Pack and unpack the origin chain depth and UAF flag to/from the extra bits
+ * provided by the stack depot.
+ * The UAF flag is stored in the lowest bit, followed by the depth in the upper
+ * bits.
+ * set_dsh_extra_bits() is responsible for clamping the value.
+ */
+static __always_inline unsigned int kmsan_extra_bits(unsigned int depth,
+						     bool uaf)
+{
+	return (depth << 1) | uaf;
+}
+
+static __always_inline bool kmsan_uaf_from_eb(unsigned int extra_bits)
+{
+	return extra_bits & 1;
+}
+
+static __always_inline unsigned int kmsan_depth_from_eb(unsigned int extra_bits)
+{
+	return extra_bits >> 1;
+}
+
+void kmsan_internal_poison_shadow(void *address, size_t size, gfp_t flags,
+				  unsigned int poison_flags);
+void kmsan_internal_unpoison_shadow(void *address, size_t size, bool checked);
+void kmsan_internal_memset_shadow(void *address, int b, size_t size,
+				  bool checked);
+depot_stack_handle_t kmsan_internal_chain_origin(depot_stack_handle_t id);
+void kmsan_write_aligned_origin(void *var, size_t size, u32 origin);
+
+void kmsan_internal_task_create(struct task_struct *task);
+void kmsan_internal_set_origin(void *addr, int size, u32 origin);
+void kmsan_set_origin_checked(void *addr, int size, u32 origin, bool checked);
+
+struct kmsan_context_state *kmsan_task_context_state(void);
+
+bool metadata_is_contiguous(void *addr, size_t size, bool is_origin);
+void kmsan_internal_check_memory(void *addr, size_t size, const void *user_addr,
+				 int reason);
+
+struct page *vmalloc_to_page_or_null(void *vaddr);
+
+/* Declared in mm/vmalloc.c */
+void __vunmap_page_range(unsigned long addr, unsigned long end);
+int __vmap_page_range_noflush(unsigned long start, unsigned long end,
+				   pgprot_t prot, struct page **pages);
+
+void *kmsan_internal_return_address(int arg);
+bool kmsan_internal_is_module_addr(void *vaddr);
+bool kmsan_internal_is_vmalloc_addr(void *addr);
+
+#endif  /* __MM_KMSAN_KMSAN_H */
diff --git a/mm/kmsan/kmsan_init.c b/mm/kmsan/kmsan_init.c
new file mode 100644
index 000000000000..12c84efa70ff
--- /dev/null
+++ b/mm/kmsan/kmsan_init.c
@@ -0,0 +1,79 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * KMSAN initialization routines.
+ *
+ * Copyright (C) 2017-2019 Google LLC
+ * Author: Alexander Potapenko <glider@google.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+#include "kmsan.h"
+
+#include <asm/cpu_entry_area.h>
+#include <asm/sections.h>
+#include <linux/mm.h>
+#include <linux/memblock.h>
+
+#define NUM_FUTURE_RANGES 128
+struct start_end_pair {
+	void *start, *end;
+};
+
+static struct start_end_pair start_end_pairs[NUM_FUTURE_RANGES] __initdata;
+static int future_index __initdata;
+
+/*
+ * Record a range of memory for which the metadata pages will be created once
+ * the page allocator becomes available.
+ */
+static void __init kmsan_record_future_shadow_range(void *start, void *end)
+{
+	BUG_ON(future_index == NUM_FUTURE_RANGES);
+	BUG_ON((start >= end) || !start || !end);
+	start_end_pairs[future_index].start = start;
+	start_end_pairs[future_index].end = end;
+	future_index++;
+}
+
+/*
+ * Initialize the shadow for existing mappings during kernel initialization.
+ * These include kernel text/data sections, NODE_DATA and future ranges
+ * registered while creating other data (e.g. percpu).
+ *
+ * Allocations via memblock can be only done before slab is initialized.
+ */
+void __init kmsan_initialize_shadow(void)
+{
+	int nid;
+	u64 i;
+	const size_t nd_size = roundup(sizeof(pg_data_t), PAGE_SIZE);
+	phys_addr_t p_start, p_end;
+
+	for_each_reserved_mem_region(i, &p_start, &p_end)
+		kmsan_record_future_shadow_range(phys_to_virt(p_start),
+						 phys_to_virt(p_end+1));
+	/* Allocate shadow for .data */
+	kmsan_record_future_shadow_range(_sdata, _edata);
+
+	for_each_online_node(nid)
+		kmsan_record_future_shadow_range(
+			NODE_DATA(nid), (char *)NODE_DATA(nid) + nd_size);
+
+	for (i = 0; i < future_index; i++)
+		kmsan_init_alloc_meta_for_range(start_end_pairs[i].start,
+						start_end_pairs[i].end);
+}
+EXPORT_SYMBOL(kmsan_initialize_shadow);
+
+void __init kmsan_initialize(void)
+{
+	/* Assuming current is init_task */
+	kmsan_internal_task_create(current);
+	pr_info("Starting KernelMemorySanitizer\n");
+	kmsan_ready = true;
+}
+EXPORT_SYMBOL(kmsan_initialize);
diff --git a/mm/kmsan/kmsan_report.c b/mm/kmsan/kmsan_report.c
new file mode 100644
index 000000000000..7455fa7d10bb
--- /dev/null
+++ b/mm/kmsan/kmsan_report.c
@@ -0,0 +1,143 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * KMSAN error reporting routines.
+ *
+ * Copyright (C) 2019 Google LLC
+ * Author: Alexander Potapenko <glider@google.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+#include <linux/console.h>
+#include <linux/stackdepot.h>
+#include <linux/stacktrace.h>
+
+#include "kmsan.h"
+
+DEFINE_SPINLOCK(report_lock);
+
+void kmsan_print_origin(depot_stack_handle_t origin)
+{
+	unsigned long *entries = NULL, *chained_entries = NULL;
+	unsigned long nr_entries, chained_nr_entries, magic;
+	char *descr = NULL;
+	void *pc1 = NULL, *pc2 = NULL;
+	depot_stack_handle_t head;
+
+	if (!origin)
+		return;
+
+	while (true) {
+		nr_entries = stack_depot_fetch(origin, &entries);
+		magic = nr_entries ? (entries[0] & KMSAN_MAGIC_MASK) : 0;
+		if ((nr_entries == 4) && (magic == KMSAN_ALLOCA_MAGIC_ORIGIN)) {
+			descr = (char *)entries[1];
+			pc1 = (void *)entries[2];
+			pc2 = (void *)entries[3];
+			pr_err("Local variable %s created at:\n", descr);
+			pr_err(" %pS\n", pc1);
+			pr_err(" %pS\n", pc2);
+			break;
+		}
+		if ((nr_entries == 3) &&
+		    (magic == KMSAN_CHAIN_MAGIC_ORIGIN_FULL)) {
+			head = entries[1];
+			origin = entries[2];
+			pr_err("Uninit was stored to memory at:\n");
+			chained_nr_entries =
+				stack_depot_fetch(head, &chained_entries);
+			stack_trace_print(chained_entries, chained_nr_entries,
+					  0);
+			pr_err("\n");
+			continue;
+		}
+		pr_err("Uninit was created at:\n");
+		if (nr_entries)
+			stack_trace_print(entries, nr_entries, 0);
+		else
+			pr_err("(stack is not available)\n");
+		break;
+	}
+}
+
+/**
+ * kmsan_report() - Report a use of uninitialized value.
+ * @origin:    Stack ID of the uninitialized value.
+ * @address:   Address at which the memory access happens.
+ * @size:      Memory access size.
+ * @off_first: Offset (from @address) of the first byte to be reported.
+ * @off_last:  Offset (from @address) of the last byte to be reported.
+ * @user_addr: When non-NULL, denotes the userspace address to which the kernel
+ *             is leaking data.
+ * @reason:    Error type from KMSAN_BUG_REASON enum.
+ *
+ * kmsan_report() prints an error message for a consequent group of bytes
+ * sharing the same origin. If an uninitialized value is used in a comparison,
+ * this function is called once without specifying the addresses. When checking
+ * a memory range, KMSAN may call kmsan_report() multiple times with the same
+ * @address, @size, @user_addr and @reason, but different @off_first and
+ * @off_last corresponding to different @origin values.
+ */
+void kmsan_report(depot_stack_handle_t origin,
+		  void *address, int size, int off_first, int off_last,
+		  const void *user_addr, int reason)
+{
+	unsigned long flags;
+	bool is_uaf;
+	char *bug_type = NULL;
+
+	if (!kmsan_ready)
+		return;
+	if (!current->kmsan.allow_reporting)
+		return;
+	if (!origin)
+		return;
+
+	current->kmsan.allow_reporting = false;
+	spin_lock_irqsave(&report_lock, flags);
+	pr_err("=====================================================\n");
+	is_uaf = kmsan_uaf_from_eb(get_dsh_extra_bits(origin));
+	switch (reason) {
+	case REASON_ANY:
+		bug_type = is_uaf ? "use-after-free" : "uninit-value";
+		break;
+	case REASON_COPY_TO_USER:
+		bug_type = is_uaf ? "kernel-infoleak-after-free" :
+				    "kernel-infoleak";
+		break;
+	case REASON_SUBMIT_URB:
+		bug_type = is_uaf ? "kernel-usb-infoleak-after-free" :
+				    "kernel-usb-infoleak";
+		break;
+	}
+	pr_err("BUG: KMSAN: %s in %pS\n",
+	       bug_type, kmsan_internal_return_address(2));
+	dump_stack();
+	pr_err("\n");
+
+	kmsan_print_origin(origin);
+
+	if (size) {
+		pr_err("\n");
+		if (off_first == off_last)
+			pr_err("Byte %d of %d is uninitialized\n",
+			       off_first, size);
+		else
+			pr_err("Bytes %d-%d of %d are uninitialized\n",
+			       off_first, off_last, size);
+	}
+	if (address)
+		pr_err("Memory access of size %d starts at %px\n",
+		       size, address);
+	if (user_addr && reason == REASON_COPY_TO_USER)
+		pr_err("Data copied to user address %px\n", user_addr);
+	pr_err("=====================================================\n");
+	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
+	spin_unlock_irqrestore(&report_lock, flags);
+	if (panic_on_warn)
+		panic("panic_on_warn set ...\n");
+	current->kmsan.allow_reporting = true;
+}
diff --git a/mm/kmsan/kmsan_shadow.c b/mm/kmsan/kmsan_shadow.c
new file mode 100644
index 000000000000..bcd4f1faa7a6
--- /dev/null
+++ b/mm/kmsan/kmsan_shadow.c
@@ -0,0 +1,456 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * KMSAN shadow implementation.
+ *
+ * Copyright (C) 2017-2019 Google LLC
+ * Author: Alexander Potapenko <glider@google.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+#include <asm/cpu_entry_area.h>
+#include <asm/page.h>
+#include <asm/pgtable_64_types.h>
+#include <asm/tlbflush.h>
+#include <linux/memblock.h>
+#include <linux/mm_types.h>
+#include <linux/percpu-defs.h>
+#include <linux/slab.h>
+#include <linux/smp.h>
+#include <linux/stddef.h>
+
+#include "kmsan.h"
+#include "kmsan_shadow.h"
+
+#define shadow_page_for(page) ((page)->shadow)
+
+#define origin_page_for(page) ((page)->origin)
+
+#define shadow_ptr_for(page) (page_address((page)->shadow))
+
+#define origin_ptr_for(page) (page_address((page)->origin))
+
+#define has_shadow_page(page) (!!((page)->shadow))
+
+#define has_origin_page(page) (!!((page)->origin))
+
+#define set_no_shadow_origin_page(page)	\
+	do {				\
+		(page)->shadow = NULL;	\
+		(page)->origin = NULL;	\
+	} while (0) /**/
+
+#define is_ignored_page(page) (!!(((u64)((page)->shadow)) % 2))
+
+#define ignore_page(pg)	\
+	((pg)->shadow = (struct page *)((u64)((pg)->shadow) | 1))
+
+DEFINE_PER_CPU(char[CPU_ENTRY_AREA_SIZE], cpu_entry_area_shadow);
+DEFINE_PER_CPU(char[CPU_ENTRY_AREA_SIZE], cpu_entry_area_origin);
+
+/*
+ * Dummy load and store pages to be used when the real metadata is unavailable.
+ * There are separate pages for loads and stores, so that every load returns a
+ * zero, and every store doesn't affect other loads.
+ */
+char dummy_load_page[PAGE_SIZE] __aligned(PAGE_SIZE);
+char dummy_store_page[PAGE_SIZE] __aligned(PAGE_SIZE);
+
+/*
+ * Taken from arch/x86/mm/physaddr.h to avoid using an instrumented version.
+ */
+static int kmsan_phys_addr_valid(unsigned long addr)
+{
+#ifdef CONFIG_PHYS_ADDR_T_64BIT
+	return !(addr >> boot_cpu_data.x86_phys_bits);
+#else
+	return 1;
+#endif
+}
+
+/*
+ * Taken from arch/x86/mm/physaddr.c to avoid using an instrumented version.
+ */
+static bool kmsan_virt_addr_valid(void *addr)
+{
+	unsigned long x = (unsigned long)addr;
+	unsigned long y = x - __START_KERNEL_map;
+
+	/* use the carry flag to determine if x was < __START_KERNEL_map */
+	if (unlikely(x > y)) {
+		x = y + phys_base;
+
+		if (y >= KERNEL_IMAGE_SIZE)
+			return false;
+	} else {
+		x = y + (__START_KERNEL_map - PAGE_OFFSET);
+
+		/* carry flag will be set if starting x was >= PAGE_OFFSET */
+		if ((x > y) || !kmsan_phys_addr_valid(x))
+			return false;
+	}
+
+	return pfn_valid(x >> PAGE_SHIFT);
+}
+
+static unsigned long vmalloc_meta(void *addr, bool is_origin)
+{
+	unsigned long addr64 = (unsigned long)addr, off;
+
+	BUG_ON(is_origin && !IS_ALIGNED(addr64, ORIGIN_SIZE));
+	if (kmsan_internal_is_vmalloc_addr(addr))
+		return addr64 + (is_origin ? VMALLOC_ORIGIN_OFFSET
+					   : VMALLOC_SHADOW_OFFSET);
+	if (kmsan_internal_is_module_addr(addr)) {
+		off = addr64 - MODULES_VADDR;
+		return off + (is_origin ? MODULES_ORIGIN_START
+					: MODULES_SHADOW_START);
+	}
+	return 0;
+}
+
+static void *get_cea_meta_or_null(void *addr, bool is_origin)
+{
+	int cpu = smp_processor_id();
+	int off;
+	char *metadata_array;
+
+	if (((u64)addr < CPU_ENTRY_AREA_BASE) ||
+	    ((u64)addr >= (CPU_ENTRY_AREA_BASE + CPU_ENTRY_AREA_MAP_SIZE)))
+		return NULL;
+	off = (char *)addr - (char *)get_cpu_entry_area(cpu);
+	if ((off < 0) || (off >= CPU_ENTRY_AREA_SIZE))
+		return NULL;
+	metadata_array = is_origin ? cpu_entry_area_origin :
+				     cpu_entry_area_shadow;
+	return &per_cpu(metadata_array[off], cpu);
+}
+
+static struct page *virt_to_page_or_null(void *vaddr)
+{
+	if (kmsan_virt_addr_valid(vaddr))
+		return virt_to_page(vaddr);
+	else
+		return NULL;
+}
+
+struct shadow_origin_ptr kmsan_get_shadow_origin_ptr(void *address, u64 size,
+						     bool store)
+{
+	struct shadow_origin_ptr ret;
+	void *shadow;
+
+	if (size > PAGE_SIZE)
+		panic("size too big in %s(%px, %d, %d)\n",
+		     __func__, address, size, store);
+
+	if (!kmsan_ready || kmsan_in_runtime())
+		goto return_dummy;
+
+	BUG_ON(!metadata_is_contiguous(address, size, META_SHADOW));
+	shadow = kmsan_get_metadata(address, size, META_SHADOW);
+	if (!shadow)
+		goto return_dummy;
+
+	ret.s = shadow;
+	ret.o = kmsan_get_metadata(address, size, META_ORIGIN);
+	return ret;
+
+return_dummy:
+	if (store) {
+		ret.s = dummy_store_page;
+		ret.o = dummy_store_page;
+	} else {
+		ret.s = dummy_load_page;
+		ret.o = dummy_load_page;
+	}
+	return ret;
+}
+
+/*
+ * Obtain the shadow or origin pointer for the given address, or NULL if there's
+ * none. The caller must check the return value for being non-NULL if needed.
+ * The return value of this function should not depend on whether we're in the
+ * runtime or not.
+ */
+void *kmsan_get_metadata(void *address, size_t size, bool is_origin)
+{
+	struct page *page;
+	void *ret;
+	u64 addr = (u64)address, pad, off;
+
+	if (is_origin && !IS_ALIGNED(addr, ORIGIN_SIZE)) {
+		pad = addr % ORIGIN_SIZE;
+		addr -= pad;
+		size += pad;
+	}
+	address = (void *)addr;
+	if (kmsan_internal_is_vmalloc_addr(address) ||
+	    kmsan_internal_is_module_addr(address))
+		return (void *)vmalloc_meta(address, is_origin);
+
+	ret = get_cea_meta_or_null(address, is_origin);
+	if (ret)
+		return ret;
+
+	page = virt_to_page_or_null(address);
+	if (!page)
+		return NULL;
+	if (is_ignored_page(page))
+		return NULL;
+	if (!has_shadow_page(page) || !has_origin_page(page))
+		return NULL;
+	off = addr % PAGE_SIZE;
+
+	ret = (is_origin ? origin_ptr_for(page) : shadow_ptr_for(page)) + off;
+	return ret;
+}
+
+void __init kmsan_init_alloc_meta_for_range(void *start, void *end)
+{
+	u64 addr, size;
+	struct page *page;
+	void *shadow, *origin;
+	struct page *shadow_p, *origin_p;
+
+	start = (void *)ALIGN_DOWN((u64)start, PAGE_SIZE);
+	size = ALIGN((u64)end - (u64)start, PAGE_SIZE);
+	shadow = memblock_alloc(size, PAGE_SIZE);
+	origin = memblock_alloc(size, PAGE_SIZE);
+	for (addr = 0; addr < size; addr += PAGE_SIZE) {
+		page = virt_to_page_or_null((char *)start + addr);
+		shadow_p = virt_to_page_or_null((char *)shadow + addr);
+		set_no_shadow_origin_page(shadow_p);
+		shadow_page_for(page) = shadow_p;
+		origin_p = virt_to_page_or_null((char *)origin + addr);
+		set_no_shadow_origin_page(origin_p);
+		origin_page_for(page) = origin_p;
+	}
+}
+
+/* Called from mm/memory.c */
+void kmsan_copy_page_meta(struct page *dst, struct page *src)
+{
+	unsigned long irq_flags;
+
+	if (!kmsan_ready || kmsan_in_runtime())
+		return;
+	if (!has_shadow_page(src)) {
+		kmsan_internal_unpoison_shadow(page_address(dst), PAGE_SIZE,
+					       /*checked*/false);
+		return;
+	}
+	if (!has_shadow_page(dst))
+		return;
+	if (is_ignored_page(src)) {
+		ignore_page(dst);
+		return;
+	}
+
+	irq_flags = kmsan_enter_runtime();
+	__memcpy(shadow_ptr_for(dst), shadow_ptr_for(src),
+		PAGE_SIZE);
+	BUG_ON(!has_origin_page(src) || !has_origin_page(dst));
+	__memcpy(origin_ptr_for(dst), origin_ptr_for(src),
+		PAGE_SIZE);
+	kmsan_leave_runtime(irq_flags);
+}
+EXPORT_SYMBOL(kmsan_copy_page_meta);
+
+/* Helper function to allocate page metadata. */
+static int kmsan_internal_alloc_meta_for_pages(struct page *page,
+					       unsigned int order,
+					       gfp_t flags, int node)
+{
+	struct page *shadow, *origin;
+	int pages = 1 << order;
+	int i;
+	bool initialized = (flags & __GFP_ZERO) || !kmsan_ready;
+	depot_stack_handle_t handle;
+
+	if (flags & __GFP_NO_KMSAN_SHADOW) {
+		for (i = 0; i < pages; i++)
+			set_no_shadow_origin_page(&page[i]);
+		return 0;
+	}
+
+	/*
+	 * We always want metadata allocations to succeed and to finish fast.
+	 */
+	flags = GFP_ATOMIC;
+	if (initialized)
+		flags |= __GFP_ZERO;
+	shadow = alloc_pages_node(node, flags | __GFP_NO_KMSAN_SHADOW, order);
+	if (!shadow) {
+		for (i = 0; i < pages; i++) {
+			set_no_shadow_origin_page(&page[i]);
+			set_no_shadow_origin_page(&page[i]);
+		}
+		return -ENOMEM;
+	}
+	if (!initialized)
+		__memset(page_address(shadow), -1, PAGE_SIZE * pages);
+
+	origin = alloc_pages_node(node, flags | __GFP_NO_KMSAN_SHADOW, order);
+	/* Assume we've allocated the origin. */
+	if (!origin) {
+		__free_pages(shadow, order);
+		for (i = 0; i < pages; i++)
+			set_no_shadow_origin_page(&page[i]);
+		return -ENOMEM;
+	}
+
+	if (!initialized) {
+		handle = kmsan_save_stack_with_flags(flags, /*extra_bits*/0);
+		/*
+		 * Addresses are page-aligned, pages are contiguous, so it's ok
+		 * to just fill the origin pages with |handle|.
+		 */
+		for (i = 0; i < PAGE_SIZE * pages / sizeof(handle); i++) {
+			((depot_stack_handle_t *)page_address(origin))[i] =
+						handle;
+		}
+	}
+
+	for (i = 0; i < pages; i++) {
+		shadow_page_for(&page[i]) = &shadow[i];
+		set_no_shadow_origin_page(shadow_page_for(&page[i]));
+		origin_page_for(&page[i]) = &origin[i];
+		set_no_shadow_origin_page(origin_page_for(&page[i]));
+	}
+	return 0;
+}
+
+/* Called from mm/page_alloc.c */
+int kmsan_alloc_page(struct page *page, unsigned int order, gfp_t flags)
+{
+	unsigned long irq_flags;
+	int ret;
+
+	if (kmsan_in_runtime())
+		return 0;
+	irq_flags = kmsan_enter_runtime();
+	ret = kmsan_internal_alloc_meta_for_pages(page, order, flags, -1);
+	kmsan_leave_runtime(irq_flags);
+	return ret;
+}
+
+/* Called from mm/page_alloc.c */
+void kmsan_free_page(struct page *page, unsigned int order)
+{
+	struct page *shadow, *origin, *cur_page;
+	int pages = 1 << order;
+	int i;
+	unsigned long irq_flags;
+
+	if (!shadow_page_for(page)) {
+		for (i = 0; i < pages; i++) {
+			cur_page = &page[i];
+			BUG_ON(shadow_page_for(cur_page));
+		}
+		return;
+	}
+
+	if (!kmsan_ready) {
+		for (i = 0; i < pages; i++) {
+			cur_page = &page[i];
+			set_no_shadow_origin_page(cur_page);
+		}
+		return;
+	}
+
+	irq_flags = kmsan_enter_runtime();
+	shadow = shadow_page_for(&page[0]);
+	origin = origin_page_for(&page[0]);
+
+	for (i = 0; i < pages; i++) {
+		BUG_ON(has_shadow_page(shadow_page_for(&page[i])));
+		BUG_ON(has_shadow_page(origin_page_for(&page[i])));
+		set_no_shadow_origin_page(&page[i]);
+	}
+	BUG_ON(has_shadow_page(shadow));
+	__free_pages(shadow, order);
+
+	BUG_ON(has_shadow_page(origin));
+	__free_pages(origin, order);
+	kmsan_leave_runtime(irq_flags);
+}
+EXPORT_SYMBOL(kmsan_free_page);
+
+/* Called from mm/page_alloc.c */
+void kmsan_split_page(struct page *page, unsigned int order)
+{
+	struct page *shadow, *origin;
+	unsigned long irq_flags;
+
+	if (!kmsan_ready || kmsan_in_runtime())
+		return;
+
+	irq_flags = kmsan_enter_runtime();
+	if (!has_shadow_page(&page[0])) {
+		BUG_ON(has_origin_page(&page[0]));
+		kmsan_leave_runtime(irq_flags);
+		return;
+	}
+	shadow = shadow_page_for(&page[0]);
+	split_page(shadow, order);
+
+	origin = origin_page_for(&page[0]);
+	split_page(origin, order);
+	kmsan_leave_runtime(irq_flags);
+}
+EXPORT_SYMBOL(kmsan_split_page);
+
+/* Called from mm/vmalloc.c */
+void kmsan_vmap_page_range_noflush(unsigned long start, unsigned long end,
+				   pgprot_t prot, struct page **pages)
+{
+	int nr, i, mapped;
+	struct page **s_pages, **o_pages;
+	unsigned long shadow_start, shadow_end, origin_start, origin_end;
+
+	if (!kmsan_ready || kmsan_in_runtime())
+		return;
+	shadow_start = vmalloc_meta((void *)start, META_SHADOW);
+	if (!shadow_start)
+		return;
+
+	BUG_ON(start >= end);
+	nr = (end - start) / PAGE_SIZE;
+	s_pages = kcalloc(nr, sizeof(struct page *), GFP_KERNEL);
+	o_pages = kcalloc(nr, sizeof(struct page *), GFP_KERNEL);
+	if (!s_pages || !o_pages)
+		goto ret;
+	for (i = 0; i < nr; i++) {
+		s_pages[i] = shadow_page_for(pages[i]);
+		o_pages[i] = origin_page_for(pages[i]);
+	}
+	prot = __pgprot(pgprot_val(prot) | _PAGE_NX);
+	prot = PAGE_KERNEL;
+
+	shadow_end = vmalloc_meta((void *)end, META_SHADOW);
+	origin_start = vmalloc_meta((void *)start, META_ORIGIN);
+	origin_end = vmalloc_meta((void *)end, META_ORIGIN);
+	mapped = __vmap_page_range_noflush(shadow_start, shadow_end,
+					   prot, s_pages);
+	BUG_ON(mapped != nr);
+	flush_tlb_kernel_range(shadow_start, shadow_end);
+	mapped = __vmap_page_range_noflush(origin_start, origin_end,
+					   prot, o_pages);
+	BUG_ON(mapped != nr);
+	flush_tlb_kernel_range(origin_start, origin_end);
+ret:
+	kfree(s_pages);
+	kfree(o_pages);
+}
+
+void kmsan_ignore_page(struct page *page, int order)
+{
+	int i;
+
+	for (i = 0; i < 1 << order; i++)
+		ignore_page(&page[i]);
+}
diff --git a/mm/kmsan/kmsan_shadow.h b/mm/kmsan/kmsan_shadow.h
new file mode 100644
index 000000000000..eaa7f771b6a5
--- /dev/null
+++ b/mm/kmsan/kmsan_shadow.h
@@ -0,0 +1,30 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * KMSAN shadow API.
+ *
+ * This should be agnostic to shadow implementation details.
+ *
+ * Copyright (C) 2017-2019 Google LLC
+ * Author: Alexander Potapenko <glider@google.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+#ifndef __MM_KMSAN_KMSAN_SHADOW_H
+#define __MM_KMSAN_KMSAN_SHADOW_H
+
+#include <asm/cpu_entry_area.h>  /* for CPU_ENTRY_AREA_MAP_SIZE */
+
+struct shadow_origin_ptr {
+	void *s, *o;
+};
+
+struct shadow_origin_ptr kmsan_get_shadow_origin_ptr(void *addr, u64 size,
+						     bool store);
+void *kmsan_get_metadata(void *addr, size_t size, bool is_origin);
+void __init kmsan_init_alloc_meta_for_range(void *start, void *end);
+
+#endif  /* __MM_KMSAN_KMSAN_SHADOW_H */
diff --git a/scripts/Makefile.kmsan b/scripts/Makefile.kmsan
new file mode 100644
index 000000000000..8b3844b66b22
--- /dev/null
+++ b/scripts/Makefile.kmsan
@@ -0,0 +1,12 @@
+ifdef CONFIG_KMSAN
+
+CFLAGS_KMSAN := -fsanitize=kernel-memory
+
+ifeq ($(call cc-option, $(CFLAGS_KMSAN) -Werror),)
+   ifneq ($(CONFIG_COMPILE_TEST),y)
+        $(warning Cannot use CONFIG_KMSAN: \
+            -fsanitize=kernel-memory is not supported by compiler)
+   endif
+endif
+
+endif
-- 
2.24.1.735.g03f4e72817-goog