[RFC 5/9] Popcorn Utility

[Javier Malave <javier.malave@narfindustries.com>]

This patch contains utility and other
Popcorn supplementary modules, including
initialization routines.
---
 include/popcorn/bundle.h      |  38 ++++
 include/popcorn/debug.h       |  38 ++++
 include/popcorn/regset.h      |  96 +++++++++
 include/popcorn/stat.h        |  16 ++
 include/popcorn/types.h       |  20 ++
 kernel/popcorn/Makefile       |   7 +
 kernel/popcorn/bundle.c       | 115 +++++++++++
 kernel/popcorn/init.c         |  58 ++++++
 kernel/popcorn/stat.c         | 165 ++++++++++++++++
 kernel/popcorn/trace_events.h |  76 ++++++++
 kernel/popcorn/types.h        | 358 ++++++++++++++++++++++++++++++++++
 kernel/popcorn/util.c         | 121 ++++++++++++
 kernel/popcorn/util.h         |  14 ++
 kernel/popcorn/wait_station.c |  84 ++++++++
 kernel/popcorn/wait_station.h |  27 +++
 15 files changed, 1233 insertions(+)
 create mode 100644 include/popcorn/bundle.h
 create mode 100644 include/popcorn/debug.h
 create mode 100644 include/popcorn/regset.h
 create mode 100644 include/popcorn/stat.h
 create mode 100644 include/popcorn/types.h
 create mode 100644 kernel/popcorn/Makefile
 create mode 100644 kernel/popcorn/bundle.c
 create mode 100644 kernel/popcorn/init.c
 create mode 100644 kernel/popcorn/stat.c
 create mode 100644 kernel/popcorn/trace_events.h
 create mode 100644 kernel/popcorn/types.h
 create mode 100644 kernel/popcorn/util.c
 create mode 100644 kernel/popcorn/util.h
 create mode 100644 kernel/popcorn/wait_station.c
 create mode 100644 kernel/popcorn/wait_station.h

diff --git a/include/popcorn/bundle.h b/include/popcorn/bundle.h
new file mode 100644
index 000000000..280325e10
--- /dev/null
+++ b/include/popcorn/bundle.h
@@ -0,0 +1,38 @@
+// SPDX-License-Identifier: GPL-2.0, 3-clause BSD
+#ifndef __POPCORN_RACK_H__
+#define __POPCORN_RACK_H__
+
+#define MAX_POPCORN_NODES 32
+#if (MAX_POPCORN_NODES > 62)
+#error Currently support up to 62 nodes
+#endif
+
+enum popcorn_arch {
+	POPCORN_ARCH_UNKNOWN = -1,
+	POPCORN_ARCH_X86 = 1,
+	POPCORN_ARCH_PPC = 2,
+	POPCORN_ARCH_MAX,
+};
+
+extern int my_nid;
+extern const enum popcorn_arch my_arch;
+
+bool get_popcorn_node_online(int nid);
+int get_popcorn_node_arch(int nid);
+int popcorn_nodes_init(void);
+void set_popcorn_node_online(int nid, bool online);
+void broadcast_my_node_info(int nr_nodes);
+
+struct popcorn_thread_status {
+	int current_nid;
+	int peer_nid;
+	pid_t peer_pid;
+};
+
+struct popcorn_node_info {
+	unsigned int status;
+	int arch;
+	int distance;
+};
+
+#endif
diff --git a/include/popcorn/debug.h b/include/popcorn/debug.h
new file mode 100644
index 000000000..4a12a7ba7
--- /dev/null
+++ b/include/popcorn/debug.h
@@ -0,0 +1,38 @@
+// SPDX-License-Identifier: GPL-2.0, 3-clause BSD
+#ifndef __INCLUDE_POPCORN_DEBUG_H__
+#define __INCLUDE_POPCORN_DEBUG_H__
+
+#define PCNPRINTK(...) printk(KERN_INFO "popcorn: " __VA_ARGS__)
+#define PCNPRINTK_ERR(...) printk(KERN_ERR "popcorn: " __VA_ARGS__)
+
+#ifdef CONFIG_POPCORN_DEBUG
+#define PRINTK(...) printk(KERN_INFO __VA_ARGS__)
+#else
+#define PRINTK(...)
+#endif
+
+#ifdef CONFIG_POPCORN_DEBUG_PROCESS_SERVER
+#define PSPRINTK(...) printk(KERN_INFO __VA_ARGS__)
+#else
+#define PSPRINTK(...)
+#endif
+
+#ifdef CONFIG_POPCORN_DEBUG_VMA_SERVER
+#define VSPRINTK(...) printk(KERN_INFO __VA_ARGS__)
+#else
+#define VSPRINTK(...)
+#endif
+
+#ifdef CONFIG_POPCORN_DEBUG_PAGE_SERVER
+#define PGPRINTK(...) printk(KERN_INFO __VA_ARGS__)
+#else
+#define PGPRINTK(...)
+#endif
+
+#ifdef CONFIG_POPCORN_DEBUG_MSG_LAYER
+#define MSGPRINTK(...) printk(KERN_INFO __VA_ARGS__)
+#else
+#define MSGPRINTK(...)
+#endif
+
+#endif /*  __INCLUDE_POPCORN_DEBUG_H__ */
diff --git a/include/popcorn/regset.h b/include/popcorn/regset.h
new file mode 100644
index 000000000..c13c90525
--- /dev/null
+++ b/include/popcorn/regset.h
@@ -0,0 +1,96 @@
+// SPDX-License-Identifier: GPL-2.0, 3-clause BSD
+/*
+ * /include/popcorn/regset.h
+ *
+ *  This file provides the architecture specific macro and structures of the
+ *  helper functionality implementation of the process server
+ *
+ * author, Sharath Kumar Bhat, SSRG, VirginiaTech 2014
+ *
+ */
+
+#ifndef PROCESS_SERVER_ARCH_MACROS_H_
+#define PROCESS_SERVER_ARCH_MACROS_H_
+
+#include <popcorn/bundle.h>
+
+struct regset_x86_64 {
+	/* Program counter/instruction pointer */
+	uint64_t rip;
+
+	/* General purpose registers */
+	uint64_t rax, rdx, rcx, rbx,
+			 rsi, rdi, rbp, rsp,
+			 r8, r9, r10, r11,
+			 r12, r13, r14, r15;
+
+	/* Multimedia-extension (MMX) registers */
+	uint64_t mmx[8];
+
+	/* Streaming SIMD Extension (SSE) registers */
+	unsigned __int128 xmm[16];
+
+	/* x87 floating point registers */
+	long double st[8];
+
+	/* Segment registers */
+	uint32_t cs, ss, ds, es, fs, gs;
+
+	/* Flag register */
+	uint64_t rflags;
+};
+
+struct regset_aarch64 {
+	/* Stack pointer & program counter */
+	uint64_t sp;
+	uint64_t pc;
+
+	/* General purpose registers */
+	uint64_t x[31];
+
+	/* FPU/SIMD registers */
+	unsigned __int128 v[32];
+};
+
+struct regset_powerpc {
+	unsigned long nip;
+	unsigned long msr;
+	unsigned long ctr;
+	unsigned long link;
+	unsigned long xer;
+	unsigned long ccr;
+
+	unsigned long gpr[32];
+	uint64_t fpr[32];
+
+	unsigned long orig_gpr3;	/* Used for restarting system calls */
+	unsigned long softe;		/* Soft enabled/disabled */
+};
+
+struct field_arch {
+	unsigned long tls;
+	unsigned long oob[4];
+	bool fpu_active;
+
+	union {
+		unsigned long regsets;
+		struct regset_x86_64 regs_x86;
+		struct regset_aarch64 regs_aarch;
+		struct regset_powerpc regs_ppc;
+	};
+};
+
+static inline size_t regset_size(int arch) {
+	const size_t sizes[] = {
+		sizeof(struct regset_aarch64),
+		sizeof(struct regset_x86_64),
+		sizeof(struct regset_powerpc),
+	};
+
+	if(arch <= POPCORN_ARCH_UNKNOWN || arch >= POPCORN_ARCH_MAX)
+		return -EINVAL;
+
+	return sizes[arch];
+}
+
+#endif
diff --git a/include/popcorn/stat.h b/include/popcorn/stat.h
new file mode 100644
index 000000000..cbd024314
--- /dev/null
+++ b/include/popcorn/stat.h
@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: GPL-2.0, 3-clause BSD
+#ifndef __KERNEL_POPCORN_STAT_H__
+#define __KERNEL_POPCORN_STAT_H__
+
+struct pcn_kmsg_message;
+
+void account_pcn_message_sent(struct pcn_kmsg_message *msg);
+void account_pcn_message_recv(struct pcn_kmsg_message *msg);
+
+void account_pcn_rdma_write(size_t size);
+void account_pcn_rdma_read(size_t size);
+
+#define POPCORN_STAT_FMT  "%12llu  %12llu  %s\n"
+#define POPCORN_STAT_FMT2 "%8llu.%03llu  %8llu.%03llu  %s\n"
+
+#endif /* KERNEL_POPCORN_STAT_H_ */
diff --git a/include/popcorn/types.h b/include/popcorn/types.h
new file mode 100644
index 000000000..aaab9d923
--- /dev/null
+++ b/include/popcorn/types.h
@@ -0,0 +1,20 @@
+// SPDX-License-Identifier: GPL-2.0, 3-clause BSD
+#ifndef __INCLUDE_POPCORN_TYPES_H__
+#define __INCLUDE_POPCORN_TYPES_H__
+
+#include <linux/sched.h>
+
+static inline bool distributed_process(struct task_struct *tsk)
+{
+	if (!tsk->mm) return false;
+	return !!tsk->mm->remote;
+}
+
+static inline bool distributed_remote_process(struct task_struct *tsk)
+{
+	return distributed_process(tsk) && tsk->at_remote;
+}
+
+#include <popcorn/debug.h>
+
+#endif /* __INCLUDE_POPCORN_TYPES_H__ */
diff --git a/kernel/popcorn/Makefile b/kernel/popcorn/Makefile
new file mode 100644
index 000000000..6821ec217
--- /dev/null
+++ b/kernel/popcorn/Makefile
@@ -0,0 +1,7 @@
+obj-$(CONFIG_POPCORN)	+= init.o util.o
+obj-$(CONFIG_POPCORN)	+= wait_station.o
+obj-$(CONFIG_POPCORN)	+= process_server.o vma_server.o
+obj-$(CONFIG_POPCORN)	+= page_server.o fh_action.o
+obj-$(CONFIG_POPCORN)	+= bundle.o
+obj-$(CONFIG_POPCORN)	+= pcn_kmsg.o
+obj-$(CONFIG_POPCORN)	+= stat.o
diff --git a/kernel/popcorn/bundle.c b/kernel/popcorn/bundle.c
new file mode 100644
index 000000000..54ffb59e1
--- /dev/null
+++ b/kernel/popcorn/bundle.c
@@ -0,0 +1,115 @@
+// SPDX-License-Identifier: GPL-2.0, BSD
+/*
+ * /kernel/popcorn/bundle.c
+ *
+ * Popcorn node init
+ *
+ * Original file developed by SSRG at Virginia Tech.
+ *
+ * author, Javier Malave, Rebecca Shapiro, Andrew Hughes,
+ * Narf Industries 2020 (modifications for upstream RFC)
+ */
+
+#include <asm/bug.h>
+#include <linux/kernel.h>
+#include <linux/errno.h>
+#include <linux/init.h>
+#include <linux/list.h>
+#include <linux/spinlock.h>
+#include <linux/mm.h>
+
+#include <popcorn/pcn_kmsg.h>
+#include <popcorn/bundle.h>
+#include <popcorn/debug.h>
+#include "types.h"
+
+struct popcorn_node {
+	enum popcorn_arch arch;
+	int bundle_id;
+
+	bool is_connected;
+};
+
+static struct popcorn_node popcorn_nodes[MAX_POPCORN_NODES];
+
+bool get_popcorn_node_online(int nid)
+{
+	return popcorn_nodes[nid].is_connected;
+}
+EXPORT_SYMBOL(get_popcorn_node_online);
+
+void set_popcorn_node_online(int nid, bool online)
+{
+        popcorn_nodes[nid].is_connected = online;
+}
+EXPORT_SYMBOL(set_popcorn_node_online);
+
+int my_nid __read_mostly = -1;
+EXPORT_SYMBOL(my_nid);
+
+const enum popcorn_arch my_arch = POPCORN_ARCH_X86;
+EXPORT_SYMBOL(my_arch);
+
+int get_popcorn_node_arch(int nid)
+{
+	return popcorn_nodes[nid].arch;
+}
+EXPORT_SYMBOL(get_popcorn_node_arch);
+
+const char *archs_sz[] = {
+	"aarch64",
+	"x86_64",
+	"ppc64le",
+};
+
+void broadcast_my_node_info(int nr_nodes)
+{
+	int i;
+	node_info_t info = {
+		.nid = my_nid,
+		.arch = my_arch,
+	};
+	for (i = 0; i < nr_nodes; i++) {
+		if (i == my_nid)
+                        continue;
+		pcn_kmsg_send(PCN_KMSG_TYPE_NODE_INFO, i, &info, sizeof(info));
+	}
+}
+EXPORT_SYMBOL(broadcast_my_node_info);
+
+static bool my_node_info_printed = false;
+
+static int handle_node_info(struct pcn_kmsg_message *msg)
+{
+	node_info_t *info = (node_info_t *)msg;
+
+	if (my_nid != -1 && !my_node_info_printed) {
+		popcorn_nodes[my_nid].arch = my_arch;
+		my_node_info_printed = true;
+	}
+
+	PCNPRINTK("   %d joined, %s\n", info->nid, archs_sz[info->arch]);
+	popcorn_nodes[info->nid].arch = info->arch;
+	smp_mb();
+
+	pcn_kmsg_done(msg);
+	return 0;
+}
+
+int __init popcorn_nodes_init(void)
+{
+	int i;
+	BUG_ON(my_arch == POPCORN_ARCH_UNKNOWN);
+
+	for (i = 0; i < MAX_POPCORN_NODES; i++) {
+		struct popcorn_node *pn = popcorn_nodes + i;
+
+		pn->is_connected = false;
+		pn->arch = POPCORN_ARCH_UNKNOWN;
+		pn->bundle_id = -1;
+	}
+
+	REGISTER_KMSG_HANDLER(PCN_KMSG_TYPE_NODE_INFO, node_info);
+
+	return 0;
+}
diff --git a/kernel/popcorn/init.c b/kernel/popcorn/init.c
new file mode 100644
index 000000000..a0cc9796f
--- /dev/null
+++ b/kernel/popcorn/init.c
@@ -0,0 +1,58 @@
+// SPDX-License-Identifier: GPL-2.0, BSD
+/*
+ * /kernel/popcorn/init.c
+ *
+ * Popcorn node init
+ *
+ * Copyright (c) 2013 - 2014 Akshay Giridhar
+ *
+ * author, Javier Malave, Rebecca Shapiro, Andrew Hughes,
+ * Narf Industries 2020 (modifications for upstream RFC)
+ * author, rewritten by Sang-Hoon Kim, 2016-2017
+ * author, modified by Antonio Barbalace, 2014
+ */
+
+#include <linux/kernel.h>
+#include <linux/workqueue.h>
+
+#include <popcorn/debug.h>
+#include "types.h"
+
+#define CREATE_TRACE_POINTS
+#include "trace_events.h"
+
+struct workqueue_struct *popcorn_wq;
+struct workqueue_struct *popcorn_ordered_wq;
+EXPORT_SYMBOL(popcorn_wq);
+EXPORT_SYMBOL(popcorn_ordered_wq);
+
+extern int pcn_kmsg_init(void);
+extern int popcorn_nodes_init(void);
+extern int process_server_init(void);
+extern int vma_server_init(void);
+extern int page_server_init(void);
+extern int statistics_init(void);
+
+static int __init popcorn_init(void)
+{
+	PRINTK("Initialize Popcorn subsystems...\n");
+
+	/*
+	 * Create work queues so that we can do bottom side
+	 * processing on data that was brought in by the
+	 * communications module interrupt handlers.
+	 */
+	popcorn_ordered_wq = create_singlethread_workqueue("pcn_wq_ordered");
+	popcorn_wq = alloc_workqueue("pcn_wq", WQ_MEM_RECLAIM, 0);
+
+	pcn_kmsg_init();
+
+	popcorn_nodes_init();
+	vma_server_init();
+	process_server_init();
+	page_server_init();
+
+	statistics_init();
+	return 0;
+}
+late_initcall(popcorn_init);
diff --git a/kernel/popcorn/stat.c b/kernel/popcorn/stat.c
new file mode 100644
index 000000000..55f05caf4
--- /dev/null
+++ b/kernel/popcorn/stat.c
@@ -0,0 +1,165 @@
+// SPDX-License-Identifier: GPL-2.0, BSD
+/*
+ * /kernel/popcorn/stat.c
+ *
+ * Original file developed by SSRG at Virginia Tech.
+ *
+ * author , Javier Malave, Rebecca Shapiro, Andrew Hughes,
+ * Narf Industries 2020 (modifications for upstream RFC)
+ */
+
+#include <linux/kernel.h>
+#include <linux/ktime.h>
+#include <linux/slab.h>
+#include <linux/proc_fs.h>
+#include <linux/seq_file.h>
+#include <linux/percpu.h>
+#include <asm/uaccess.h>
+
+#include <popcorn/pcn_kmsg.h>
+#include <popcorn/stat.h>
+
+static unsigned long long sent_stats[PCN_KMSG_TYPE_MAX] = {0};
+static unsigned long long recv_stats[PCN_KMSG_TYPE_MAX] = {0};
+
+static DEFINE_PER_CPU(unsigned long long, bytes_sent) = 0;
+static DEFINE_PER_CPU(unsigned long long, bytes_recv) = 0;
+static DEFINE_PER_CPU(unsigned long long, bytes_rdma_written) = 0;
+static DEFINE_PER_CPU(unsigned long long, bytes_rdma_read) = 0;
+
+static unsigned long long last_bytes_sent = 0;
+static unsigned long long last_bytes_recv = 0;
+static unsigned long long last_bytes_rdma_written = 0;
+static unsigned long long last_bytes_rdma_read = 0;
+static ktime_t last_stat = 0;
+
+const char *pcn_kmsg_type_name[PCN_KMSG_TYPE_MAX] = {
+	[PCN_KMSG_TYPE_TASK_MIGRATE] = "migration",
+	[PCN_KMSG_TYPE_VMA_INFO_REQUEST] = "VMA info",
+	[PCN_KMSG_TYPE_VMA_OP_REQUEST] = "VMA op",
+	[PCN_KMSG_TYPE_REMOTE_PAGE_REQUEST] = "remote page",
+	[PCN_KMSG_TYPE_PAGE_INVALIDATE_REQUEST] = "invalidate",
+	[PCN_KMSG_TYPE_FUTEX_REQUEST] = "futex",
+};
+
+void account_pcn_message_sent(struct pcn_kmsg_message *msg)
+{
+	struct pcn_kmsg_hdr *h = (struct pcn_kmsg_hdr *)msg;
+	this_cpu_add(bytes_sent, h->size);
+}
+
+void account_pcn_message_recv(struct pcn_kmsg_message *msg)
+{
+	struct pcn_kmsg_hdr *h = (struct pcn_kmsg_hdr *)msg;
+	this_cpu_add(bytes_recv, h->size);
+}
+
+void account_pcn_rdma_write(size_t size)
+{
+	this_cpu_add(bytes_rdma_written, size);
+}
+
+void account_pcn_rdma_read(size_t size)
+{
+	this_cpu_add(bytes_rdma_read, size);
+}
+
+void fh_action_stat(struct seq_file *seq, void *);
+
+static int __show_stats(struct seq_file *seq, void *v)
+{
+	int i;
+	unsigned long long sent = 0;
+	unsigned long long recv = 0;
+	ktime_t now;
+	unsigned long long rate_sent, rate_recv;
+	unsigned long elapsed;
+
+	now = ktime_get_real();
+	elapsed = last_stat - now;
+	last_stat = now;
+
+	for_each_present_cpu(i) {
+		sent += per_cpu(bytes_sent, i);
+		recv += per_cpu(bytes_recv, i);
+	}
+	seq_printf(seq, POPCORN_STAT_FMT, sent, recv, "Total network I/O");
+
+	rate_sent = (sent - last_bytes_sent);
+	rate_recv = (recv - last_bytes_recv);
+	seq_printf(seq, POPCORN_STAT_FMT2,
+			rate_sent / elapsed, (rate_sent % elapsed) * 1000 / elapsed,
+			rate_recv / elapsed, (rate_recv % elapsed) * 1000 / elapsed,
+			"MB/s");
+	last_bytes_sent = sent;
+	last_bytes_recv = recv;
+
+	if (pcn_kmsg_has_features(PCN_KMSG_FEATURE_RDMA) && elapsed) {
+		recv = sent = 0;
+		for_each_present_cpu(i) {
+			sent += per_cpu(bytes_rdma_written, i);
+			recv += per_cpu(bytes_rdma_read, i);
+		}
+		seq_printf(seq, POPCORN_STAT_FMT, sent, recv, "RDMA");
+
+		rate_sent = (sent - last_bytes_rdma_written);
+		rate_recv = (recv - last_bytes_rdma_read);
+		seq_printf(seq, POPCORN_STAT_FMT2,
+				rate_sent / elapsed, (rate_sent % elapsed) * 1000 / elapsed,
+				rate_recv / elapsed, (rate_recv % elapsed) * 1000 / elapsed,
+				"MB/s");
+		last_bytes_rdma_written = sent;
+		last_bytes_rdma_read = recv;
+	}
+
+	pcn_kmsg_stat(seq, NULL);
+
+	return 0;
+}
+
+static ssize_t __write_stats(struct file *file, const char __user *buffer,
+			     size_t size, loff_t *offset)
+{
+	int i;
+	for_each_present_cpu(i) {
+		per_cpu(bytes_sent, i) = 0;
+		per_cpu(bytes_recv, i) = 0;
+		per_cpu(bytes_rdma_written, i) = 0;
+		per_cpu(bytes_rdma_read, i) = 0;
+	}
+	pcn_kmsg_stat(NULL, NULL);
+
+	for (i = 0 ; i < PCN_KMSG_TYPE_MAX; i++) {
+		sent_stats[i] = 0;
+		recv_stats[i] = 0;
+	}
+	fh_action_stat(NULL, NULL);
+
+	return size;
+}
+
+static int __open_stats(struct inode *inode, struct file *file)
+{
+	return single_open(file, __show_stats, inode->i_private);
+}
+
+static struct file_operations stats_ops = {
+	.owner = THIS_MODULE,
+	.open = __open_stats,
+	.read = seq_read,
+	.llseek  = seq_lseek,
+	.release = single_release,
+	.write = __write_stats,
+};
+
+static struct proc_dir_entry *proc_entry = NULL;
+
+int statistics_init(void)
+{
+	proc_entry = proc_create("popcorn_stat", S_IRUGO | S_IWUGO, NULL, &stats_ops);
+	if (proc_entry == NULL) {
+		printk(KERN_ERR"cannot create proc_fs entry for popcorn stats\n");
+		return -ENOMEM;
+	}
+	return 0;
+}
diff --git a/kernel/popcorn/trace_events.h b/kernel/popcorn/trace_events.h
new file mode 100644
index 000000000..7ea5e8b6a
--- /dev/null
+++ b/kernel/popcorn/trace_events.h
@@ -0,0 +1,76 @@
+// SPDX-License-Identifier: GPL-2.0, BSD
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM popcorn
+
+#if !defined(_TRACE_EVENTS_POPCORN_H_) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_EVENTS_POPCORN_H_
+
+#include <linux/tracepoint.h>
+
+
+TRACE_EVENT(pgfault,
+	TP_PROTO(const int nid, const int pid, const char rw,
+		const unsigned long instr_addr, const unsigned long addr,
+		const int result),
+
+	TP_ARGS(nid, pid, rw, instr_addr, addr, result),
+
+	TP_STRUCT__entry(
+		__field(int, nid)
+		__field(int, pid)
+		__field(char, rw)
+		__field(unsigned long, instr_addr)
+		__field(unsigned long, addr)
+		__field(int, result)
+	),
+
+	TP_fast_assign(
+		__entry->nid = nid;
+		__entry->pid = pid;
+		__entry->rw = rw;
+		__entry->instr_addr = instr_addr;
+		__entry->addr = addr;
+		__entry->result = result;
+	),
+
+	TP_printk("%d %d %c %lx %lx %d",
+		__entry->nid, __entry->pid, __entry->rw,
+		__entry->instr_addr, __entry->addr, __entry->result)
+);
+
+
+TRACE_EVENT(pgfault_stat,
+	TP_PROTO(const unsigned long instr_addr, const unsigned long addr,
+		const int result, const int retries, const unsigned long time_us),
+
+	TP_ARGS(instr_addr, addr, result, retries, time_us),
+
+	TP_STRUCT__entry(
+		__field(unsigned long, instr_addr)
+		__field(unsigned long, addr)
+		__field(int, result)
+		__field(int, retries)
+		__field(unsigned long, time_us)
+	),
+
+	TP_fast_assign(
+		__entry->instr_addr = instr_addr;
+		__entry->addr = addr;
+		__entry->result = result;
+		__entry->retries = retries;
+		__entry->time_us = time_us;
+	),
+
+	TP_printk("%lx %lx %d %d %lu",
+		__entry->instr_addr, __entry->addr, __entry->result,
+		__entry->retries, __entry->time_us)
+);
+
+#endif
+
+#undef TRACE_INCLUDE_PATH
+#undef TRACE_INCLUDE_FILE
+
+#define TRACE_INCLUDE_PATH ../../kernel/popcorn
+#define TRACE_INCLUDE_FILE trace_events
+#include <trace/define_trace.h>
diff --git a/kernel/popcorn/types.h b/kernel/popcorn/types.h
new file mode 100644
index 000000000..bd6f3db0e
--- /dev/null
+++ b/kernel/popcorn/types.h
@@ -0,0 +1,358 @@
+// SPDX-License-Identifier: GPL-2.0, BSD
+#ifndef __INTERNAL_POPCORN_TYPES_H__
+#define __INTERNAL_POPCORN_TYPES_H__
+
+#include <linux/list.h>
+#include <linux/spinlock.h>
+#include <linux/completion.h>
+#include <linux/workqueue.h>
+#include <linux/signal.h>
+#include <linux/slab.h>
+#include <linux/radix-tree.h>
+#include <linux/sched/task.h>
+#include <popcorn/pcn_kmsg.h>
+#include <popcorn/regset.h>
+#include <linux/sched.h>
+#include <popcorn/types.h>
+
+#define FAULTS_HASH 31
+
+/*
+ * Remote execution context
+ */
+struct remote_context {
+	struct list_head list;
+	atomic_t count;
+	struct mm_struct *mm;
+
+	int tgid;
+	bool for_remote;
+
+	/* Tracking page status */
+	struct radix_tree_root pages;
+
+	/* For page replication protocol */
+	spinlock_t faults_lock[FAULTS_HASH];
+	struct hlist_head faults[FAULTS_HASH];
+
+	/* For VMA management */
+	spinlock_t vmas_lock;
+	struct list_head vmas;
+
+	/* Remote worker */
+	bool stop_remote_worker;
+
+	struct task_struct *remote_worker;
+	struct completion remote_works_ready;
+	spinlock_t remote_works_lock;
+	struct list_head remote_works;
+
+	pid_t remote_tgids[MAX_POPCORN_NODES];
+};
+
+struct remote_context *__get_mm_remote(struct mm_struct *mm);
+struct remote_context *get_task_remote(struct task_struct *tsk);
+bool put_task_remote(struct task_struct *tsk);
+bool __put_task_remote(struct remote_context *rc);
+
+
+/*
+ * Process migration
+ */
+#define BACK_MIGRATION_FIELDS \
+	int remote_nid;\
+	pid_t remote_pid;\
+	pid_t origin_pid;\
+	unsigned int personality;\
+	struct field_arch arch;
+DEFINE_PCN_KMSG(back_migration_request_t, BACK_MIGRATION_FIELDS);
+
+#define CLONE_FIELDS \
+	pid_t origin_tgid;\
+	pid_t origin_pid;\
+	unsigned long task_size; \
+	unsigned long stack_start; \
+	unsigned long env_start;\
+	unsigned long env_end;\
+	unsigned long arg_start;\
+	unsigned long arg_end;\
+	unsigned long start_brk;\
+	unsigned long brk;\
+	unsigned long start_code ;\
+	unsigned long end_code;\
+	unsigned long start_data;\
+	unsigned long end_data;\
+	unsigned int personality;\
+	unsigned long def_flags;\
+	char exe_path[512];\
+	struct field_arch arch;
+DEFINE_PCN_KMSG(clone_request_t, CLONE_FIELDS);
+
+
+/*
+ * This message is sent in response to a clone request.
+ * Its purpose is to notify the requesting cpu that make
+ * the specified pid is executing on behalf of the
+ * requesting cpu.
+ */
+#define REMOTE_TASK_PAIRING_FIELDS \
+	pid_t my_tgid; \
+	pid_t my_pid; \
+	pid_t your_pid;
+DEFINE_PCN_KMSG(remote_task_pairing_t, REMOTE_TASK_PAIRING_FIELDS);
+
+
+#define REMOTE_TASK_EXIT_FIELDS  \
+	pid_t origin_pid; \
+	pid_t remote_pid; \
+	int exit_code;
+DEFINE_PCN_KMSG(remote_task_exit_t, REMOTE_TASK_EXIT_FIELDS);
+
+#define ORIGIN_TASK_EXIT_FIELDS \
+	pid_t origin_pid; \
+	pid_t remote_pid; \
+	int exit_code;
+DEFINE_PCN_KMSG(origin_task_exit_t, ORIGIN_TASK_EXIT_FIELDS);
+
+
+/*
+ * VMA management
+ */
+#define VMA_INFO_REQUEST_FIELDS \
+	pid_t origin_pid;	\
+	pid_t remote_pid;	\
+	unsigned long addr;
+DEFINE_PCN_KMSG(vma_info_request_t, VMA_INFO_REQUEST_FIELDS);
+
+#define VMA_INFO_RESPONSE_FIELDS \
+	pid_t remote_pid;	 \
+	int result;		 \
+	unsigned long addr;	 \
+	unsigned long vm_start;	 \
+	unsigned long vm_end;	 \
+	unsigned long vm_flags;	 \
+	unsigned long vm_pgoff;	 \
+	char vm_file_path[512];
+DEFINE_PCN_KMSG(vma_info_response_t, VMA_INFO_RESPONSE_FIELDS);
+
+#define vma_info_anon(x) ((x)->vm_file_path[0] == '\0' ? true : false)
+
+
+#define VMA_OP_REQUEST_FIELDS \
+	pid_t origin_pid; \
+	pid_t remote_pid; \
+	int remote_ws; \
+	int operation; \
+	union { \
+		unsigned long addr; \
+		unsigned long start; \
+		unsigned long brk; \
+	}; \
+	union { \
+		unsigned long len;		/* mmap */ \
+		unsigned long old_len;	/* mremap */ \
+	}; \
+	union { \
+		unsigned long prot;		/* mmap */ \
+		int behavior;			/* madvise */ \
+		unsigned long new_len;	/* mremap */ \
+	}; \
+	unsigned long flags;		/* mmap, remap */ \
+	union { \
+		unsigned long pgoff;	/* mmap */ \
+		unsigned long new_addr;	/* mremap */ \
+	}; \
+	char path[512];
+DEFINE_PCN_KMSG(vma_op_request_t, VMA_OP_REQUEST_FIELDS);
+
+#define VMA_OP_RESPONSE_FIELDS \
+	pid_t origin_pid;      \
+	pid_t remote_pid;      \
+	int remote_ws;	       \
+	int operation;	       \
+	long ret;	       \
+	union {			    \
+		unsigned long addr; \
+		unsigned long start; \
+		unsigned long brk; \
+	}; \
+	unsigned long len;
+DEFINE_PCN_KMSG(vma_op_response_t, VMA_OP_RESPONSE_FIELDS);
+
+
+/*
+ * Page management
+ */
+#define REMOTE_PAGE_REQUEST_FIELDS \
+	pid_t origin_pid;	   \
+	int origin_ws;		   \
+	pid_t remote_pid;	   \
+	unsigned long addr;	   \
+	unsigned long fault_flags; \
+	unsigned long instr_addr;  \
+	dma_addr_t rdma_addr;	   \
+	u32 rdma_key;
+DEFINE_PCN_KMSG(remote_page_request_t, REMOTE_PAGE_REQUEST_FIELDS);
+
+#define REMOTE_PAGE_RESPONSE_COMMON_FIELDS \
+	pid_t remote_pid;		   \
+	pid_t origin_pid;		   \
+	int origin_ws;			   \
+	unsigned long addr;		   \
+	int result;
+
+#define REMOTE_PAGE_RESPONSE_FIELDS \
+	REMOTE_PAGE_RESPONSE_COMMON_FIELDS \
+	unsigned char page[PAGE_SIZE];
+DEFINE_PCN_KMSG(remote_page_response_t, REMOTE_PAGE_RESPONSE_FIELDS);
+
+#define REMOTE_PAGE_GRANT_FIELDS \
+	REMOTE_PAGE_RESPONSE_COMMON_FIELDS
+DEFINE_PCN_KMSG(remote_page_response_short_t, REMOTE_PAGE_GRANT_FIELDS);
+
+
+#define REMOTE_PAGE_FLUSH_COMMON_FIELDS \
+	pid_t origin_pid;		\
+	int remote_nid;			\
+	pid_t remote_pid;		\
+	int remote_ws;			\
+	unsigned long addr;		\
+	unsigned long flags;
+
+#define REMOTE_PAGE_FLUSH_FIELDS \
+	REMOTE_PAGE_FLUSH_COMMON_FIELDS \
+	unsigned char page[PAGE_SIZE];
+DEFINE_PCN_KMSG(remote_page_flush_t, REMOTE_PAGE_FLUSH_FIELDS);
+
+#define REMOTE_PAGE_RELEASE_FIELDS \
+	REMOTE_PAGE_FLUSH_COMMON_FIELDS
+DEFINE_PCN_KMSG(remote_page_release_t, REMOTE_PAGE_RELEASE_FIELDS);
+
+#define REMOTE_PAGE_FLUSH_ACK_FIELDS \
+	int remote_ws; \
+	unsigned long flags;
+DEFINE_PCN_KMSG(remote_page_flush_ack_t, REMOTE_PAGE_FLUSH_ACK_FIELDS);
+
+
+#define PAGE_INVALIDATE_REQUEST_FIELDS \
+	pid_t origin_pid; \
+	int origin_ws; \
+	pid_t remote_pid; \
+	unsigned long addr;
+DEFINE_PCN_KMSG(page_invalidate_request_t, PAGE_INVALIDATE_REQUEST_FIELDS);
+
+#define PAGE_INVALIDATE_RESPONSE_FIELDS \
+	pid_t origin_pid; \
+	int origin_ws; \
+	pid_t remote_pid;
+DEFINE_PCN_KMSG(page_invalidate_response_t, PAGE_INVALIDATE_RESPONSE_FIELDS);
+
+
+/*
+ * Futex
+ */
+#define REMOTE_FUTEX_REQ_FIELDS \
+	pid_t origin_pid;	\
+	int remote_ws;		\
+	int op;			\
+	u32 val;		\
+	struct timespec64 ts;	\
+	void *uaddr;		\
+	void *uaddr2;		\
+	u32 val2;		\
+	u32 val3;
+DEFINE_PCN_KMSG(remote_futex_request, REMOTE_FUTEX_REQ_FIELDS);
+
+#define REMOTE_FUTEX_RES_FIELDS \
+	int remote_ws;		\
+	long ret;
+DEFINE_PCN_KMSG(remote_futex_response, REMOTE_FUTEX_RES_FIELDS);
+
+/*
+ * Node information
+ */
+#define NODE_INFO_FIELDS \
+	int nid;	 \
+	int bundle_id;	 \
+	int arch;
+DEFINE_PCN_KMSG(node_info_t, NODE_INFO_FIELDS);
+
+
+/*
+ * Schedule server. Not yet completely ported though
+ */
+#define SCHED_PERIODIC_FIELDS \
+	int power_1;	      \
+	int power_2;	      \
+	int power_3;
+DEFINE_PCN_KMSG(sched_periodic_req, SCHED_PERIODIC_FIELDS);
+
+/*
+ * Message routing using work queues
+ */
+extern struct workqueue_struct *popcorn_wq;
+extern struct workqueue_struct *popcorn_ordered_wq;
+
+struct pcn_kmsg_work {
+	struct work_struct work;
+	void *msg;
+};
+
+static inline int __handle_popcorn_work(struct pcn_kmsg_message *msg,
+					void (*handler)(struct work_struct *),
+					struct workqueue_struct *wq)
+{
+	struct pcn_kmsg_work *w = kmalloc(sizeof(*w), GFP_ATOMIC);
+	BUG_ON(!w);
+
+	w->msg = msg;
+	INIT_WORK(&w->work, handler);
+	smp_wmb();
+	queue_work(wq, &w->work);
+
+	return 0;
+}
+
+int request_remote_work(pid_t pid, struct pcn_kmsg_message *req);
+
+#define DEFINE_KMSG_WQ_HANDLER(x) \
+static inline int handle_##x(struct pcn_kmsg_message *msg) {\
+	return __handle_popcorn_work(msg, process_##x, popcorn_wq);\
+}
+#define DEFINE_KMSG_ORDERED_WQ_HANDLER(x) \
+static inline int handle_##x(struct pcn_kmsg_message *msg) {\
+	return __handle_popcorn_work(msg, process_##x, popcorn_ordered_wq);\
+}
+#define DEFINE_KMSG_RW_HANDLER(x,type,member) \
+static inline int handle_##x(struct pcn_kmsg_message *msg) {\
+	type *req = (type *)msg; \
+	return request_remote_work(req->member, msg); \
+}
+
+#define REGISTER_KMSG_WQ_HANDLER(x, y) \
+	pcn_kmsg_register_callback(x, handle_##y)
+
+#define REGISTER_KMSG_HANDLER(x, y) \
+	pcn_kmsg_register_callback(x, handle_##y)
+
+#define START_KMSG_WORK(type, name, work) \
+	struct pcn_kmsg_work *__pcn_kmsg_work__ = (struct pcn_kmsg_work *)(work); \
+	type *name = __pcn_kmsg_work__->msg
+
+#define END_KMSG_WORK(name) \
+	pcn_kmsg_done(name); \
+	kfree(__pcn_kmsg_work__);
+
+static inline struct task_struct *__get_task_struct(pid_t pid)
+{
+	struct task_struct *tsk = NULL;
+	rcu_read_lock();
+	tsk = find_task_by_vpid(pid);
+	if (likely(tsk)) {
+		get_task_struct(tsk);
+	}
+	rcu_read_unlock();
+	return tsk;
+}
+
+#endif /* __INTERNAL_POPCORN_TYPES_H__ */
diff --git a/kernel/popcorn/util.c b/kernel/popcorn/util.c
new file mode 100644
index 000000000..cb031200b
--- /dev/null
+++ b/kernel/popcorn/util.c
@@ -0,0 +1,121 @@
+// SPDX-License-Identifier: GPL-2.0, BSD
+/*
+ * /kernel/popcorn/util.c
+ *
+ * General Utility Functions
+ *
+ * Original file developed by SSRG at Virginia Tech.
+ *
+ * author, Javier Malave, Rebecca Shapiro, Andrew Hughes,
+ * Narf Industries 2020 (modifications for upstream RFC)
+ */
+
+#include <linux/mm.h>
+#include <linux/slab.h>
+#include <linux/sched/task_stack.h>
+#include <popcorn/bundle.h>
+
+void print_page_data(unsigned char *addr)
+{
+	int i;
+	for (i = 0; i < PAGE_SIZE; i++) {
+		if (i % 16 == 0) {
+			printk(KERN_INFO"%08lx:", (unsigned long)(addr + i));
+		}
+		if (i % 4 == 0) {
+			printk(" ");
+		}
+		printk("%02x", *(addr + i));
+	}
+	printk("\n");
+}
+
+void print_page_signature(unsigned char *addr)
+{
+	unsigned char *p = addr;
+	int i, j;
+	for (i = 0; i < PAGE_SIZE / 128; i++) {
+		unsigned char signature = 0;
+		for (j = 0; j < 32; j++) {
+			signature = (signature + *p++) & 0xff;
+		}
+		printk("%02x", signature);
+	}
+	printk("\n");
+}
+
+void print_page_signature_pid(pid_t pid, unsigned char *addr)
+{
+	printk("  [%d] ", pid);
+	print_page_signature(addr);
+}
+
+static DEFINE_SPINLOCK(__print_lock);
+static char *__print_buffer = NULL;
+
+void print_page_owner(unsigned long addr, unsigned long *owners, pid_t pid)
+{
+	if (unlikely(!__print_buffer)) {
+		__print_buffer = kmalloc(PAGE_SIZE, GFP_KERNEL);
+	}
+	spin_lock(&__print_lock);
+	bitmap_print_to_pagebuf(
+			true, __print_buffer, owners, MAX_POPCORN_NODES);
+	printk("  [%d] %lx %s", pid, addr, __print_buffer);
+	spin_unlock(&__print_lock);
+}
+
+#include <linux/fs.h>
+
+static DEFINE_SPINLOCK(__file_path_lock);
+static char *__file_path_buffer = NULL;
+
+int get_file_path(struct file *file, char *sz, size_t size)
+{
+	char *ppath;
+	int retval = 0;
+
+	if (!file) {
+		BUG_ON(size < 1);
+		sz[0] = '\0';
+		return -EINVAL;
+	}
+
+	if (unlikely(!__file_path_buffer)) {
+		__file_path_buffer = kmalloc(PAGE_SIZE, GFP_KERNEL);
+	}
+
+	spin_lock(&__file_path_lock);
+	ppath = file_path(file, __file_path_buffer, PAGE_SIZE);
+	if (IS_ERR(ppath)) {
+		retval = -ESRCH;
+		goto out_unlock;
+	}
+
+	strncpy(sz, ppath, size);
+
+out_unlock:
+	spin_unlock(&__file_path_lock);
+	return 0;
+}
+
+
+static const char *__comm_to_trace[] = {
+};
+
+#include <linux/kernel.h>
+#include <linux/sched.h>
+#include <linux/ptrace.h>
+
+void trace_task_status(void)
+{
+	int i;
+	for (i = 0; i < ARRAY_SIZE(__comm_to_trace); i++) {
+		const char *comm = __comm_to_trace[i];
+		if (memcmp(current->comm, comm, strlen(comm)) == 0) {
+			printk("@@[%d] %s %lx\n", current->pid,
+					current->comm, instruction_pointer(current_pt_regs()));
+			break;
+		}
+	}
+}
diff --git a/kernel/popcorn/util.h b/kernel/popcorn/util.h
new file mode 100644
index 000000000..b94fb192a
--- /dev/null
+++ b/kernel/popcorn/util.h
@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: GPL-2.0, BSD
+#ifndef __POPCORN_KERNEL_UTIL_H__
+#define __POPCORN_KERNEL_UTIL_H__
+struct page;
+
+void print_page_data(unsigned char *addr);
+void print_page_signature(unsigned char *addr);
+void print_page_signature_pid(pid_t pid, unsigned char *addr);
+void print_page_owner(unsigned long addr, unsigned long *owners, pid_t pid);
+
+int get_file_path(struct file *file, char *sz, size_t size);
+
+void trace_task_status(void);
+#endif
diff --git a/kernel/popcorn/wait_station.c b/kernel/popcorn/wait_station.c
new file mode 100644
index 000000000..3367222f4
--- /dev/null
+++ b/kernel/popcorn/wait_station.c
@@ -0,0 +1,84 @@
+// SPDX-License-Identifier: GPL-2.0, BSD
+/*
+ * /kernel/popcorn/wait_station.c
+ *
+ * Waiting stations allows threads to be waited for a given
+ * number of events are completed
+ *
+ * Original file developed by SSRG at Virginia Tech.
+ *
+ * author, Javier Malave, Rebecca Shapiro, Andrew Hughes,
+ * Narf Industries 2020 (modifications for upstream RFC)
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/sched.h>
+#include <linux/err.h>
+
+#include "wait_station.h"
+
+#define MAX_WAIT_STATIONS 1024
+
+static struct wait_station wait_stations[MAX_WAIT_STATIONS];
+
+static DEFINE_SPINLOCK(wait_station_lock);
+static DECLARE_BITMAP(wait_station_available, MAX_WAIT_STATIONS) = { 0 };
+
+struct wait_station *get_wait_station_multiple(struct task_struct *tsk,
+					       int count)
+{
+	int id;
+	struct wait_station *ws;
+
+	spin_lock(&wait_station_lock);
+	id = find_first_zero_bit(wait_station_available, MAX_WAIT_STATIONS);
+	BUG_ON(id >= MAX_WAIT_STATIONS);
+	ws = wait_stations + id;
+	set_bit(id, wait_station_available);
+	spin_unlock(&wait_station_lock);
+
+	ws->id = id;
+	ws->pid = tsk->pid;
+	ws->private = (void *)0xbad0face;
+	init_completion(&ws->pendings);
+	atomic_set(&ws->pendings_count, count);
+	smp_wmb();
+
+	return ws;
+}
+EXPORT_SYMBOL_GPL(get_wait_station_multiple);
+
+struct wait_station *wait_station(int id)
+{
+	smp_rmb();
+	return wait_stations + id;
+}
+EXPORT_SYMBOL_GPL(wait_station);
+
+void put_wait_station(struct wait_station *ws)
+{
+	int id = ws->id;
+	spin_lock(&wait_station_lock);
+	BUG_ON(!test_bit(id, wait_station_available));
+	clear_bit(id, wait_station_available);
+	spin_unlock(&wait_station_lock);
+}
+EXPORT_SYMBOL_GPL(put_wait_station);
+
+void *wait_at_station(struct wait_station *ws)
+{
+	void *ret;
+	if (!try_wait_for_completion(&ws->pendings)) {
+		if (wait_for_completion_io_timeout(&ws->pendings, 300 * HZ) == 0) {
+			ret = ERR_PTR(-ETIMEDOUT);
+			goto out;
+		}
+	}
+	smp_rmb();
+	ret = (void *)ws->private;
+out:
+	put_wait_station(ws);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(wait_at_station);
diff --git a/kernel/popcorn/wait_station.h b/kernel/popcorn/wait_station.h
new file mode 100644
index 000000000..3ced49b84
--- /dev/null
+++ b/kernel/popcorn/wait_station.h
@@ -0,0 +1,27 @@
+// SPDX-License-Identifier: GPL-2.0, BSD
+#ifndef _POPCORN_WAIT_STATION_H_
+#define _POPCORN_WAIT_STATION_H_
+
+#include <linux/completion.h>
+#include <linux/atomic.h>
+
+struct wait_station {
+	int id;
+	pid_t pid;
+	volatile void *private;
+	struct completion pendings;
+	atomic_t pendings_count;
+};
+
+struct task_struct;
+
+struct wait_station *get_wait_station_multiple(struct task_struct *tsk,
+					int count);
+static inline struct wait_station *get_wait_station(struct task_struct *tsk)
+{
+	return get_wait_station_multiple(tsk, 1);
+}
+struct wait_station *wait_station(int id);
+void put_wait_station(struct wait_station *ws);
+void *wait_at_station(struct wait_station *ws);
+#endif
-- 
2.17.1