LKML Archive on lore.kernel.org
 help / color / Atom feed
From: Andra Paraschiv <andraprs@amazon.com>
To: <linux-kernel@vger.kernel.org>
Cc: Anthony Liguori <aliguori@amazon.com>,
	Benjamin Herrenschmidt <benh@amazon.com>,
	Colm MacCarthaigh <colmmacc@amazon.com>,
	Bjoern Doebel <doebel@amazon.de>,
	David Woodhouse <dwmw@amazon.co.uk>,
	Frank van der Linden <fllinden@amazon.com>,
	Alexander Graf <graf@amazon.de>,
	Martin Pohlack <mpohlack@amazon.de>, Matt Wilson <msw@amazon.com>,
	Paolo Bonzini <pbonzini@redhat.com>,
	Balbir Singh <sblbir@amazon.com>,
	Stewart Smith <trawets@amazon.com>,
	Uwe Dannowski <uwed@amazon.de>, <kvm@vger.kernel.org>,
	<ne-devel-upstream@amazon.com>,
	Andra Paraschiv <andraprs@amazon.com>
Subject: [PATCH v1 02/15] nitro_enclaves: Define the PCI device interface
Date: Tue, 21 Apr 2020 21:41:37 +0300
Message-ID: <20200421184150.68011-3-andraprs@amazon.com> (raw)
In-Reply-To: <20200421184150.68011-1-andraprs@amazon.com>

The Nitro Enclaves (NE) driver communicates with a new PCI device, that
is exposed to a virtual machine (VM) and handles commands meant for
handling enclaves lifetime e.g. creation, termination, setting memory
regions. The communication with the PCI device is handled using a MMIO
space and MSI-X interrupts.

This device communicates with the hypervisor on the host, where the VM
that spawned the enclave itself run, e.g. to launch a VM that is used
for the enclave.

Define the MMIO space of the PCI device, the commands that are
provided by this device. Add an internal data structure used as private
data for the PCI device driver and the functions for the PCI device init
/ uninit and command requests handling.

Signed-off-by: Alexandru-Catalin Vasile <lexnv@amazon.com>
Signed-off-by: Alexandru Ciobotaru <alcioa@amazon.com>
Signed-off-by: Andra Paraschiv <andraprs@amazon.com>
---
 .../virt/amazon/nitro_enclaves/ne_pci_dev.h   | 266 ++++++++++++++++++
 1 file changed, 266 insertions(+)
 create mode 100644 drivers/virt/amazon/nitro_enclaves/ne_pci_dev.h

diff --git a/drivers/virt/amazon/nitro_enclaves/ne_pci_dev.h b/drivers/virt/amazon/nitro_enclaves/ne_pci_dev.h
new file mode 100644
index 000000000000..e703419ed29d
--- /dev/null
+++ b/drivers/virt/amazon/nitro_enclaves/ne_pci_dev.h
@@ -0,0 +1,266 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _NE_PCI_DEV_H_
+#define _NE_PCI_DEV_H_
+
+#include <linux/atomic.h>
+#include <linux/list.h>
+#include <linux/mutex.h>
+#include <linux/pci.h>
+#include <linux/pci_ids.h>
+#include <linux/wait.h>
+
+/* Nitro Enclaves (NE) PCI device identifier */
+
+#define PCI_DEVICE_ID_NE (0xe4c1)
+#define PCI_BAR_NE (0x03)
+
+/* Device registers */
+
+/**
+ * (1 byte) Register to notify the device that the driver is using it
+ * (Read/Write).
+ */
+#define NE_ENABLE (0x0000)
+#define NE_ENABLE_ON (0x00)
+#define NE_ENABLE_OFF (0x01)
+
+/* (2 bytes) Register to select the device run-time version (Read/Write). */
+#define NE_VERSION (0x0002)
+#define NE_VERSION_MAX (0x0001)
+
+/**
+ * (4 bytes) Register to notify the device what command was requested
+ * (Write-Only).
+ */
+#define NE_COMMAND (0x0004)
+
+/**
+ * (4 bytes) Register to notify the driver that a reply or a device event
+ * is available (Read-Only):
+ * - Lower half  - command reply counter
+ * - Higher half - out-of-band device event counter
+ */
+#define NE_EVTCNT (0x000c)
+#define NE_EVTCNT_REPLY_SHIFT (0)
+#define NE_EVTCNT_REPLY_MASK (0x0000ffff)
+#define NE_EVTCNT_REPLY(cnt) (((cnt) & NE_EVTCNT_REPLY_MASK) >> \
+			      NE_EVTCNT_REPLY_SHIFT)
+#define NE_EVTCNT_EVENT_SHIFT (16)
+#define NE_EVTCNT_EVENT_MASK (0xffff0000)
+#define NE_EVTCNT_EVENT(cnt) (((cnt) & NE_EVTCNT_EVENT_MASK) >> \
+			      NE_EVTCNT_EVENT_SHIFT)
+
+/* (240 bytes) Buffer for sending the command request payload (Read/Write). */
+#define NE_SEND_DATA (0x0010)
+
+/* (240 bytes) Buffer for receiving the command reply payload (Read-Only). */
+#define NE_RECV_DATA (0x0100)
+
+/* Device MMIO buffer sizes */
+
+/* 240 bytes for send / recv buffer. */
+#define NE_SEND_DATA_SIZE (240)
+#define NE_RECV_DATA_SIZE (240)
+
+/* MSI-X interrupt vectors */
+
+/* MSI-X vector used for command reply notification. */
+#define NE_VEC_REPLY (0)
+
+/* MSI-X vector used for out-of-band events e.g. enclave crash. */
+#define NE_VEC_EVENT (1)
+
+/* Device command types. */
+enum ne_pci_dev_cmd_type {
+	INVALID_CMD = 0,
+	ENCLAVE_START = 1,
+	ENCLAVE_GET_SLOT = 2,
+	ENCLAVE_STOP = 3,
+	SLOT_ALLOC = 4,
+	SLOT_FREE = 5,
+	SLOT_ADD_MEM = 6,
+	SLOT_ADD_VCPU = 7,
+	SLOT_COUNT = 8,
+	NEXT_SLOT = 9,
+	SLOT_INFO = 10,
+	SLOT_ADD_BULK_VCPUS = 11,
+	MAX_CMD,
+};
+
+/* Device commands - payload structure for requests and replies. */
+
+struct enclave_start_req {
+	/* Slot unique id mapped to the enclave to start. */
+	u64 slot_uid;
+
+	/**
+	 * Context ID (CID) for the enclave vsock device.
+	 * If 0, CID is autogenerated.
+	 */
+	u64 enclave_cid;
+
+	/* Flags for the enclave to start with (e.g. debug mode). */
+	u64 flags;
+} __attribute__ ((__packed__));
+
+struct enclave_get_slot_req {
+	/* Context ID (CID) for the enclave vsock device. */
+	u64 enclave_cid;
+} __attribute__ ((__packed__));
+
+struct enclave_stop_req {
+	/* Slot unique id mapped to the enclave to stop. */
+	u64 slot_uid;
+} __attribute__ ((__packed__));
+
+struct slot_alloc_req {
+	/* In order to avoid weird sizeof edge cases. */
+	u8 unused;
+} __attribute__ ((__packed__));
+
+struct slot_free_req {
+	/* Slot unique id mapped to the slot to free. */
+	u64 slot_uid;
+} __attribute__ ((__packed__));
+
+struct slot_add_mem_req {
+	/* Slot unique id mapped to the slot to add the memory region to. */
+	u64 slot_uid;
+
+	/* Physical address of the memory region to add to the slot. */
+	u64 paddr;
+
+	/* Memory size, in bytes, of the memory region to add to the slot. */
+	u64 size;
+} __attribute__ ((__packed__));
+
+struct slot_add_vcpu_req {
+	/* Slot unique id mapped to the slot to add the vCPU to. */
+	u64 slot_uid;
+
+	/* vCPU ID of the CPU to add to the enclave. */
+	u32 vcpu_id;
+} __attribute__ ((__packed__));
+
+struct slot_count_req {
+	/* In order to avoid weird sizeof edge cases. */
+	u8 unused;
+} __attribute__ ((__packed__));
+
+struct next_slot_req {
+	/* Slot unique id of the next slot in the iteration. */
+	u64 slot_uid;
+} __attribute__ ((__packed__));
+
+struct slot_info_req {
+	/* Slot unique id mapped to the slot to get information about. */
+	u64 slot_uid;
+} __attribute__ ((__packed__));
+
+struct slot_add_bulk_vcpus_req {
+	/* Slot unique id mapped to the slot to add vCPUs to. */
+	u64 slot_uid;
+
+	/* Number of vCPUs to add to the slot. */
+	u64 nr_vcpus;
+} __attribute__ ((__packed__));
+
+struct ne_pci_dev_cmd_reply {
+	s32 rc;
+
+	/* Valid for all commands except SLOT_COUNT. */
+	u64 slot_uid;
+
+	/* Valid for ENCLAVE_START command. */
+	u64 enclave_cid;
+
+	/* Valid for SLOT_COUNT command. */
+	u64 slot_count;
+
+	/* Valid for SLOT_ALLOC and SLOT_INFO commands. */
+	u64 mem_regions;
+
+	/* Valid for SLOT_INFO command. */
+	u64 mem_size;
+
+	/* Valid for SLOT_INFO command. */
+	u64 nr_vcpus;
+
+	/* Valid for SLOT_INFO command. */
+	u64 flags;
+
+	/* Valid for SLOT_INFO command. */
+	u16 state;
+} __attribute__ ((__packed__));
+
+/* Nitro Enclaves (NE) PCI device. */
+struct ne_pci_dev {
+	/* Variable set if a reply has been sent by the PCI device. */
+	atomic_t cmd_reply_avail;
+
+	/* Wait queue for handling command reply from the PCI device. */
+	wait_queue_head_t cmd_reply_wait_q;
+
+	/* List of the enclaves managed by the PCI device. */
+	struct list_head enclaves_list;
+
+	/* Mutex for accessing the list of enclaves. */
+	struct mutex enclaves_list_mutex;
+
+	/**
+	 * Work queue for handling out-of-band events triggered by the Nitro
+	 * Hypervisor which require enclave state scanning and propagation to
+	 * the enclave process.
+	 */
+	struct workqueue_struct *event_wq;
+
+	/* MMIO region of the PCI device. */
+	void __iomem *iomem_base;
+
+	/* Work item for every received out-of-band event. */
+	struct work_struct notify_work;
+
+	/* Mutex for accessing the PCI dev MMIO space. */
+	struct mutex pci_dev_mutex;
+};
+
+/**
+ * ne_do_request - Submit command request to the PCI device based on the command
+ * type and retrieve the associated reply.
+ *
+ * This function uses the ne_pci_dev mutex to handle one command at a time.
+ *
+ * @pdev: PCI device to send the command to and receive the reply from.
+ * @cmd_type: command type of the request sent to the PCI device.
+ * @cmd_request: command request payload.
+ * @cmd_request_size: size of the command request payload.
+ * @cmd_reply: command reply payload.
+ * @cmd_reply_size: size of the command reply payload.
+ *
+ * @returns: 0 on success, negative return value on failure.
+ */
+int ne_do_request(struct pci_dev *pdev, enum ne_pci_dev_cmd_type cmd_type,
+		  void *cmd_request, size_t cmd_request_size,
+		  struct ne_pci_dev_cmd_reply *cmd_reply,
+		  size_t cmd_reply_size);
+
+/* Nitro Enclaves (NE) PCI device driver */
+extern struct pci_driver ne_pci_driver;
+
+#endif /* _NE_PCI_DEV_H_ */
-- 
2.20.1 (Apple Git-117)




Amazon Development Center (Romania) S.R.L. registered office: 27A Sf. Lazar Street, UBC5, floor 2, Iasi, Iasi County, 700045, Romania. Registered in Romania. Registration number J22/2621/2005.


  parent reply index

Thread overview: 77+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2020-04-21 18:41 [PATCH v1 00/15] Add support for Nitro Enclaves Andra Paraschiv
2020-04-21 18:41 ` [PATCH v1 01/15] nitro_enclaves: Add ioctl interface definition Andra Paraschiv
2020-04-21 18:47   ` Randy Dunlap
2020-04-21 21:45     ` Paolo Bonzini
2020-04-22 15:49       ` Paraschiv, Andra-Irina
2020-04-21 18:41 ` Andra Paraschiv [this message]
2020-04-21 21:22   ` [PATCH v1 02/15] nitro_enclaves: Define the PCI device interface Paolo Bonzini
2020-04-23 13:37     ` Paraschiv, Andra-Irina
2020-04-24 15:10       ` Paraschiv, Andra-Irina
2020-04-21 18:41 ` [PATCH v1 03/15] nitro_enclaves: Define enclave info for internal bookkeeping Andra Paraschiv
2020-04-21 18:41 ` [PATCH v1 04/15] nitro_enclaves: Init PCI device driver Andra Paraschiv
2020-04-25 14:25   ` Liran Alon
2020-04-29 16:31     ` Paraschiv, Andra-Irina
2020-04-21 18:41 ` [PATCH v1 05/15] nitro_enclaves: Handle PCI device command requests Andra Paraschiv
2020-04-25 14:52   ` Liran Alon
2020-04-29 17:00     ` Paraschiv, Andra-Irina
2020-04-21 18:41 ` [PATCH v1 06/15] nitro_enclaves: Handle out-of-band PCI device events Andra Paraschiv
2020-04-21 18:41 ` [PATCH v1 07/15] nitro_enclaves: Init misc device providing the ioctl interface Andra Paraschiv
2020-04-21 18:41 ` [PATCH v1 08/15] nitro_enclaves: Add logic for enclave vm creation Andra Paraschiv
2020-04-21 18:41 ` [PATCH v1 09/15] nitro_enclaves: Add logic for enclave vcpu creation Andra Paraschiv
2020-04-21 18:41 ` [PATCH v1 10/15] nitro_enclaves: Add logic for enclave memory region set Andra Paraschiv
2020-04-21 18:41 ` [PATCH v1 11/15] nitro_enclaves: Add logic for enclave start Andra Paraschiv
2020-04-21 18:41 ` [PATCH v1 12/15] nitro_enclaves: Add logic for enclave termination Andra Paraschiv
2020-04-21 18:41 ` [PATCH v1 13/15] nitro_enclaves: Add Kconfig for the Nitro Enclaves driver Andra Paraschiv
2020-04-21 18:50   ` Randy Dunlap
2020-04-22 14:35     ` Paraschiv, Andra-Irina
2020-04-21 18:41 ` [PATCH v1 14/15] nitro_enclaves: Add Makefile " Andra Paraschiv
2020-04-23  8:12   ` kbuild test robot
2020-04-24 17:00     ` Paraschiv, Andra-Irina
2020-04-23  8:43   ` kbuild test robot
2020-04-24 15:27     ` Paraschiv, Andra-Irina
2020-04-21 18:41 ` [PATCH v1 15/15] MAINTAINERS: Add entry " Andra Paraschiv
2020-04-21 21:46 ` [PATCH v1 00/15] Add support for Nitro Enclaves Paolo Bonzini
2020-04-23 13:19   ` Paraschiv, Andra-Irina
2020-04-23 13:42     ` Paolo Bonzini
2020-04-23 17:42       ` Paraschiv, Andra-Irina
2020-04-23 17:51         ` Paolo Bonzini
2020-04-23 20:56           ` Alexander Graf
2020-04-23 21:18             ` Paolo Bonzini
2020-04-24 12:56               ` Alexander Graf
2020-04-24 16:27                 ` Paolo Bonzini
2020-04-24 19:11                   ` Alexander Graf
2020-04-25 16:05                     ` Paolo Bonzini
2020-04-27  9:15                       ` Paraschiv, Andra-Irina
2020-04-27  9:22                       ` Paraschiv, Andra-Irina
2020-04-27  9:46                         ` Paolo Bonzini
2020-04-27 10:00                           ` Paraschiv, Andra-Irina
2020-04-28 15:07                       ` Alexander Graf
2020-04-29 13:20                         ` Paolo Bonzini
2020-04-30 13:59                           ` Paraschiv, Andra-Irina
2020-04-30 10:34                         ` Paolo Bonzini
2020-04-30 11:21                           ` Alexander Graf
2020-04-30 11:38                             ` Paolo Bonzini
2020-04-30 11:47                               ` Alexander Graf
2020-04-30 11:58                                 ` Paolo Bonzini
2020-04-30 12:19                                   ` Alexander Graf
2020-05-07 17:44       ` Pavel Machek
2020-05-08  7:00         ` Paraschiv, Andra-Irina
2020-05-09 19:21           ` Pavel Machek
2020-05-10 11:02             ` Herrenschmidt, Benjamin
2020-05-11 10:49               ` Paraschiv, Andra-Irina
2020-05-11 13:49               ` Stefan Hajnoczi
2020-04-24  3:04     ` Longpeng (Mike, Cloud Infrastructure Service Product Dept.)
2020-04-24  8:19       ` Paraschiv, Andra-Irina
2020-04-24  9:54         ` Paraschiv, Andra-Irina
2020-04-26  1:55           ` Longpeng (Mike, Cloud Infrastructure Service Product Dept.)
2020-04-27 18:39             ` Paraschiv, Andra-Irina
2020-04-24  9:59     ` Tian, Kevin
2020-04-24 13:59       ` Paraschiv, Andra-Irina
2020-04-26  8:16         ` Tian, Kevin
2020-04-27 19:05           ` Paraschiv, Andra-Irina
     [not found]         ` <CAKXe6SLonLQLAOY9Q_2AzTeg4uJxiknsAWnJpTF0hMcXEG5Tew@mail.gmail.com>
2020-05-11 12:05           ` Paraschiv, Andra-Irina
2020-04-25 15:25     ` Liran Alon
2020-04-27  7:56       ` Paraschiv, Andra-Irina
2020-04-27 11:44         ` Liran Alon
2020-04-28 15:25           ` Alexander Graf
2020-04-28 16:01             ` Liran Alon

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20200421184150.68011-3-andraprs@amazon.com \
    --to=andraprs@amazon.com \
    --cc=aliguori@amazon.com \
    --cc=benh@amazon.com \
    --cc=colmmacc@amazon.com \
    --cc=doebel@amazon.de \
    --cc=dwmw@amazon.co.uk \
    --cc=fllinden@amazon.com \
    --cc=graf@amazon.de \
    --cc=kvm@vger.kernel.org \
    --cc=linux-kernel@vger.kernel.org \
    --cc=mpohlack@amazon.de \
    --cc=msw@amazon.com \
    --cc=ne-devel-upstream@amazon.com \
    --cc=pbonzini@redhat.com \
    --cc=sblbir@amazon.com \
    --cc=trawets@amazon.com \
    --cc=uwed@amazon.de \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

LKML Archive on lore.kernel.org

Archives are clonable:
	git clone --mirror https://lore.kernel.org/lkml/0 lkml/git/0.git
	git clone --mirror https://lore.kernel.org/lkml/1 lkml/git/1.git
	git clone --mirror https://lore.kernel.org/lkml/2 lkml/git/2.git
	git clone --mirror https://lore.kernel.org/lkml/3 lkml/git/3.git
	git clone --mirror https://lore.kernel.org/lkml/4 lkml/git/4.git
	git clone --mirror https://lore.kernel.org/lkml/5 lkml/git/5.git
	git clone --mirror https://lore.kernel.org/lkml/6 lkml/git/6.git
	git clone --mirror https://lore.kernel.org/lkml/7 lkml/git/7.git
	git clone --mirror https://lore.kernel.org/lkml/8 lkml/git/8.git
	git clone --mirror https://lore.kernel.org/lkml/9 lkml/git/9.git

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V2 lkml lkml/ https://lore.kernel.org/lkml \
		linux-kernel@vger.kernel.org
	public-inbox-index lkml

Example config snippet for mirrors

Newsgroup available over NNTP:
	nntp://nntp.lore.kernel.org/org.kernel.vger.linux-kernel


AGPL code for this site: git clone https://public-inbox.org/public-inbox.git