From: Muneendra <muneendra.kumar@broadcom.com>
To: linux-block@vger.kernel.org, linux-scsi@vger.kernel.org,
tj@kernel.org, linux-nvme@lists.infradead.org, hare@suse.de
Cc: jsmart2021@gmail.com, emilne@redhat.com, mkumar@redhat.com,
Gaurav Srivastava <gaurav.srivastava@broadcom.com>
Subject: [PATCH v9 04/13] lpfc: vmid: Add the datastructure for supporting VMID in lpfc
Date: Wed, 7 Apr 2021 04:36:28 +0530 [thread overview]
Message-ID: <1617750397-26466-5-git-send-email-muneendra.kumar@broadcom.com> (raw)
In-Reply-To: <1617750397-26466-1-git-send-email-muneendra.kumar@broadcom.com>
From: Gaurav Srivastava <gaurav.srivastava@broadcom.com>
This patch adds the primary datastructures needed to implement VMID
in lpfc driver. It maintains the capability, current state,
hash table for the vmid/appid along with other information.
The implementation supports the two versions of vmid implementation
(app header and priority tagging)
Signed-off-by: Gaurav Srivastava <gaurav.srivastava@broadcom.com>
Signed-off-by: James Smart <jsmart2021@gmail.com>
---
v9:
Updated the data structures
Merged patch 5 and 6 of previous version v8 to this patch
v8:
modify structure member to uniform data type naming scheme
v7:
No change
v6:
No change
v5:
No Change
v4:
No change
v3:
No change
v2:
Ported the patch on top of 5.10/scsi-queue
Removed unused variable.
---
drivers/scsi/lpfc/lpfc.h | 122 +++++++++++++++++++++++++++++++++
drivers/scsi/lpfc/lpfc_crtn.h | 11 +++
drivers/scsi/lpfc/lpfc_disc.h | 1 +
drivers/scsi/lpfc/lpfc_hw.h | 124 ++++++++++++++++++++++++++++++++--
drivers/scsi/lpfc/lpfc_sli.h | 8 +++
5 files changed, 262 insertions(+), 4 deletions(-)
diff --git a/drivers/scsi/lpfc/lpfc.h b/drivers/scsi/lpfc/lpfc.h
index f8de0d10620b..4d56cb77fe77 100644
--- a/drivers/scsi/lpfc/lpfc.h
+++ b/drivers/scsi/lpfc/lpfc.h
@@ -303,6 +303,64 @@ struct lpfc_stats {
struct lpfc_hba;
+#define LPFC_VMID_TIMER 300 /* timer interval in seconds. */
+
+#define LPFC_MAX_VMID_SIZE 256
+#define LPFC_COMPRESS_VMID_SIZE 16
+
+union lpfc_vmid_io_tag {
+ u32 app_id; /* App Id vmid */
+ u8 cs_ctl_vmid; /* Priority tag vmid */
+};
+
+#define JIFFIES_PER_HR (HZ * 60 * 60)
+
+struct lpfc_vmid {
+ u8 flag;
+#define LPFC_VMID_SLOT_FREE 0x0
+#define LPFC_VMID_SLOT_USED 0x1
+#define LPFC_VMID_REQ_REGISTER 0x2
+#define LPFC_VMID_REGISTERED 0x4
+#define LPFC_VMID_DE_REGISTER 0x8
+ char host_vmid[LPFC_MAX_VMID_SIZE];
+ union lpfc_vmid_io_tag un;
+ struct hlist_node hnode;
+ u64 io_rd_cnt;
+ u64 io_wr_cnt;
+ u8 vmid_len;
+ u8 delete_inactive; /* Delete if inactive flag 0 = no, 1 = yes */
+ u32 hash_index;
+ u64 __percpu *last_io_time;
+};
+
+#define lpfc_vmid_is_type_priority_tag(vport)\
+ (vport->vmid_priority_tagging ? 1 : 0)
+
+#define LPFC_VMID_HASH_SIZE 256
+#define LPFC_VMID_HASH_MASK 255
+#define LPFC_VMID_HASH_SHIFT 6
+
+struct lpfc_vmid_context {
+ struct lpfc_vmid *vmp;
+ struct lpfc_nodelist *nlp;
+ bool instantiated;
+};
+
+struct lpfc_vmid_priority_range {
+ u8 low;
+ u8 high;
+ u8 qos;
+};
+
+struct lpfc_vmid_priority_info {
+ u32 num_descriptors;
+ struct lpfc_vmid_priority_range *vmid_range;
+};
+
+#define QFPA_EVEN_ONLY 0x01
+#define QFPA_ODD_ONLY 0x02
+#define QFPA_EVEN_ODD 0x03
+
enum discovery_state {
LPFC_VPORT_UNKNOWN = 0, /* vport state is unknown */
LPFC_VPORT_FAILED = 1, /* vport has failed */
@@ -442,6 +500,9 @@ struct lpfc_vport {
#define WORKER_RAMP_DOWN_QUEUE 0x800 /* hba: Decrease Q depth */
#define WORKER_RAMP_UP_QUEUE 0x1000 /* hba: Increase Q depth */
#define WORKER_SERVICE_TXQ 0x2000 /* hba: IOCBs on the txq */
+#define WORKER_CHECK_INACTIVE_VMID 0x4000 /* hba: check inactive vmids */
+#define WORKER_CHECK_VMID_ISSUE_QFPA 0x8000 /* vport: Check if qfpa need */
+ /* to issue */
struct timer_list els_tmofunc;
struct timer_list delayed_disc_tmo;
@@ -452,6 +513,8 @@ struct lpfc_vport {
#define FC_LOADING 0x1 /* HBA in process of loading drvr */
#define FC_UNLOADING 0x2 /* HBA in process of unloading drvr */
#define FC_ALLOW_FDMI 0x4 /* port is ready for FDMI requests */
+#define FC_ALLOW_VMID 0x8 /* Allow VMID IO's */
+#define FC_DEREGISTER_ALL_APP_ID 0x10 /* Deregister all vmid's */
/* Vport Config Parameters */
uint32_t cfg_scan_down;
uint32_t cfg_lun_queue_depth;
@@ -470,9 +533,36 @@ struct lpfc_vport {
uint32_t cfg_tgt_queue_depth;
uint32_t cfg_first_burst_size;
uint32_t dev_loss_tmo_changed;
+ /* VMID parameters */
+ u8 lpfc_vmid_host_uuid[LPFC_COMPRESS_VMID_SIZE];
+ u32 max_vmid; /* maximum VMIDs allowed per port */
+ u32 cur_vmid_cnt; /* Current VMID count */
+#define LPFC_MIN_VMID 4
+#define LPFC_MAX_VMID 255
+ u32 vmid_inactivity_timeout; /* Time after which the VMID */
+ /* deregisters from switch */
+ u32 vmid_priority_tagging;
+#define LPFC_VMID_PRIO_TAG_DISABLE 0 /* Disable */
+#define LPFC_VMID_PRIO_TAG_SUP_TARGETS 1 /* Allow supported targets only */
+#define LPFC_VMID_PRIO_TAG_ALL_TARGETS 2 /* Allow all targets */
+ unsigned long *vmid_priority_range;
+#define LPFC_VMID_MAX_PRIORITY_RANGE 256
+#define LPFC_VMID_PRIORITY_BITMAP_SIZE 32
+ u8 vmid_flag;
+#define LPFC_VMID_IN_USE 0x1
+#define LPFC_VMID_ISSUE_QFPA 0x2
+#define LPFC_VMID_QFPA_CMPL 0x4
+#define LPFC_VMID_QOS_ENABLED 0x8
+#define LPFC_VMID_TIMER_ENBLD 0x10
+ struct fc_qfpa_res *qfpa_res;
struct fc_vport *fc_vport;
+ struct lpfc_vmid *vmid;
+ DECLARE_HASHTABLE(hash_table, 8);
+ rwlock_t vmid_lock;
+ struct lpfc_vmid_priority_info vmid_priority;
+
#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
struct dentry *debug_disc_trc;
struct dentry *debug_nodelist;
@@ -938,6 +1028,13 @@ struct lpfc_hba {
struct nvmet_fc_target_port *targetport;
lpfc_vpd_t vpd; /* vital product data */
+ u32 cfg_max_vmid; /* maximum VMIDs allowed per port */
+ u32 cfg_vmid_app_header;
+#define LPFC_VMID_APP_HEADER_DISABLE 0
+#define LPFC_VMID_APP_HEADER_ENABLE 1
+ u32 cfg_vmid_priority_tagging;
+ u32 cfg_vmid_inactivity_timeout; /* Time after which the VMID */
+ /* deregisters from switch */
struct pci_dev *pcidev;
struct list_head work_list;
uint32_t work_ha; /* Host Attention Bits for WT */
@@ -1178,6 +1275,7 @@ struct lpfc_hba {
struct list_head ct_ev_waiters;
struct unsol_rcv_ct_ctx ct_ctx[LPFC_CT_CTX_MAX];
uint32_t ctx_idx;
+ struct timer_list inactive_vmid_poll;
/* RAS Support */
struct lpfc_ras_fwlog ras_fwlog;
@@ -1419,3 +1517,27 @@ static const char *routine(enum enum_name table_key) \
} \
return name; \
}
+
+/**
+ * lpfc_is_vmid_enabled - returns if VMID is enabled for either switch types
+ * @phba: Pointer to HBA context object.
+ *
+ * Relationship between the enable, target support and if vmid tag is required
+ * for the particular combination
+ * ---------------------------------------------------
+ * Switch Enable Flag Target Support VMID Needed
+ * ---------------------------------------------------
+ * App Id 0 NA N
+ * App Id 1 0 N
+ * App Id 1 1 Y
+ * Pr Tag 0 NA N
+ * Pr Tag 1 0 N
+ * Pr Tag 1 1 Y
+ * Pr Tag 2 * Y
+ ---------------------------------------------------
+ *
+ **/
+static inline int lpfc_is_vmid_enabled(struct lpfc_hba *phba)
+{
+ return phba->cfg_vmid_app_header || phba->cfg_vmid_priority_tagging;
+}
diff --git a/drivers/scsi/lpfc/lpfc_crtn.h b/drivers/scsi/lpfc/lpfc_crtn.h
index eb4cf36229d5..16dea9a4a305 100644
--- a/drivers/scsi/lpfc/lpfc_crtn.h
+++ b/drivers/scsi/lpfc/lpfc_crtn.h
@@ -610,3 +610,14 @@ extern unsigned long lpfc_no_hba_reset[];
extern union lpfc_wqe128 lpfc_iread_cmd_template;
extern union lpfc_wqe128 lpfc_iwrite_cmd_template;
extern union lpfc_wqe128 lpfc_icmnd_cmd_template;
+
+/* vmid interface */
+int lpfc_vmid_uvem(struct lpfc_vport *vport, struct lpfc_vmid *vmid, bool ins);
+uint32_t lpfc_vmid_get_cs_ctl(struct lpfc_vport *vport);
+int lpfc_vmid_cmd(struct lpfc_vport *vport,
+ int cmdcode, struct lpfc_vmid *vmid);
+int lpfc_vmid_hash_fn(const char *vmid, int len);
+struct lpfc_vmid *lpfc_get_vmid_from_hastable(struct lpfc_vport *vport,
+ uint32_t hash, uint8_t *buf);
+void lpfc_vmid_vport_cleanup(struct lpfc_vport *vport);
+int lpfc_issue_els_qfpa(struct lpfc_vport *vport);
diff --git a/drivers/scsi/lpfc/lpfc_disc.h b/drivers/scsi/lpfc/lpfc_disc.h
index 08999aad6a10..3942dd890931 100644
--- a/drivers/scsi/lpfc/lpfc_disc.h
+++ b/drivers/scsi/lpfc/lpfc_disc.h
@@ -124,6 +124,7 @@ struct lpfc_nodelist {
uint8_t nlp_fcp_info; /* class info, bits 0-3 */
#define NLP_FCP_2_DEVICE 0x10 /* FCP-2 device */
u8 nlp_nvme_info; /* NVME NSLER Support */
+ uint8_t vmid_support; /* destination VMID support */
#define NLP_NVME_NSLER 0x1 /* NVME NSLER device */
struct timer_list nlp_delayfunc; /* Used for delayed ELS cmds */
diff --git a/drivers/scsi/lpfc/lpfc_hw.h b/drivers/scsi/lpfc/lpfc_hw.h
index 42682d95af52..4a5a85ed42ec 100644
--- a/drivers/scsi/lpfc/lpfc_hw.h
+++ b/drivers/scsi/lpfc/lpfc_hw.h
@@ -275,6 +275,7 @@ struct lpfc_sli_ct_request {
#define SLI_CT_ACCESS_DENIED 0x10
#define SLI_CT_INVALID_PORT_ID 0x11
#define SLI_CT_DATABASE_EMPTY 0x12
+#define SLI_CT_APP_ID_NOT_AVAILABLE 0x40
/*
* Name Server Command Codes
@@ -400,16 +401,16 @@ struct csp {
uint16_t altBbCredit:1; /* FC Word 1, bit 27 */
uint16_t edtovResolution:1; /* FC Word 1, bit 26 */
uint16_t multicast:1; /* FC Word 1, bit 25 */
- uint16_t broadcast:1; /* FC Word 1, bit 24 */
+ uint16_t app_hdr_support:1; /* FC Word 1, bit 24 */
- uint16_t huntgroup:1; /* FC Word 1, bit 23 */
+ uint16_t priority_tagging:1; /* FC Word 1, bit 23 */
uint16_t simplex:1; /* FC Word 1, bit 22 */
uint16_t word1Reserved1:3; /* FC Word 1, bit 21:19 */
uint16_t dhd:1; /* FC Word 1, bit 18 */
uint16_t contIncSeqCnt:1; /* FC Word 1, bit 17 */
uint16_t payloadlength:1; /* FC Word 1, bit 16 */
#else /* __LITTLE_ENDIAN_BITFIELD */
- uint16_t broadcast:1; /* FC Word 1, bit 24 */
+ uint16_t app_hdr_support:1; /* FC Word 1, bit 24 */
uint16_t multicast:1; /* FC Word 1, bit 25 */
uint16_t edtovResolution:1; /* FC Word 1, bit 26 */
uint16_t altBbCredit:1; /* FC Word 1, bit 27 */
@@ -423,7 +424,7 @@ struct csp {
uint16_t dhd:1; /* FC Word 1, bit 18 */
uint16_t word1Reserved1:3; /* FC Word 1, bit 21:19 */
uint16_t simplex:1; /* FC Word 1, bit 22 */
- uint16_t huntgroup:1; /* FC Word 1, bit 23 */
+ uint16_t priority_tagging:1; /* FC Word 1, bit 23 */
#endif
uint8_t bbRcvSizeMsb; /* Upper nibble is reserved */
@@ -607,6 +608,8 @@ struct fc_vft_header {
#define ELS_CMD_LIRR 0x7A000000
#define ELS_CMD_LCB 0x81000000
#define ELS_CMD_FPIN 0x16000000
+#define ELS_CMD_QFPA 0xB0000000
+#define ELS_CMD_UVEM 0xB1000000
#else /* __LITTLE_ENDIAN_BITFIELD */
#define ELS_CMD_MASK 0xffff
#define ELS_RSP_MASK 0xff
@@ -649,6 +652,8 @@ struct fc_vft_header {
#define ELS_CMD_LIRR 0x7A
#define ELS_CMD_LCB 0x81
#define ELS_CMD_FPIN ELS_FPIN
+#define ELS_CMD_QFPA 0xB0
+#define ELS_CMD_UVEM 0xB1
#endif
/*
@@ -1317,6 +1322,117 @@ struct fc_rdp_res_frame {
};
+/* UVEM */
+
+#define LPFC_UVEM_SIZE 60
+#define LPFC_UVEM_VEM_ID_DESC_SIZE 16
+#define LPFC_UVEM_VE_MAP_DESC_SIZE 20
+
+#define VEM_ID_DESC_TAG 0x0001000A
+struct lpfc_vem_id_desc {
+ uint32_t tag;
+ uint32_t length;
+ uint8_t vem_id[16];
+};
+
+#define LPFC_QFPA_SIZE 4
+
+#define INSTANTIATED_VE_DESC_TAG 0x0001000B
+struct instantiated_ve_desc {
+ uint32_t tag;
+ uint32_t length;
+ uint8_t global_vem_id[16];
+ uint32_t word6;
+#define lpfc_instantiated_local_id_SHIFT 0
+#define lpfc_instantiated_local_id_MASK 0x000000ff
+#define lpfc_instantiated_local_id_WORD word6
+#define lpfc_instantiated_nport_id_SHIFT 8
+#define lpfc_instantiated_nport_id_MASK 0x00ffffff
+#define lpfc_instantiated_nport_id_WORD word6
+};
+
+#define DEINSTANTIATED_VE_DESC_TAG 0x0001000C
+struct deinstantiated_ve_desc {
+ uint32_t tag;
+ uint32_t length;
+ uint8_t global_vem_id[16];
+ uint32_t word6;
+#define lpfc_deinstantiated_nport_id_SHIFT 0
+#define lpfc_deinstantiated_nport_id_MASK 0x000000ff
+#define lpfc_deinstantiated_nport_id_WORD word6
+#define lpfc_deinstantiated_local_id_SHIFT 24
+#define lpfc_deinstantiated_local_id_MASK 0x00ffffff
+#define lpfc_deinstantiated_local_id_WORD word6
+};
+
+/* Query Fabric Priority Allocation Response */
+#define LPFC_PRIORITY_RANGE_DESC_SIZE 12
+
+struct priority_range_desc {
+ uint32_t tag;
+ uint32_t length;
+ uint8_t lo_range;
+ uint8_t hi_range;
+ uint8_t qos_priority;
+ uint8_t local_ve_id;
+};
+
+struct fc_qfpa_res {
+ uint32_t reply_sequence; /* LS_ACC or LS_RJT */
+ uint32_t length; /* FC Word 1 */
+ struct priority_range_desc desc[1];
+};
+
+/* Application Server command code */
+/* VMID */
+
+#define SLI_CT_APP_SEV_Subtypes 0x20 /* Application Server subtype */
+
+#define SLI_CTAS_GAPPIA_ENT 0x0100 /* Get Application Identifier */
+#define SLI_CTAS_GALLAPPIA 0x0101 /* Get All Application Identifier */
+#define SLI_CTAS_GALLAPPIA_ID 0x0102 /* Get All Application Identifier */
+ /* for Nport */
+#define SLI_CTAS_GAPPIA_IDAPP 0x0103 /* Get Application Identifier */
+ /* for Nport */
+#define SLI_CTAS_RAPP_IDENT 0x0200 /* Register Application Identifier */
+#define SLI_CTAS_DAPP_IDENT 0x0300 /* Deregister Application */
+ /* Identifier */
+#define SLI_CTAS_DALLAPP_ID 0x0301 /* Deregister All Application */
+ /* Identifier */
+
+struct entity_id_object {
+ uint8_t entity_id_len;
+ uint8_t entity_id[255]; /* VM UUID */
+};
+
+struct app_id_object {
+ uint32_t port_id;
+ uint32_t app_id;
+ struct entity_id_object obj;
+};
+
+struct lpfc_vmid_rapp_ident_list {
+ uint32_t no_of_objects;
+ struct entity_id_object obj[1];
+};
+
+struct lpfc_vmid_dapp_ident_list {
+ uint32_t no_of_objects;
+ struct entity_id_object obj[1];
+};
+
+#define GALLAPPIA_ID_LAST 0x80
+struct lpfc_vmid_gallapp_ident_list {
+ uint8_t control;
+ uint8_t reserved[3];
+ struct app_id_object app_id;
+};
+
+#define RAPP_IDENT_OFFSET (offsetof(struct lpfc_sli_ct_request, un) + 4)
+#define DAPP_IDENT_OFFSET (offsetof(struct lpfc_sli_ct_request, un) + 4)
+#define GALLAPPIA_ID_SIZE (offsetof(struct lpfc_sli_ct_request, un) + 4)
+#define DALLAPP_ID_SIZE (offsetof(struct lpfc_sli_ct_request, un) + 4)
+
/******** FDMI ********/
/* lpfc_sli_ct_request defines the CT_IU preamble for FDMI commands */
diff --git a/drivers/scsi/lpfc/lpfc_sli.h b/drivers/scsi/lpfc/lpfc_sli.h
index 4f6936014ff5..ff5a2a492405 100644
--- a/drivers/scsi/lpfc/lpfc_sli.h
+++ b/drivers/scsi/lpfc/lpfc_sli.h
@@ -35,6 +35,12 @@ typedef enum _lpfc_ctx_cmd {
LPFC_CTX_HOST
} lpfc_ctx_cmd;
+union lpfc_vmid_iocb_tag {
+ uint32_t app_id;
+ uint8_t cs_ctl_vmid;
+ struct lpfc_vmid_context *vmid_context; /* UVEM context information */
+};
+
struct lpfc_cq_event {
struct list_head list;
uint16_t hdwq;
@@ -100,6 +106,7 @@ struct lpfc_iocbq {
#define LPFC_IO_NVME 0x200000 /* NVME FCP command */
#define LPFC_IO_NVME_LS 0x400000 /* NVME LS command */
#define LPFC_IO_NVMET 0x800000 /* NVMET command */
+#define LPFC_IO_VMID 0x1000000 /* VMID tagged IO */
uint32_t drvrTimeout; /* driver timeout in seconds */
struct lpfc_vport *vport;/* virtual port pointer */
@@ -114,6 +121,7 @@ struct lpfc_iocbq {
struct lpfc_node_rrq *rrq;
} context_un;
+ union lpfc_vmid_iocb_tag vmid_tag;
void (*fabric_iocb_cmpl)(struct lpfc_hba *, struct lpfc_iocbq *,
struct lpfc_iocbq *);
void (*wait_iocb_cmpl)(struct lpfc_hba *, struct lpfc_iocbq *,
--
2.26.2
WARNING: multiple messages have this Message-ID (diff)
From: Muneendra <muneendra.kumar@broadcom.com>
To: linux-block@vger.kernel.org, linux-scsi@vger.kernel.org,
tj@kernel.org, linux-nvme@lists.infradead.org, hare@suse.de
Cc: jsmart2021@gmail.com, emilne@redhat.com, mkumar@redhat.com,
Gaurav Srivastava <gaurav.srivastava@broadcom.com>
Subject: [PATCH v9 04/13] lpfc: vmid: Add the datastructure for supporting VMID in lpfc
Date: Wed, 7 Apr 2021 04:36:28 +0530 [thread overview]
Message-ID: <1617750397-26466-5-git-send-email-muneendra.kumar@broadcom.com> (raw)
In-Reply-To: <1617750397-26466-1-git-send-email-muneendra.kumar@broadcom.com>
From: Gaurav Srivastava <gaurav.srivastava@broadcom.com>
This patch adds the primary datastructures needed to implement VMID
in lpfc driver. It maintains the capability, current state,
hash table for the vmid/appid along with other information.
The implementation supports the two versions of vmid implementation
(app header and priority tagging)
Signed-off-by: Gaurav Srivastava <gaurav.srivastava@broadcom.com>
Signed-off-by: James Smart <jsmart2021@gmail.com>
---
v9:
Updated the data structures
Merged patch 5 and 6 of previous version v8 to this patch
v8:
modify structure member to uniform data type naming scheme
v7:
No change
v6:
No change
v5:
No Change
v4:
No change
v3:
No change
v2:
Ported the patch on top of 5.10/scsi-queue
Removed unused variable.
---
drivers/scsi/lpfc/lpfc.h | 122 +++++++++++++++++++++++++++++++++
drivers/scsi/lpfc/lpfc_crtn.h | 11 +++
drivers/scsi/lpfc/lpfc_disc.h | 1 +
drivers/scsi/lpfc/lpfc_hw.h | 124 ++++++++++++++++++++++++++++++++--
drivers/scsi/lpfc/lpfc_sli.h | 8 +++
5 files changed, 262 insertions(+), 4 deletions(-)
diff --git a/drivers/scsi/lpfc/lpfc.h b/drivers/scsi/lpfc/lpfc.h
index f8de0d10620b..4d56cb77fe77 100644
--- a/drivers/scsi/lpfc/lpfc.h
+++ b/drivers/scsi/lpfc/lpfc.h
@@ -303,6 +303,64 @@ struct lpfc_stats {
struct lpfc_hba;
+#define LPFC_VMID_TIMER 300 /* timer interval in seconds. */
+
+#define LPFC_MAX_VMID_SIZE 256
+#define LPFC_COMPRESS_VMID_SIZE 16
+
+union lpfc_vmid_io_tag {
+ u32 app_id; /* App Id vmid */
+ u8 cs_ctl_vmid; /* Priority tag vmid */
+};
+
+#define JIFFIES_PER_HR (HZ * 60 * 60)
+
+struct lpfc_vmid {
+ u8 flag;
+#define LPFC_VMID_SLOT_FREE 0x0
+#define LPFC_VMID_SLOT_USED 0x1
+#define LPFC_VMID_REQ_REGISTER 0x2
+#define LPFC_VMID_REGISTERED 0x4
+#define LPFC_VMID_DE_REGISTER 0x8
+ char host_vmid[LPFC_MAX_VMID_SIZE];
+ union lpfc_vmid_io_tag un;
+ struct hlist_node hnode;
+ u64 io_rd_cnt;
+ u64 io_wr_cnt;
+ u8 vmid_len;
+ u8 delete_inactive; /* Delete if inactive flag 0 = no, 1 = yes */
+ u32 hash_index;
+ u64 __percpu *last_io_time;
+};
+
+#define lpfc_vmid_is_type_priority_tag(vport)\
+ (vport->vmid_priority_tagging ? 1 : 0)
+
+#define LPFC_VMID_HASH_SIZE 256
+#define LPFC_VMID_HASH_MASK 255
+#define LPFC_VMID_HASH_SHIFT 6
+
+struct lpfc_vmid_context {
+ struct lpfc_vmid *vmp;
+ struct lpfc_nodelist *nlp;
+ bool instantiated;
+};
+
+struct lpfc_vmid_priority_range {
+ u8 low;
+ u8 high;
+ u8 qos;
+};
+
+struct lpfc_vmid_priority_info {
+ u32 num_descriptors;
+ struct lpfc_vmid_priority_range *vmid_range;
+};
+
+#define QFPA_EVEN_ONLY 0x01
+#define QFPA_ODD_ONLY 0x02
+#define QFPA_EVEN_ODD 0x03
+
enum discovery_state {
LPFC_VPORT_UNKNOWN = 0, /* vport state is unknown */
LPFC_VPORT_FAILED = 1, /* vport has failed */
@@ -442,6 +500,9 @@ struct lpfc_vport {
#define WORKER_RAMP_DOWN_QUEUE 0x800 /* hba: Decrease Q depth */
#define WORKER_RAMP_UP_QUEUE 0x1000 /* hba: Increase Q depth */
#define WORKER_SERVICE_TXQ 0x2000 /* hba: IOCBs on the txq */
+#define WORKER_CHECK_INACTIVE_VMID 0x4000 /* hba: check inactive vmids */
+#define WORKER_CHECK_VMID_ISSUE_QFPA 0x8000 /* vport: Check if qfpa need */
+ /* to issue */
struct timer_list els_tmofunc;
struct timer_list delayed_disc_tmo;
@@ -452,6 +513,8 @@ struct lpfc_vport {
#define FC_LOADING 0x1 /* HBA in process of loading drvr */
#define FC_UNLOADING 0x2 /* HBA in process of unloading drvr */
#define FC_ALLOW_FDMI 0x4 /* port is ready for FDMI requests */
+#define FC_ALLOW_VMID 0x8 /* Allow VMID IO's */
+#define FC_DEREGISTER_ALL_APP_ID 0x10 /* Deregister all vmid's */
/* Vport Config Parameters */
uint32_t cfg_scan_down;
uint32_t cfg_lun_queue_depth;
@@ -470,9 +533,36 @@ struct lpfc_vport {
uint32_t cfg_tgt_queue_depth;
uint32_t cfg_first_burst_size;
uint32_t dev_loss_tmo_changed;
+ /* VMID parameters */
+ u8 lpfc_vmid_host_uuid[LPFC_COMPRESS_VMID_SIZE];
+ u32 max_vmid; /* maximum VMIDs allowed per port */
+ u32 cur_vmid_cnt; /* Current VMID count */
+#define LPFC_MIN_VMID 4
+#define LPFC_MAX_VMID 255
+ u32 vmid_inactivity_timeout; /* Time after which the VMID */
+ /* deregisters from switch */
+ u32 vmid_priority_tagging;
+#define LPFC_VMID_PRIO_TAG_DISABLE 0 /* Disable */
+#define LPFC_VMID_PRIO_TAG_SUP_TARGETS 1 /* Allow supported targets only */
+#define LPFC_VMID_PRIO_TAG_ALL_TARGETS 2 /* Allow all targets */
+ unsigned long *vmid_priority_range;
+#define LPFC_VMID_MAX_PRIORITY_RANGE 256
+#define LPFC_VMID_PRIORITY_BITMAP_SIZE 32
+ u8 vmid_flag;
+#define LPFC_VMID_IN_USE 0x1
+#define LPFC_VMID_ISSUE_QFPA 0x2
+#define LPFC_VMID_QFPA_CMPL 0x4
+#define LPFC_VMID_QOS_ENABLED 0x8
+#define LPFC_VMID_TIMER_ENBLD 0x10
+ struct fc_qfpa_res *qfpa_res;
struct fc_vport *fc_vport;
+ struct lpfc_vmid *vmid;
+ DECLARE_HASHTABLE(hash_table, 8);
+ rwlock_t vmid_lock;
+ struct lpfc_vmid_priority_info vmid_priority;
+
#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
struct dentry *debug_disc_trc;
struct dentry *debug_nodelist;
@@ -938,6 +1028,13 @@ struct lpfc_hba {
struct nvmet_fc_target_port *targetport;
lpfc_vpd_t vpd; /* vital product data */
+ u32 cfg_max_vmid; /* maximum VMIDs allowed per port */
+ u32 cfg_vmid_app_header;
+#define LPFC_VMID_APP_HEADER_DISABLE 0
+#define LPFC_VMID_APP_HEADER_ENABLE 1
+ u32 cfg_vmid_priority_tagging;
+ u32 cfg_vmid_inactivity_timeout; /* Time after which the VMID */
+ /* deregisters from switch */
struct pci_dev *pcidev;
struct list_head work_list;
uint32_t work_ha; /* Host Attention Bits for WT */
@@ -1178,6 +1275,7 @@ struct lpfc_hba {
struct list_head ct_ev_waiters;
struct unsol_rcv_ct_ctx ct_ctx[LPFC_CT_CTX_MAX];
uint32_t ctx_idx;
+ struct timer_list inactive_vmid_poll;
/* RAS Support */
struct lpfc_ras_fwlog ras_fwlog;
@@ -1419,3 +1517,27 @@ static const char *routine(enum enum_name table_key) \
} \
return name; \
}
+
+/**
+ * lpfc_is_vmid_enabled - returns if VMID is enabled for either switch types
+ * @phba: Pointer to HBA context object.
+ *
+ * Relationship between the enable, target support and if vmid tag is required
+ * for the particular combination
+ * ---------------------------------------------------
+ * Switch Enable Flag Target Support VMID Needed
+ * ---------------------------------------------------
+ * App Id 0 NA N
+ * App Id 1 0 N
+ * App Id 1 1 Y
+ * Pr Tag 0 NA N
+ * Pr Tag 1 0 N
+ * Pr Tag 1 1 Y
+ * Pr Tag 2 * Y
+ ---------------------------------------------------
+ *
+ **/
+static inline int lpfc_is_vmid_enabled(struct lpfc_hba *phba)
+{
+ return phba->cfg_vmid_app_header || phba->cfg_vmid_priority_tagging;
+}
diff --git a/drivers/scsi/lpfc/lpfc_crtn.h b/drivers/scsi/lpfc/lpfc_crtn.h
index eb4cf36229d5..16dea9a4a305 100644
--- a/drivers/scsi/lpfc/lpfc_crtn.h
+++ b/drivers/scsi/lpfc/lpfc_crtn.h
@@ -610,3 +610,14 @@ extern unsigned long lpfc_no_hba_reset[];
extern union lpfc_wqe128 lpfc_iread_cmd_template;
extern union lpfc_wqe128 lpfc_iwrite_cmd_template;
extern union lpfc_wqe128 lpfc_icmnd_cmd_template;
+
+/* vmid interface */
+int lpfc_vmid_uvem(struct lpfc_vport *vport, struct lpfc_vmid *vmid, bool ins);
+uint32_t lpfc_vmid_get_cs_ctl(struct lpfc_vport *vport);
+int lpfc_vmid_cmd(struct lpfc_vport *vport,
+ int cmdcode, struct lpfc_vmid *vmid);
+int lpfc_vmid_hash_fn(const char *vmid, int len);
+struct lpfc_vmid *lpfc_get_vmid_from_hastable(struct lpfc_vport *vport,
+ uint32_t hash, uint8_t *buf);
+void lpfc_vmid_vport_cleanup(struct lpfc_vport *vport);
+int lpfc_issue_els_qfpa(struct lpfc_vport *vport);
diff --git a/drivers/scsi/lpfc/lpfc_disc.h b/drivers/scsi/lpfc/lpfc_disc.h
index 08999aad6a10..3942dd890931 100644
--- a/drivers/scsi/lpfc/lpfc_disc.h
+++ b/drivers/scsi/lpfc/lpfc_disc.h
@@ -124,6 +124,7 @@ struct lpfc_nodelist {
uint8_t nlp_fcp_info; /* class info, bits 0-3 */
#define NLP_FCP_2_DEVICE 0x10 /* FCP-2 device */
u8 nlp_nvme_info; /* NVME NSLER Support */
+ uint8_t vmid_support; /* destination VMID support */
#define NLP_NVME_NSLER 0x1 /* NVME NSLER device */
struct timer_list nlp_delayfunc; /* Used for delayed ELS cmds */
diff --git a/drivers/scsi/lpfc/lpfc_hw.h b/drivers/scsi/lpfc/lpfc_hw.h
index 42682d95af52..4a5a85ed42ec 100644
--- a/drivers/scsi/lpfc/lpfc_hw.h
+++ b/drivers/scsi/lpfc/lpfc_hw.h
@@ -275,6 +275,7 @@ struct lpfc_sli_ct_request {
#define SLI_CT_ACCESS_DENIED 0x10
#define SLI_CT_INVALID_PORT_ID 0x11
#define SLI_CT_DATABASE_EMPTY 0x12
+#define SLI_CT_APP_ID_NOT_AVAILABLE 0x40
/*
* Name Server Command Codes
@@ -400,16 +401,16 @@ struct csp {
uint16_t altBbCredit:1; /* FC Word 1, bit 27 */
uint16_t edtovResolution:1; /* FC Word 1, bit 26 */
uint16_t multicast:1; /* FC Word 1, bit 25 */
- uint16_t broadcast:1; /* FC Word 1, bit 24 */
+ uint16_t app_hdr_support:1; /* FC Word 1, bit 24 */
- uint16_t huntgroup:1; /* FC Word 1, bit 23 */
+ uint16_t priority_tagging:1; /* FC Word 1, bit 23 */
uint16_t simplex:1; /* FC Word 1, bit 22 */
uint16_t word1Reserved1:3; /* FC Word 1, bit 21:19 */
uint16_t dhd:1; /* FC Word 1, bit 18 */
uint16_t contIncSeqCnt:1; /* FC Word 1, bit 17 */
uint16_t payloadlength:1; /* FC Word 1, bit 16 */
#else /* __LITTLE_ENDIAN_BITFIELD */
- uint16_t broadcast:1; /* FC Word 1, bit 24 */
+ uint16_t app_hdr_support:1; /* FC Word 1, bit 24 */
uint16_t multicast:1; /* FC Word 1, bit 25 */
uint16_t edtovResolution:1; /* FC Word 1, bit 26 */
uint16_t altBbCredit:1; /* FC Word 1, bit 27 */
@@ -423,7 +424,7 @@ struct csp {
uint16_t dhd:1; /* FC Word 1, bit 18 */
uint16_t word1Reserved1:3; /* FC Word 1, bit 21:19 */
uint16_t simplex:1; /* FC Word 1, bit 22 */
- uint16_t huntgroup:1; /* FC Word 1, bit 23 */
+ uint16_t priority_tagging:1; /* FC Word 1, bit 23 */
#endif
uint8_t bbRcvSizeMsb; /* Upper nibble is reserved */
@@ -607,6 +608,8 @@ struct fc_vft_header {
#define ELS_CMD_LIRR 0x7A000000
#define ELS_CMD_LCB 0x81000000
#define ELS_CMD_FPIN 0x16000000
+#define ELS_CMD_QFPA 0xB0000000
+#define ELS_CMD_UVEM 0xB1000000
#else /* __LITTLE_ENDIAN_BITFIELD */
#define ELS_CMD_MASK 0xffff
#define ELS_RSP_MASK 0xff
@@ -649,6 +652,8 @@ struct fc_vft_header {
#define ELS_CMD_LIRR 0x7A
#define ELS_CMD_LCB 0x81
#define ELS_CMD_FPIN ELS_FPIN
+#define ELS_CMD_QFPA 0xB0
+#define ELS_CMD_UVEM 0xB1
#endif
/*
@@ -1317,6 +1322,117 @@ struct fc_rdp_res_frame {
};
+/* UVEM */
+
+#define LPFC_UVEM_SIZE 60
+#define LPFC_UVEM_VEM_ID_DESC_SIZE 16
+#define LPFC_UVEM_VE_MAP_DESC_SIZE 20
+
+#define VEM_ID_DESC_TAG 0x0001000A
+struct lpfc_vem_id_desc {
+ uint32_t tag;
+ uint32_t length;
+ uint8_t vem_id[16];
+};
+
+#define LPFC_QFPA_SIZE 4
+
+#define INSTANTIATED_VE_DESC_TAG 0x0001000B
+struct instantiated_ve_desc {
+ uint32_t tag;
+ uint32_t length;
+ uint8_t global_vem_id[16];
+ uint32_t word6;
+#define lpfc_instantiated_local_id_SHIFT 0
+#define lpfc_instantiated_local_id_MASK 0x000000ff
+#define lpfc_instantiated_local_id_WORD word6
+#define lpfc_instantiated_nport_id_SHIFT 8
+#define lpfc_instantiated_nport_id_MASK 0x00ffffff
+#define lpfc_instantiated_nport_id_WORD word6
+};
+
+#define DEINSTANTIATED_VE_DESC_TAG 0x0001000C
+struct deinstantiated_ve_desc {
+ uint32_t tag;
+ uint32_t length;
+ uint8_t global_vem_id[16];
+ uint32_t word6;
+#define lpfc_deinstantiated_nport_id_SHIFT 0
+#define lpfc_deinstantiated_nport_id_MASK 0x000000ff
+#define lpfc_deinstantiated_nport_id_WORD word6
+#define lpfc_deinstantiated_local_id_SHIFT 24
+#define lpfc_deinstantiated_local_id_MASK 0x00ffffff
+#define lpfc_deinstantiated_local_id_WORD word6
+};
+
+/* Query Fabric Priority Allocation Response */
+#define LPFC_PRIORITY_RANGE_DESC_SIZE 12
+
+struct priority_range_desc {
+ uint32_t tag;
+ uint32_t length;
+ uint8_t lo_range;
+ uint8_t hi_range;
+ uint8_t qos_priority;
+ uint8_t local_ve_id;
+};
+
+struct fc_qfpa_res {
+ uint32_t reply_sequence; /* LS_ACC or LS_RJT */
+ uint32_t length; /* FC Word 1 */
+ struct priority_range_desc desc[1];
+};
+
+/* Application Server command code */
+/* VMID */
+
+#define SLI_CT_APP_SEV_Subtypes 0x20 /* Application Server subtype */
+
+#define SLI_CTAS_GAPPIA_ENT 0x0100 /* Get Application Identifier */
+#define SLI_CTAS_GALLAPPIA 0x0101 /* Get All Application Identifier */
+#define SLI_CTAS_GALLAPPIA_ID 0x0102 /* Get All Application Identifier */
+ /* for Nport */
+#define SLI_CTAS_GAPPIA_IDAPP 0x0103 /* Get Application Identifier */
+ /* for Nport */
+#define SLI_CTAS_RAPP_IDENT 0x0200 /* Register Application Identifier */
+#define SLI_CTAS_DAPP_IDENT 0x0300 /* Deregister Application */
+ /* Identifier */
+#define SLI_CTAS_DALLAPP_ID 0x0301 /* Deregister All Application */
+ /* Identifier */
+
+struct entity_id_object {
+ uint8_t entity_id_len;
+ uint8_t entity_id[255]; /* VM UUID */
+};
+
+struct app_id_object {
+ uint32_t port_id;
+ uint32_t app_id;
+ struct entity_id_object obj;
+};
+
+struct lpfc_vmid_rapp_ident_list {
+ uint32_t no_of_objects;
+ struct entity_id_object obj[1];
+};
+
+struct lpfc_vmid_dapp_ident_list {
+ uint32_t no_of_objects;
+ struct entity_id_object obj[1];
+};
+
+#define GALLAPPIA_ID_LAST 0x80
+struct lpfc_vmid_gallapp_ident_list {
+ uint8_t control;
+ uint8_t reserved[3];
+ struct app_id_object app_id;
+};
+
+#define RAPP_IDENT_OFFSET (offsetof(struct lpfc_sli_ct_request, un) + 4)
+#define DAPP_IDENT_OFFSET (offsetof(struct lpfc_sli_ct_request, un) + 4)
+#define GALLAPPIA_ID_SIZE (offsetof(struct lpfc_sli_ct_request, un) + 4)
+#define DALLAPP_ID_SIZE (offsetof(struct lpfc_sli_ct_request, un) + 4)
+
/******** FDMI ********/
/* lpfc_sli_ct_request defines the CT_IU preamble for FDMI commands */
diff --git a/drivers/scsi/lpfc/lpfc_sli.h b/drivers/scsi/lpfc/lpfc_sli.h
index 4f6936014ff5..ff5a2a492405 100644
--- a/drivers/scsi/lpfc/lpfc_sli.h
+++ b/drivers/scsi/lpfc/lpfc_sli.h
@@ -35,6 +35,12 @@ typedef enum _lpfc_ctx_cmd {
LPFC_CTX_HOST
} lpfc_ctx_cmd;
+union lpfc_vmid_iocb_tag {
+ uint32_t app_id;
+ uint8_t cs_ctl_vmid;
+ struct lpfc_vmid_context *vmid_context; /* UVEM context information */
+};
+
struct lpfc_cq_event {
struct list_head list;
uint16_t hdwq;
@@ -100,6 +106,7 @@ struct lpfc_iocbq {
#define LPFC_IO_NVME 0x200000 /* NVME FCP command */
#define LPFC_IO_NVME_LS 0x400000 /* NVME LS command */
#define LPFC_IO_NVMET 0x800000 /* NVMET command */
+#define LPFC_IO_VMID 0x1000000 /* VMID tagged IO */
uint32_t drvrTimeout; /* driver timeout in seconds */
struct lpfc_vport *vport;/* virtual port pointer */
@@ -114,6 +121,7 @@ struct lpfc_iocbq {
struct lpfc_node_rrq *rrq;
} context_un;
+ union lpfc_vmid_iocb_tag vmid_tag;
void (*fabric_iocb_cmpl)(struct lpfc_hba *, struct lpfc_iocbq *,
struct lpfc_iocbq *);
void (*wait_iocb_cmpl)(struct lpfc_hba *, struct lpfc_iocbq *,
--
2.26.2
_______________________________________________
Linux-nvme mailing list
Linux-nvme@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-nvme
next prev parent reply other threads:[~2021-04-07 5:59 UTC|newest]
Thread overview: 68+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-04-06 23:06 [PATCH v9 00/13] blkcg:Support to track FC storage blk io traffic Muneendra
2021-04-06 23:06 ` Muneendra
2021-04-06 23:06 ` [PATCH v9 01/13] cgroup: Added cgroup_get_from_id Muneendra
2021-04-06 23:06 ` Muneendra
2021-04-08 8:26 ` Hannes Reinecke
2021-04-08 8:26 ` Hannes Reinecke
2021-04-06 23:06 ` [PATCH v9 02/13] blkcg: Added a app identifier support for blkcg Muneendra
2021-04-06 23:06 ` Muneendra
2021-04-08 8:26 ` Hannes Reinecke
2021-04-08 8:26 ` Hannes Reinecke
2021-04-06 23:06 ` [PATCH v9 03/13] nvme: Added a newsysfs attribute appid_store Muneendra
2021-04-06 23:06 ` Muneendra
2021-04-18 15:32 ` Benjamin Block
2021-04-18 15:32 ` Benjamin Block
2021-04-20 6:54 ` Muneendra Kumar M
2021-04-20 6:54 ` Muneendra Kumar M
2021-04-20 11:09 ` Benjamin Block
2021-04-20 11:09 ` Benjamin Block
2021-04-22 23:29 ` James Smart
2021-04-22 23:29 ` James Smart
2021-04-23 10:14 ` Benjamin Block
2021-04-23 10:14 ` Benjamin Block
2021-04-06 23:06 ` Muneendra [this message]
2021-04-06 23:06 ` [PATCH v9 04/13] lpfc: vmid: Add the datastructure for supporting VMID in lpfc Muneendra
2021-04-08 8:28 ` Hannes Reinecke
2021-04-08 8:28 ` Hannes Reinecke
2021-04-06 23:06 ` [PATCH v9 05/13] lpfc: vmid: VMID params initialization Muneendra
2021-04-06 23:06 ` Muneendra
2021-04-08 8:29 ` Hannes Reinecke
2021-04-08 8:29 ` Hannes Reinecke
2021-04-06 23:06 ` [PATCH v9 06/13] lpfc: vmid: Add support for vmid in mailbox command, does vmid resource allocation and vmid cleanup Muneendra
2021-04-06 23:06 ` Muneendra
2021-04-08 8:32 ` Hannes Reinecke
2021-04-08 8:32 ` Hannes Reinecke
2021-04-06 23:06 ` [PATCH v9 07/13] lpfc: vmid: Implements ELS commands for appid patch Muneendra
2021-04-06 23:06 ` Muneendra
2021-04-08 8:34 ` Hannes Reinecke
2021-04-08 8:34 ` Hannes Reinecke
2021-04-20 12:38 ` Benjamin Block
2021-04-20 12:38 ` Benjamin Block
2021-04-21 22:55 ` James Smart
2021-04-21 22:55 ` James Smart
2021-04-22 9:28 ` Benjamin Block
2021-04-22 9:28 ` Benjamin Block
2021-04-06 23:06 ` [PATCH v9 08/13] lpfc: vmid: Functions to manage vmids Muneendra
2021-04-06 23:06 ` Muneendra
2021-04-08 8:35 ` Hannes Reinecke
2021-04-08 8:35 ` Hannes Reinecke
2021-04-06 23:06 ` [PATCH v9 09/13] lpfc: vmid: Implements CT commands for appid Muneendra
2021-04-06 23:06 ` Muneendra
2021-04-08 8:37 ` Hannes Reinecke
2021-04-08 8:37 ` Hannes Reinecke
2021-04-06 23:06 ` [PATCH v9 10/13] lpfc: vmid: Appends the vmid in the wqe before sending Muneendra
2021-04-06 23:06 ` Muneendra
2021-04-06 23:06 ` [PATCH v9 11/13] lpfc: vmid: Timeout implementation for vmid Muneendra
2021-04-06 23:06 ` Muneendra
2021-04-08 8:38 ` Hannes Reinecke
2021-04-08 8:38 ` Hannes Reinecke
2021-04-06 23:06 ` [PATCH v9 12/13] lpfc: vmid: Adding qfpa and vmid timeout check in worker thread Muneendra
2021-04-06 23:06 ` Muneendra
2021-04-06 23:06 ` [PATCH v9 13/13] lpfc: vmid: Introducing vmid in io path Muneendra
2021-04-06 23:06 ` Muneendra
2021-04-08 8:46 ` Hannes Reinecke
2021-04-08 8:46 ` Hannes Reinecke
2021-04-10 15:00 ` James Smart
2021-04-10 15:00 ` James Smart
2021-04-12 5:27 ` Hannes Reinecke
2021-04-12 5:27 ` Hannes Reinecke
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1617750397-26466-5-git-send-email-muneendra.kumar@broadcom.com \
--to=muneendra.kumar@broadcom.com \
--cc=emilne@redhat.com \
--cc=gaurav.srivastava@broadcom.com \
--cc=hare@suse.de \
--cc=jsmart2021@gmail.com \
--cc=linux-block@vger.kernel.org \
--cc=linux-nvme@lists.infradead.org \
--cc=linux-scsi@vger.kernel.org \
--cc=mkumar@redhat.com \
--cc=tj@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.