PCIe Data Object Exchange (DOE) protocol for PCIe and CXL is available > > https://gitlab.com/avery-qemu/cxl2.0-v3-doe/ > > based on Ben Widawsky's CXL QEMU cxl2.0-v3 gitlab branch > > https://lore.kernel.org/qemu-devel/20210202005948.241655-1-ben.widawsky@intel.com > > which is located at > > https://gitlab.com/bwidawsk/qemu > > The changes from Ben┬ latest cxl-2.0v3 are: > > MAINTAINERS | 7 + > hw/cxl/cxl-component-utils.c | 151 +++++++++++ > hw/mem/cxl_type3.c | 121 +++++++++ > hw/pci/meson.build | 1 + > hw/pci/pcie.c | 4 +- > hw/pci/pcie_doe.c | 360 +++++++++++++++++++++++++ > include/hw/cxl/cxl_component.h | 120 +++++++++ > include/hw/cxl/cxl_pci.h | 428 > ++++++++++++++++++++++++++++++ > include/hw/pci/pcie.h | 5 + > include/hw/pci/pcie_doe.h | 130 +++++++++ > include/hw/pci/pcie_regs.h | 4 + > include/standard-headers/linux/pci_regs.h | 3 +- > 12 files changed, 1332 insertions(+), 2 deletions(-) > > The DOE protocol defines a mailbox method that allows either UEFI or OS > methods > to read the device and do further setup of ACPI tables, etc. > > There are 2 PCIe DOE protocols (PCI-SIG ECN Data Object Exchange (DOE) March > 2020) > - Discovery > - Component Measurement (CMA) > > And 2 CXL specific ones: > - Compliance Mode (Compute Express Link Specification September 2, 2020 > Revision: 2.0, Version 1.0) > - CDAT (Coherent Device Attribute Table (CDAT) Specification > October 2020 Revision 1.02) > > For CXL, the CDAT table defines the memory device so that UEFI or OS can read > it out of device using DOE and then can configure the system┬ ACPI SRAT/HMAT > tables for system memory, and DEVSEC, Component, and Device registers in CXL > device. > > Current version provides fixed CDAT table defined in the CXL Type3 device > model. Updates are planned shortly to allow for user to provide CDAT tables > through -device option property to vary from run to run. The format will be > ASCII with structure/field-values pairs that are read by the device during > initialization as shown here: > > -device > cxl-type3,bus=rp0,memdev=cxl-mem1,id=cxl-pmem0,size=256M,cdat_file= > > For testing, a cxl_app.c user program is enhanced to test all supported DOE > protocols which are comprised of sequences of CFG RD/WR to various DOE cap > registers. The Linux kernel updates and CXL Type3 Device driver provide > sufficient ioctl() support to exercise the DOE protocol. See > > https://lore.kernel.org/linux-cxl/20210130002438.1872527-1-ben.widawsky@intel.com > > ======== cxl_app.c > #include > #include > #include > #include > #include > > #include "cxl_mem_wrapper.h" > > const char* help= "\ > -h help message\n\ > -query IOCTL CXL_MEM_QUERY_COMMANDS\n\ > -cfg_rd [0xoffset] IOCTL CXL_MEM_CONFIG_WR Read Hex\n\ > -cfg_wr [0xoffset] [0xaddr] IOCTL CXL_MEM_CONFIG_WR Write Hex\n\ > -doe_discovery [0xindex=0-3] IOCTL CXL_MEM_CONFIG_WR Write Hex\n\ > -doe_cxl [0xprotocol=0 or 2] [0xreq_code=0,1 for protocol=0]\n\ > -doe_cma [0xnum = 0] IOCTL CXL_MEM_CONFIG_WR Write Hex\n\ > example:\n\ > ./cxl_app.exe -cfg_rd 0x00\n\ > ./cxl_app.exe -cfg_wr 0x10 0x00ff0004\n\ > ./cxl_app.exe -doe_discovery 0\n\ > ./cxl_app.exe -doe_cxl 2\n\ > ./cxl_app.exe -doe_cxl 0 1\n\ > ./cxl_app.exe -doe_cma 0\n\ > "; > #define READ 0 #define WRITE 1 > > int FD; > typedef struct cxl_pdev_config cxl_pdev_config; > > int cxl_query() { > typedef struct cxl_mem_query_commands cxl_mem_query_commands; > typedef struct cxl_command_info cxl_command_info; > int n_cmds= 0; > // QUERY with n_commands == 0 to get command size > ioctl(FD, CXL_MEM_QUERY_COMMANDS, &n_cmds); > printf("Querying\n"); > > cxl_mem_query_commands* cmds= malloc(sizeof(cxl_mem_query_commands) > + n_cmds * sizeof(cxl_command_info)); > cmds->n_commands= n_cmds; > // QUERY with command size & pre-alloc memory > ioctl(FD, CXL_MEM_QUERY_COMMANDS, cmds); > > for (int i= 0; i < (int)cmds->n_commands; i++) { > printf(" id %d", cmds->commands[i].id); > printf(" flags %d", cmds->commands[i].flags); > printf(" size_in %d", cmds->commands[i].size_in); > printf(" size_out %d\n", cmds->commands[i].size_out); > } > > return 0; > }; > > int cxl_config(char* offset_s, char* data_s) { > int offset, data, is_write; > cxl_pdev_config* config_payload= malloc(sizeof(cxl_pdev_config)); > if (data_s == NULL) > is_write= 0; > else { > is_write= 1; > data= strtol(data_s, NULL, 16); > } > offset= strtol(offset_s, NULL, 16); > > config_payload->offset= offset; > config_payload->data= data; > config_payload->is_write= is_write; > ioctl(FD, CXL_MEM_CONFIG_WR, config_payload); > printf("CONFIG_WR %s [%0x] ", (is_write)? "write" : "read", > config_payload->offset); > for (int i= 0; i < 32; i += 8) printf(" %02x", (config_payload->data >> > i) & 0xff); > printf("\n"); > > return 0; > }; > > void doe_config(cxl_pdev_config* config_payload, uint32_t offset, uint32_t > data, uint32_t is_write) { > config_payload->offset= offset; > config_payload->data= data; > config_payload->is_write= is_write; > ioctl(FD, CXL_MEM_CONFIG_WR, config_payload); > printf("CONFIG_%s [%0x] ", (is_write)? "WR": "RD", > config_payload->offset); > for (int i= 0; i < 32; i += 8) printf(" %02x", (config_payload->data >> > i) & 0xff); > printf("\n"); > }; > > int cxl_doe_cxl(char* entry_s, char* data_s) { > cxl_pdev_config* config_payload= malloc(sizeof(cxl_pdev_config)); > uint32_t data_obj[3]; > int length; > uint32_t do_type; > uint32_t req_type; > > do_type = strtol(entry_s, NULL, 16); > req_type = strtol(data_s, NULL, 16); > printf("DOE TYPE=%0x\n", do_type); > printf("REQ TYPE=%0x\n", req_type); > if (do_type == 0) > data_obj[0] = 0x00001e98; > if (do_type == 2) > data_obj[0] = 0x00021e98; > data_obj[1] = 0x3; > data_obj[2] = req_type; > printf("data_obj[0]=%x\n", data_obj[0]); > printf("data_obj[1]=%x\n", data_obj[1]); > printf("data_obj[2]=%x\n", data_obj[2]); > > printf("DOE\n"); > > doe_config(config_payload, 0x170, data_obj[0], WRITE); > doe_config(config_payload, 0x170, data_obj[1], WRITE); > doe_config(config_payload, 0x170, data_obj[2], WRITE); > > /* Set GO */ > doe_config(config_payload, 0x168, 0x80000000, WRITE); > /* check status READY is set 16c */ > doe_config(config_payload, 0x16c, 0, READ); > > /* read cdat response */ > doe_config(config_payload, 0x174, 0, READ); > > /* write cdat response success */ > doe_config(config_payload, 0x174, 0x00000001, WRITE); > > /* read cdat response */ > doe_config(config_payload, 0x174, 0, READ); > > length = config_payload->data & 0x0000ffff; > printf("DOE RSP LENGTH = %0d\n", length); > > /* write cdat response success */ > doe_config(config_payload, 0x174, 0x00000001, WRITE); > > for (int j= 0; j < length-2; j=j+1) { > /* read cdat response */ > doe_config(config_payload, 0x174, 0, READ); > > /* write cdat response success */ > doe_config(config_payload, 0x174, 0x00000001, WRITE); > } > return 0; > > }; > > > int cxl_doe_cma(char* entry_s, char* data_s) { > cxl_pdev_config* config_payload= malloc(sizeof(cxl_pdev_config)); > uint32_t data_obj[3]; > > data_obj[0] = 0x00010001; > data_obj[1] = 0x3; > data_obj[2] = strtol(entry_s, NULL, 16); > > printf("DOE\n"); > doe_config(config_payload, 0x170, data_obj[0], WRITE); > doe_config(config_payload, 0x170, data_obj[1], WRITE); > doe_config(config_payload, 0x170, data_obj[2], WRITE); > > /* Set GO */ > doe_config(config_payload, 0x168, 0x80000000, WRITE); > > /* check status READY is set 16c */ > doe_config(config_payload, 0x16c, 0, READ); > > /* read Discovery response */ > doe_config(config_payload, 0x174, 0, READ); > > doe_config(config_payload, 0x168, 0x1, WRITE); > > /* check status READY is set 16c */ > doe_config(config_payload, 0x16c, 0, READ); > }; > > int cxl_doe_discovery(char* entry_s, char* data_s) { > cxl_pdev_config* config_payload= malloc(sizeof(cxl_pdev_config)); > uint32_t data_obj[3]; > > data_obj[0] = 0x00000001; > data_obj[1] = 0x3; > data_obj[2] = strtol(entry_s, NULL, 16); > > printf("DOE\n"); > doe_config(config_payload, 0x170, data_obj[0], WRITE); > doe_config(config_payload, 0x170, data_obj[1], WRITE); > doe_config(config_payload, 0x170, data_obj[2], WRITE); > > /* Set GO */ > doe_config(config_payload, 0x168, 0x80000000, WRITE); > > /* check status READY is set 16c */ > doe_config(config_payload, 0x16c, 0, READ); > > /* read Discovery response */ > doe_config(config_payload, 0x174, 0, READ); > > /* write Discovery response success */ > doe_config(config_payload, 0x174, 0x00000001, WRITE); > > doe_config(config_payload, 0x174, 0, READ); > > /* write Discovery response success */ > doe_config(config_payload, 0x174, 0x00000002, WRITE); > > doe_config(config_payload, 0x174, 0, READ); > > /* write Discovery response success */ > doe_config(config_payload, 0x174, 0x00000003, WRITE); > > return 0; > }; > > int parse_input(int argc, char** argv) { > int idx; > if (argc < 2) return -1; > for (idx= 0; idx < argc; idx++) { > if (strcmp(argv[idx], "-h") == 0) > return -1; > if (strcmp(argv[idx], "-query") == 0) > return cxl_query(); > if (strcmp(argv[idx], "-cfg_rd") == 0) > return cxl_config(argv[idx + 1], NULL); > if (strcmp(argv[idx], "-cfg_wr") == 0) > return cxl_config(argv[idx + 1], argv[idx + 2]); > if (strcmp(argv[idx], "-doe_discovery") == 0) > return cxl_doe_discovery(argv[idx + 1], NULL); > if (strcmp(argv[idx], "-doe_cxl") == 0) > return cxl_doe_cxl(argv[idx + 1], argv[idx + 2]); > if (strcmp(argv[idx], "-doe_cma") == 0) > return cxl_doe_cma(argv[idx + 1], NULL); > } > return 0; > }; > > int main(int argc, char** argv) { > int ret; > char* dev_path= "/dev/cxl/mem0"; > > if ((FD= open(dev_path, O_RDWR)) < 0) { > printf("Open error loc: %s\n", dev_path); > printf("Try sudo %s\n", argv[0]); > exit(0); > } > > if ((ret= parse_input(argc, argv)) < 0) { > printf("Please specify input "); > for (int i= 0; i < argc; i++) printf(" %s", argv[i]);; > printf("\n%s\n", help); > } > printf("DOE\n"); > > close(FD); > exit(0); > } > > ======== > > The QEMU script to run is > > sudo $QEMU_HOME/qemu/build/x86_64-softmmu/qemu-system-x86_64 \ > -nic user,hostfwd=tcp::2222-:22,hostfwd=tcp::1234-:1234 \ > -machine q35,accel=kvm,nvdimm,cxl \ > -m 8192M,slots=4,maxmem=40964M \ > -smp 8,sockets=2,cores=2,threads=2 \ > -enable-kvm \ > -boot order=d \ > -k 'en-us' \ > -vga virtio \ > -drive > if=pflash,format=raw,readonly,file=$TIANOCORE_HOME/edk2/Build/OvmfX64/DEBUG_GCC5/FV/OVMF_CODE.fd > \ > -drive > if=pflash,format=raw,file=$TIANOCORE_HOME/edk2/Build/OvmfX64/DEBUG_GCC5/FV/OVMF_VARS.fd > \ > -drive file=$QCOW_HOME/ubuntu_20.qcow,format=qcow2 \ > -object memory-backend-file,id=cxl-mem1,share,mem-path=cxl-type3,size=512M \ > -object memory-backend-file,id=cxl-mem2,share,mem-path=cxl-mem2,size=512M" \ > -device > pxb-cxl,id=cxl.0,bus=pcie.0,bus_nr=52,uid=0,len-window-base=1,window-base[0]=0x4c0000000,memdev[0]=cxl-mem1 > \ > -device cxl-rp,id=rp0,bus=cxl.0,addr=0.0,chassis=0,slot=0 \ > -device cxl-type3,bus=rp0,memdev=cxl-mem1,id=cxl-pmem0,size=256M,lsa=cxl-mem2 > \ > -object memory-backend-ram,id=mem0,size=2048M \ > -numa node,nodeid=0,memdev=mem0, \ > -numa cpu,node-id=0,socket-id=0 \ > -object memory-backend-ram,id=mem1,size=2048M \ > -numa node,nodeid=1,memdev=mem1, \ > -numa cpu,node-id=1,socket-id=1 \ > -object memory-backend-ram,id=mem2,size=2048M \ > -numa node,nodeid=2,memdev=mem2, \ > -object memory-backend-ram,id=mem3,size=2048M \ > -numa node,nodeid=3,memdev=mem3, \ > -numa node,nodeid=4, \ > -object > memory-backend-file,id=nvmem0,share,mem-path=nvdimm-0,size=16384M,align=1G \ > -device nvdimm,memdev=nvmem0,id=nv0,label-size=2M,node=4 \ > -numa node,nodeid=5, \ > -object > memory-backend-file,id=nvmem1,share,mem-path=nvdimm-1,size=16384M,align=1G \ > -device nvdimm,memdev=nvmem1,id=nv1,label-size=2M,node=5 \ > -numa dist,src=0,dst=0,val=10 \ > -numa dist,src=0,dst=1,val=21 \ > -numa dist,src=0,dst=2,val=12 \ > -numa dist,src=0,dst=3,val=21 \ > -numa dist,src=0,dst=4,val=17 \ > -numa dist,src=0,dst=5,val=28 \ > -numa dist,src=1,dst=1,val=10 \ > -numa dist,src=1,dst=2,val=21 \ > -numa dist,src=1,dst=3,val=12 \ > -numa dist,src=1,dst=4,val=28 \ > -numa dist,src=1,dst=5,val=17 \ > -numa dist,src=2,dst=2,val=10 \ > -numa dist,src=2,dst=3,val=21 \ > -numa dist,src=2,dst=4,val=28 \ > -numa dist,src=2,dst=5,val=28 \ -numa dist,src=3,dst=3,val=10 \ > -numa dist,src=3,dst=4,val=28 \ > -numa dist,src=3,dst=5,val=28 \ > -numa dist,src=4,dst=4,val=10 \ > -numa dist,src=4,dst=5,val=28 \ > -numa dist,src=5,dst=5,val=10 > >