From mboxrd@z Thu Jan 1 00:00:00 1970 From: "Steven J. Hill" Subject: [PATCH v12 06/10] netdev: cavium: octeon: Add Octeon III PKO Support Date: Wed, 27 Jun 2018 16:25:15 -0500 Message-ID: <1530134719-19407-7-git-send-email-steven.hill@cavium.com> References: <1530134719-19407-1-git-send-email-steven.hill@cavium.com> Mime-Version: 1.0 Content-Type: text/plain Cc: Carlos Munoz , Chandrakala Chavva , "Steven J. Hill" To: netdev@vger.kernel.org Return-path: Received: from mail-sn1nam02on0043.outbound.protection.outlook.com ([104.47.36.43]:35527 "EHLO NAM02-SN1-obe.outbound.protection.outlook.com" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S965912AbeF0VZj (ORCPT ); Wed, 27 Jun 2018 17:25:39 -0400 In-Reply-To: <1530134719-19407-1-git-send-email-steven.hill@cavium.com> Sender: netdev-owner@vger.kernel.org List-ID: From: Carlos Munoz Add support for Octeon III PKO logic block for BGX Ethernet. Signed-off-by: Carlos Munoz Signed-off-by: Steven J. Hill --- drivers/net/ethernet/cavium/octeon/octeon3-pko.c | 1638 ++++++++++++++++++++++ drivers/net/ethernet/cavium/octeon/octeon3-pko.h | 159 +++ 2 files changed, 1797 insertions(+) create mode 100644 drivers/net/ethernet/cavium/octeon/octeon3-pko.c create mode 100644 drivers/net/ethernet/cavium/octeon/octeon3-pko.h diff --git a/drivers/net/ethernet/cavium/octeon/octeon3-pko.c b/drivers/net/ethernet/cavium/octeon/octeon3-pko.c new file mode 100644 index 0000000..238bf51 --- /dev/null +++ b/drivers/net/ethernet/cavium/octeon/octeon3-pko.c @@ -0,0 +1,1638 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Octeon III Packet-Output Processing Unit (PKO) + * + * Copyright (C) 2018 Cavium, Inc. + */ + +#include "octeon3.h" + +#define PKO_MAX_FIFO_GROUPS 8 +#define PKO_MAX_OUTPUT_MACS 28 +#define PKO_FIFO_SIZE 2560 + +#define PKO_BASE 0x1540000000000ull +#define PKO_ADDR(n) (PKO_BASE + SET_XKPHYS + NODE_OFFSET(n)) +#define PKO_Q_ADDR(n, q) (PKO_ADDR(n) + ((q) << 9)) + +#define PKO_CHANNEL_LEVEL(n) (PKO_ADDR(n) + 0x0800f0) +#define PKO_DPFI_ENA(n) (PKO_ADDR(n) + 0xc00018) +#define PKO_DPFI_FLUSH(n) (PKO_ADDR(n) + 0xc00008) +#define PKO_DPFI_FPA_AURA(n) (PKO_ADDR(n) + 0xc00010) +#define PKO_DPFI_STATUS(n) (PKO_ADDR(n) + 0xc00000) +#define PKO_DQ_SCHEDULE(n, q) (PKO_Q_ADDR(n, q) + 0x280008) +#define PKO_DQ_SW_XOFF(n, q) (PKO_Q_ADDR(n, q) + 0x2800e0) +#define PKO_DQ_TOPOLOGY(n, q) (PKO_Q_ADDR(n, q) + 0x300000) +#define PKO_DQ_WM_CTL(n, q) (PKO_Q_ADDR(n, q) + 0x000040) +#define PKO_ENABLE(n) (PKO_ADDR(n) + 0xd00008) +#define PKO_L1_SQ_LINK(n, q) (PKO_Q_ADDR(n, q) + 0x000038) +#define PKO_L1_SQ_SHAPE(n, q) (PKO_Q_ADDR(n, q) + 0x000010) +#define PKO_L1_SQ_TOPOLOGY(n, q) (PKO_Q_ADDR(n, q) + 0x080000) +#define PKO_L2_SQ_SCHEDULE(n, q) (PKO_Q_ADDR(n, q) + 0x080008) +#define PKO_L2_SQ_TOPOLOGY(n, q) (PKO_Q_ADDR(n, q) + 0x100000) +#define PKO_L3_L2_SQ_CHANNEL(n, q) (PKO_Q_ADDR(n, q) + 0x080038) +#define PKO_L3_SQ_SCHEDULE(n, q) (PKO_Q_ADDR(n, q) + 0x100008) +#define PKO_L3_SQ_TOPOLOGY(n, q) (PKO_Q_ADDR(n, q) + 0x180000) +#define PKO_L4_SQ_SCHEDULE(n, q) (PKO_Q_ADDR(n, q) + 0x180008) +#define PKO_L4_SQ_TOPOLOGY(n, q) (PKO_Q_ADDR(n, q) + 0x200000) +#define PKO_L5_SQ_SCHEDULE(n, q) (PKO_Q_ADDR(n, q) + 0x200008) +#define PKO_L5_SQ_TOPOLOGY(n, q) (PKO_Q_ADDR(n, q) + 0x280000) +#define PKO_LUT(n, c) (PKO_ADDR(n) + ((c) << 3) + 0xb00000) +#define PKO_MAC_CFG(n, m) (PKO_ADDR(n) + ((m) << 3) + 0x900000) +#define PKO_MCI0_MAX_CRED(n, m) (PKO_ADDR(n) + ((m) << 3) + 0xa00000) +#define PKO_MCI1_MAX_CRED(n, m) (PKO_ADDR(n) + ((m) << 3) + 0xa80000) +#define PKO_PDM_CFG(n) (PKO_ADDR(n) + 0x800000) +#define PKO_PDM_DQ_MINPAD(n, q) (PKO_ADDR(n) + ((q) << 3) + 0x8f0000) +#define PKO_PTF_IOBP_CFG(n) (PKO_ADDR(n) + 0x900300) +#define PKO_PTF_STATUS(n, f) (PKO_ADDR(n) + ((f) << 3) + 0x900100) +#define PKO_PTGF_CFG(n, g) (PKO_ADDR(n) + ((g) << 3) + 0x900200) +#define PKO_SHAPER_CFG(n) (PKO_ADDR(n) + 0x0800f8) +#define PKO_STATUS(n) (PKO_ADDR(n) + 0xd00000) + +/* These levels mimic the PKO internal linked queue structure */ +enum queue_level { + PQ = 1, + L2_SQ = 2, + L3_SQ = 3, + L4_SQ = 4, + L5_SQ = 5, + DQ = 6 +}; + +enum pko_dqop_e { + DQOP_SEND, + DQOP_OPEN, + DQOP_CLOSE, + DQOP_QUERY +}; + +enum pko_dqstatus_e { + PASS = 0, + BADSTATE = 0x8, + NOFPABUF = 0x9, + NOPKOBUF = 0xa, + FAILRTNPTR = 0xb, + ALREADY = 0xc, + NOTCREATED = 0xd, + NOTEMPTY = 0xe, + SENDPKTDROP = 0xf +}; + +struct mac_info { + int fifo_cnt; + int prio; + int speed; + int fifo; + int num_lmacs; +}; + +struct fifo_grp_info { + int speed; + int size; +}; + +static const int lut_index_78xx[] = { + 0x200, + 0x240, + 0x280, + 0x2c0, + 0x300, + 0x340 +}; + +static const int lut_index_73xx[] = { + 0x000, + 0x040, + 0x080 +}; + +static enum queue_level max_sq_level(void) +{ + /* 73xx and 75xx only have 3 scheduler queue levels */ + if (OCTEON_IS_MODEL(OCTEON_CN73XX) || OCTEON_IS_MODEL(OCTEON_CNF75XX)) + return L3_SQ; + return L5_SQ; +} + +static int get_num_fifos(void) +{ + if (OCTEON_IS_MODEL(OCTEON_CN73XX) || OCTEON_IS_MODEL(OCTEON_CNF75XX)) + return 16; + return 28; +} + +static int get_num_fifo_groups(void) +{ + if (OCTEON_IS_MODEL(OCTEON_CN73XX) || OCTEON_IS_MODEL(OCTEON_CNF75XX)) + return 5; + return 8; +} + +static int get_num_output_macs(void) +{ + if (OCTEON_IS_MODEL(OCTEON_CN78XX)) + return 28; + else if (OCTEON_IS_MODEL(OCTEON_CNF75XX)) + return 10; + else if (OCTEON_IS_MODEL(OCTEON_CN73XX)) + return 14; + return 0; +} + +static int get_output_mac(int interface, int index, + enum octeon3_mac_type mac_type) +{ + int mac; + + /* Output macs are hardcoded in the hardware. See PKO Output MACs + * section in the HRM. + */ + if (OCTEON_IS_MODEL(OCTEON_CN73XX) || OCTEON_IS_MODEL(OCTEON_CNF75XX)) { + if (mac_type == SRIO_MAC) + mac = 4 + 2 * interface + index; + else + mac = 2 + 4 * interface + index; + } else { + mac = 4 + 4 * interface + index; + } + return mac; +} + +static int get_num_port_queues(void) +{ + if (OCTEON_IS_MODEL(OCTEON_CN73XX) || OCTEON_IS_MODEL(OCTEON_CNF75XX)) + return 16; + return 32; +} + +static int allocate_queues(int node, enum queue_level level, int num_queues, + int *queues) +{ + struct global_resource_tag tag; + int rc, max_queues = 0; + char buf[16]; + + if (level == PQ) { + strncpy((char *)&tag.lo, "cvm_pkop", 8); + snprintf(buf, 16, "oq_%d....", node); + memcpy(&tag.hi, buf, 8); + + if (OCTEON_IS_MODEL(OCTEON_CN78XX)) + max_queues = 32; + else + max_queues = 16; + } else if (level == L2_SQ) { + strncpy((char *)&tag.lo, "cvm_pkol", 8); + snprintf(buf, 16, "2q_%d....", node); + memcpy(&tag.hi, buf, 8); + + if (OCTEON_IS_MODEL(OCTEON_CN78XX)) + max_queues = 512; + else + max_queues = 256; + } else if (level == L3_SQ) { + strncpy((char *)&tag.lo, "cvm_pkol", 8); + snprintf(buf, 16, "3q_%d....", node); + memcpy(&tag.hi, buf, 8); + + if (OCTEON_IS_MODEL(OCTEON_CN78XX)) + max_queues = 512; + else + max_queues = 256; + } else if (level == L4_SQ) { + strncpy((char *)&tag.lo, "cvm_pkol", 8); + snprintf(buf, 16, "4q_%d....", node); + memcpy(&tag.hi, buf, 8); + + if (OCTEON_IS_MODEL(OCTEON_CN78XX)) + max_queues = 1024; + else + max_queues = 0; + } else if (level == L5_SQ) { + strncpy((char *)&tag.lo, "cvm_pkol", 8); + snprintf(buf, 16, "5q_%d....", node); + memcpy(&tag.hi, buf, 8); + + if (OCTEON_IS_MODEL(OCTEON_CN78XX)) + max_queues = 1024; + else + max_queues = 0; + } else if (level == DQ) { + strncpy((char *)&tag.lo, "cvm_pkod", 8); + snprintf(buf, 16, "eq_%d....", node); + memcpy(&tag.hi, buf, 8); + + if (OCTEON_IS_MODEL(OCTEON_CN78XX)) + max_queues = 1024; + else + max_queues = 256; + } + + res_mgr_create_resource(tag, max_queues); + rc = res_mgr_alloc_range(tag, -1, num_queues, false, queues); + if (rc < 0) + return rc; + + return 0; +} + +static void free_queues(int node, enum queue_level level, int num_queues, + const int *queues) +{ + struct global_resource_tag tag; + char buf[16]; + + if (level == PQ) { + strncpy((char *)&tag.lo, "cvm_pkop", 8); + snprintf(buf, 16, "oq_%d....", node); + memcpy(&tag.hi, buf, 8); + } else if (level == L2_SQ) { + strncpy((char *)&tag.lo, "cvm_pkol", 8); + snprintf(buf, 16, "2q_%d....", node); + memcpy(&tag.hi, buf, 8); + } else if (level == L3_SQ) { + strncpy((char *)&tag.lo, "cvm_pkol", 8); + snprintf(buf, 16, "3q_%d....", node); + memcpy(&tag.hi, buf, 8); + } else if (level == L4_SQ) { + strncpy((char *)&tag.lo, "cvm_pkol", 8); + snprintf(buf, 16, "4q_%d....", node); + memcpy(&tag.hi, buf, 8); + } else if (level == L5_SQ) { + strncpy((char *)&tag.lo, "cvm_pkol", 8); + snprintf(buf, 16, "5q_%d....", node); + memcpy(&tag.hi, buf, 8); + } else if (level == DQ) { + strncpy((char *)&tag.lo, "cvm_pkod", 8); + snprintf(buf, 16, "eq_%d....", node); + memcpy(&tag.hi, buf, 8); + } + + res_mgr_free_range(tag, queues, num_queues); +} + +static int port_queue_init(int node, int pq, int mac) +{ + u64 data; + + data = mac << PKO_SQ_TOPOLOGY_LINK_SHIFT; + oct_csr_write(data, PKO_L1_SQ_TOPOLOGY(node, pq)); + + data = mac << PKO_L1_SQ_SHAPE_LINK_SHIFT; + oct_csr_write(data, PKO_L1_SQ_SHAPE(node, pq)); + + data = mac; + data <<= PKO_L1_SQ_LINK_LINK_SHIFT; + oct_csr_write(data, PKO_L1_SQ_LINK(node, pq)); + + return 0; +} + +static int scheduler_queue_l2_init(int node, int queue, int parent_q) +{ + u64 data; + + data = oct_csr_read(PKO_L1_SQ_TOPOLOGY(node, parent_q)); + data &= ~(PKO_L12_SQ_TOPOLOGY_PRIO_ANCHOR_MASK | + PKO_SQ_TOPOLOGY_RR_PRIO_MASK); + data |= (u64)queue << PKO_SQ_TOPOLOGY_PRIO_ANCHOR_SHIFT; + data |= PKO_SQ_TOPOLOGY_RR_PRIO_SHAPER << PKO_SQ_TOPOLOGY_RR_PRIO_SHIFT; + oct_csr_write(data, PKO_L1_SQ_TOPOLOGY(node, parent_q)); + + oct_csr_write(0, PKO_L2_SQ_SCHEDULE(node, queue)); + + data = parent_q << PKO_SQ_TOPOLOGY_LINK_SHIFT; + oct_csr_write(data, PKO_L2_SQ_TOPOLOGY(node, queue)); + + return 0; +} + +static int scheduler_queue_l3_init(int node, int queue, int parent_q) +{ + u64 data; + + data = oct_csr_read(PKO_L2_SQ_TOPOLOGY(node, parent_q)); + data &= ~(PKO_L345_SQ_TOPOLOGY_PRIO_ANCHOR_MASK | + PKO_SQ_TOPOLOGY_RR_PRIO_MASK); + data |= (u64)queue << PKO_SQ_TOPOLOGY_PRIO_ANCHOR_SHIFT; + data |= PKO_SQ_TOPOLOGY_RR_PRIO_SHAPER << PKO_SQ_TOPOLOGY_RR_PRIO_SHIFT; + oct_csr_write(data, PKO_L2_SQ_TOPOLOGY(node, parent_q)); + + oct_csr_write(0, PKO_L3_SQ_SCHEDULE(node, queue)); + + data = parent_q << PKO_SQ_TOPOLOGY_LINK_SHIFT; + oct_csr_write(data, PKO_L3_SQ_TOPOLOGY(node, queue)); + + return 0; +} + +static int scheduler_queue_l4_init(int node, int queue, int parent_q) +{ + u64 data; + + data = oct_csr_read(PKO_L3_SQ_TOPOLOGY(node, parent_q)); + data &= ~(PKO_L345_SQ_TOPOLOGY_PRIO_ANCHOR_MASK | + PKO_SQ_TOPOLOGY_RR_PRIO_MASK); + data |= (u64)queue << PKO_SQ_TOPOLOGY_PRIO_ANCHOR_SHIFT; + data |= PKO_SQ_TOPOLOGY_RR_PRIO_SHAPER << PKO_SQ_TOPOLOGY_RR_PRIO_SHIFT; + oct_csr_write(data, PKO_L3_SQ_TOPOLOGY(node, parent_q)); + + oct_csr_write(0, PKO_L4_SQ_SCHEDULE(node, queue)); + + data = parent_q << PKO_SQ_TOPOLOGY_LINK_SHIFT; + oct_csr_write(data, PKO_L4_SQ_TOPOLOGY(node, queue)); + + return 0; +} + +static int scheduler_queue_l5_init(int node, int queue, int parent_q) +{ + u64 data; + + data = oct_csr_read(PKO_L4_SQ_TOPOLOGY(node, parent_q)); + data &= ~(PKO_L345_SQ_TOPOLOGY_PRIO_ANCHOR_MASK | + PKO_SQ_TOPOLOGY_RR_PRIO_MASK); + data |= (u64)queue << PKO_SQ_TOPOLOGY_PRIO_ANCHOR_SHIFT; + data |= PKO_SQ_TOPOLOGY_RR_PRIO_SHAPER << PKO_SQ_TOPOLOGY_RR_PRIO_SHIFT; + oct_csr_write(data, PKO_L4_SQ_TOPOLOGY(node, parent_q)); + + oct_csr_write(0, PKO_L5_SQ_SCHEDULE(node, queue)); + + data = parent_q << PKO_SQ_TOPOLOGY_LINK_SHIFT; + oct_csr_write(data, PKO_L5_SQ_TOPOLOGY(node, queue)); + + return 0; +} + +static int descriptor_queue_init(int node, const int *queue, int parent_q, + int num_dq) +{ + int i, prio, rr_prio, rr_quantum; + u64 addr, data; + + /* Limit static priorities to the available prio field bits */ + if (num_dq > 9) { + pr_err("%s: Invalid number of dqs\n", __FILE__); + return -1; + } + + prio = 0; + + if (num_dq == 1) { + /* Single dq */ + rr_prio = PKO_SQ_TOPOLOGY_RR_PRIO_SHAPER; + rr_quantum = 0x10; + } else { + /* Multiple dqs */ + rr_prio = num_dq; + rr_quantum = 0; + } + + if (OCTEON_IS_MODEL(OCTEON_CN78XX)) + addr = PKO_L5_SQ_TOPOLOGY(node, parent_q); + else + addr = PKO_L3_SQ_TOPOLOGY(node, parent_q); + + data = oct_csr_read(addr); + data &= ~(PKO_L345_SQ_TOPOLOGY_PRIO_ANCHOR_MASK | + PKO_SQ_TOPOLOGY_RR_PRIO_MASK); + data |= (u64)queue[0] << PKO_SQ_TOPOLOGY_PRIO_ANCHOR_SHIFT; + data |= rr_prio << PKO_SQ_TOPOLOGY_RR_PRIO_SHIFT; + oct_csr_write(data, addr); + + for (i = 0; i < num_dq; i++) { + data = (prio << PKO_DQ_SCHEDULE_PRIO_SHIFT) | rr_quantum; + oct_csr_write(data, PKO_DQ_SCHEDULE(node, queue[i])); + + data = parent_q << PKO_SQ_TOPOLOGY_LINK_SHIFT; + oct_csr_write(data, PKO_DQ_TOPOLOGY(node, queue[i])); + + data = PKO_DQ_WM_CTL_KIND; + oct_csr_write(data, PKO_DQ_WM_CTL(node, queue[i])); + + if (prio << rr_prio) + prio++; + } + + return 0; +} + +static int map_channel(int node, int pq, int queue, int ipd_port) +{ + int table_index, lut_index = 0; + u64 data; + + data = oct_csr_read(PKO_L3_L2_SQ_CHANNEL(node, queue)); + data &= ~PKO_L3_L2_CHANNEL_CC_CHANNEL_MASK; + data |= (u64)ipd_port << PKO_L3_L2_CHANNEL_CC_CHANNEL_SHIFT; + oct_csr_write(data, PKO_L3_L2_SQ_CHANNEL(node, queue)); + + /* See PKO_LUT register description in the HRM for how to compose the + * lut_index. + */ + if (OCTEON_IS_MODEL(OCTEON_CN78XX)) { + table_index = ((ipd_port & PKO_LUT_PORT_MASK) - + PKO_LUT_PORT_TO_INDEX) >> PKO_LUT_QUEUE_NUMBER_SHIFT; + lut_index = lut_index_78xx[table_index]; + lut_index += ipd_port & PKO_LUT_QUEUE_NUMBER_MASK; + } else if (OCTEON_IS_MODEL(OCTEON_CN73XX)) { + table_index = ((ipd_port & PKO_LUT_PORT_MASK) - + PKO_LUT_PORT_TO_INDEX) >> PKO_LUT_QUEUE_NUMBER_SHIFT; + lut_index = lut_index_73xx[table_index]; + lut_index += ipd_port & PKO_LUT_QUEUE_NUMBER_MASK; + } else if (OCTEON_IS_MODEL(OCTEON_CNF75XX)) { + if ((ipd_port & PKO_LUT_PORT_MASK) != PKO_LUT_PORT_TO_INDEX) + return -1; + lut_index = ipd_port & PKO_LUT_QUEUE_NUMBER_MASK; + } + + data = PKO_LUT_VALID; + data |= pq << PKO_LUT_PQ_IDX_SHIFT; + data |= queue; + oct_csr_write(data, PKO_LUT(node, lut_index)); + + return 0; +} + +static int open_dq(int node, int dq) +{ + u64 data, *iobdma_addr, *scratch_addr; + enum pko_dqstatus_e status; + + /* Build the dq open query. See PKO_QUERY_DMA_S in the HRM for the + * query format. + */ + data = (LMTDMA_SCR_OFFSET >> PKO_LMTDMA_SCRADDR_SHIFT) + << PKO_QUERY_DMA_SCRADDR_SHIFT; + data |= 1ull << PKO_QUERY_DMA_RTNLEN_SHIFT; /* Must always be 1 */ + data |= 0x51ull << PKO_QUERY_DMA_DID_SHIFT; /* Must always be 51h */ + data |= (u64)node << PKO_QUERY_DMA_NODE_SHIFT; + data |= (u64)DQOP_OPEN << PKO_QUERY_DMA_DQOP_SHIFT; + data |= dq << PKO_QUERY_DMA_DQ_SHIFT; + + CVMX_SYNCWS; + preempt_disable(); + + /* Clear return location */ + scratch_addr = (u64 *)(SCRATCH_BASE_ADDR + LMTDMA_SCR_OFFSET); + *scratch_addr = ~0ull; + + /* Issue pko lmtdma command */ + iobdma_addr = (u64 *)(IOBDMA_ORDERED_IO_ADDR); + *iobdma_addr = data; + + /* Wait for lmtdma command to complete and get response*/ + CVMX_SYNCIOBDMA; + data = *scratch_addr; + + preempt_enable(); + + /* See PKO_QUERY_RTN_S in the HRM for response format */ + status = (data & PKO_QUERY_RTN_DQSTATUS_MASK) + >> PKO_QUERY_RTN_DQSTATUS_SHIFT; + if (status != PASS && status != ALREADY) { + pr_err("%s: Failed to open dq\n", __FILE__); + return -1; + } + + return 0; +} + +static s64 query_dq(int node, int dq) +{ + u64 data, *iobdma_addr, *scratch_addr; + enum pko_dqstatus_e status; + s64 depth; + + /* Build the dq open query. See PKO_QUERY_DMA_S in the HRM for the + * query format. + */ + data = (LMTDMA_SCR_OFFSET >> PKO_LMTDMA_SCRADDR_SHIFT) << + PKO_QUERY_DMA_SCRADDR_SHIFT; + data |= 1ull << PKO_QUERY_DMA_RTNLEN_SHIFT; /* Must always be 1 */ + data |= 0x51ull << PKO_QUERY_DMA_DID_SHIFT; /* Must always be 51h */ + data |= (u64)node << PKO_QUERY_DMA_NODE_SHIFT; + data |= (u64)DQOP_QUERY << PKO_QUERY_DMA_DQOP_SHIFT; + data |= dq << PKO_QUERY_DMA_DQ_SHIFT; + + CVMX_SYNCWS; + preempt_disable(); + + /* Clear return location */ + scratch_addr = (u64 *)(SCRATCH_BASE_ADDR + LMTDMA_SCR_OFFSET); + *scratch_addr = ~0ull; + + /* Issue pko lmtdma command */ + iobdma_addr = (u64 *)(IOBDMA_ORDERED_IO_ADDR); + *iobdma_addr = data; + + /* Wait for lmtdma command to complete and get response*/ + CVMX_SYNCIOBDMA; + data = *scratch_addr; + + preempt_enable(); + + /* See PKO_QUERY_RTN_S in the HRM for response format */ + status = (data & PKO_QUERY_RTN_DQSTATUS_MASK) + >> PKO_QUERY_RTN_DQSTATUS_SHIFT; + if (status != PASS) { + pr_err("%s: Failed to query dq=%d\n", __FILE__, dq); + return -1; + } + + depth = data & PKO_QUERY_RTN_DEPTH_MASK; + + return depth; +} + +static u64 close_dq(int node, int dq) +{ + u64 data, *iobdma_addr, *scratch_addr; + enum pko_dqstatus_e status; + + /* Build the dq open query. See PKO_QUERY_DMA_S in the HRM for the + * query format. + */ + data = (LMTDMA_SCR_OFFSET >> PKO_LMTDMA_SCRADDR_SHIFT) << + PKO_QUERY_DMA_SCRADDR_SHIFT; + data |= 1ull << PKO_QUERY_DMA_RTNLEN_SHIFT; /* Must always be 1 */ + data |= 0x51ull << PKO_QUERY_DMA_DID_SHIFT; /* Must always be 51h */ + data |= (u64)node << PKO_QUERY_DMA_NODE_SHIFT; + data |= (u64)DQOP_CLOSE << PKO_QUERY_DMA_DQOP_SHIFT; + data |= dq << PKO_QUERY_DMA_DQ_SHIFT; + + CVMX_SYNCWS; + preempt_disable(); + + /* Clear return location */ + scratch_addr = (u64 *)(SCRATCH_BASE_ADDR + LMTDMA_SCR_OFFSET); + *scratch_addr = ~0ull; + + /* Issue pko lmtdma command */ + iobdma_addr = (u64 *)(IOBDMA_ORDERED_IO_ADDR); + *iobdma_addr = data; + + /* Wait for lmtdma command to complete and get response*/ + CVMX_SYNCIOBDMA; + data = *scratch_addr; + + preempt_enable(); + + /* See PKO_QUERY_RTN_S in the HRM for response format */ + status = (data & PKO_QUERY_RTN_DQSTATUS_MASK) + >> PKO_QUERY_RTN_DQSTATUS_SHIFT; + if (status != PASS) { + pr_err("%s: Failed to close dq\n", __FILE__); + return -1; + } + + return 0; +} + +static int get_78xx_fifos_required(int node, struct mac_info *macs) +{ + int bgx, cnt, i, index, num_lmacs, prio, qlm, fifo_cnt = 0; + enum port_mode mode; + u64 data; + + /* The loopback mac gets 1 fifo by default */ + macs[0].fifo_cnt = 1; + macs[0].speed = 1; + fifo_cnt += 1; + + /* The dpi mac gets 1 fifo by default */ + macs[1].fifo_cnt = 1; + macs[1].speed = 50; + fifo_cnt += 1; + + /* The ilk macs get default number of fifos (module param) */ + macs[2].fifo_cnt = ilk0_lanes <= 4 ? ilk0_lanes : 4; + macs[2].speed = 40; + fifo_cnt += macs[2].fifo_cnt; + macs[3].fifo_cnt = ilk1_lanes <= 4 ? ilk1_lanes : 4; + macs[3].speed = 40; + fifo_cnt += macs[3].fifo_cnt; + + /* Assign fifos to the active bgx macs */ + for (i = 4; i < get_num_output_macs(); i += 4) { + bgx = (i - 4) / 4; + qlm = bgx_port_get_qlm(node, bgx, 0); + + data = oct_csr_read(GSER_CFG(node, qlm)); + if (data & GSER_CFG_BGX) { + data = oct_csr_read(BGX_CMR_TX_LMACS(node, bgx)); + num_lmacs = data & 7; + + for (index = 0; index < num_lmacs; index++) { + switch (num_lmacs) { + case 1: + macs[i + index].num_lmacs = 4; + break; + case 2: + macs[i + index].num_lmacs = 2; + break; + case 4: + default: + macs[i + index].num_lmacs = 1; + break; + } + + mode = bgx_port_get_mode(node, bgx, 0); + switch (mode) { + case PORT_MODE_SGMII: + case PORT_MODE_RGMII: + macs[i + index].fifo_cnt = 1; + macs[i + index].prio = 1; + macs[i + index].speed = 1; + break; + + case PORT_MODE_XAUI: + case PORT_MODE_RXAUI: + macs[i + index].fifo_cnt = 4; + macs[i + index].prio = 2; + macs[i + index].speed = 20; + break; + + case PORT_MODE_10G_KR: + case PORT_MODE_XFI: + macs[i + index].fifo_cnt = 4; + macs[i + index].prio = 2; + macs[i + index].speed = 10; + break; + + case PORT_MODE_40G_KR4: + case PORT_MODE_XLAUI: + macs[i + index].fifo_cnt = 4; + macs[i + index].prio = 3; + macs[i + index].speed = 40; + break; + + default: + macs[i + index].fifo_cnt = 0; + macs[i + index].prio = 0; + macs[i + index].speed = 0; + macs[i + index].num_lmacs = 0; + break; + } + + fifo_cnt += macs[i + index].fifo_cnt; + } + } + } + + /* If more fifos than available were assigned, reduce the number of + * fifos until within limit. Start with the lowest priority macs with 4 + * fifos. + */ + prio = 1; + cnt = 4; + while (fifo_cnt > get_num_fifos()) { + for (i = 0; i < get_num_output_macs(); i++) { + if (macs[i].prio == prio && macs[i].fifo_cnt == cnt) { + macs[i].fifo_cnt >>= 1; + fifo_cnt -= macs[i].fifo_cnt; + } + + if (fifo_cnt <= get_num_fifos()) + break; + } + + if (prio >= 3) { + prio = 1; + cnt >>= 1; + } else { + prio++; + } + + if (cnt == 0) + break; + } + + /* Assign left over fifos to dpi */ + if (get_num_fifos() - fifo_cnt > 0) { + if (get_num_fifos() - fifo_cnt >= 3) { + macs[1].fifo_cnt += 3; + fifo_cnt -= 3; + } else { + macs[1].fifo_cnt += 1; + fifo_cnt -= 1; + } + } + + return 0; +} + +static int get_75xx_fifos_required(int node, struct mac_info *macs) +{ + int bgx, cnt, i, index, prio, qlm, fifo_cnt = 0; + enum port_mode mode; + u64 data; + + /* The loopback mac gets 1 fifo by default */ + macs[0].fifo_cnt = 1; + macs[0].speed = 1; + fifo_cnt += 1; + + /* The dpi mac gets 1 fifo by default */ + macs[1].fifo_cnt = 1; + macs[1].speed = 50; + fifo_cnt += 1; + + /* Assign fifos to the active bgx macs */ + bgx = 0; + for (i = 2; i < 6; i++) { + index = i - 2; + qlm = bgx_port_get_qlm(node, bgx, index); + data = oct_csr_read(GSER_CFG(node, qlm)); + if (data & GSER_CFG_BGX) { + macs[i].num_lmacs = 1; + + mode = bgx_port_get_mode(node, bgx, index); + switch (mode) { + case PORT_MODE_SGMII: + case PORT_MODE_RGMII: + macs[i].fifo_cnt = 1; + macs[i].prio = 1; + macs[i].speed = 1; + break; + + case PORT_MODE_10G_KR: + case PORT_MODE_XFI: + macs[i].fifo_cnt = 4; + macs[i].prio = 2; + macs[i].speed = 10; + break; + + default: + macs[i].fifo_cnt = 0; + macs[i].prio = 0; + macs[i].speed = 0; + macs[i].num_lmacs = 0; + break; + } + + fifo_cnt += macs[i].fifo_cnt; + } + } + + /* If more fifos than available were assigned, reduce the number of + * fifos until within limit. Start with the lowest priority macs with 4 + * fifos. + */ + prio = 1; + cnt = 4; + while (fifo_cnt > get_num_fifos()) { + for (i = 0; i < get_num_output_macs(); i++) { + if (macs[i].prio == prio && macs[i].fifo_cnt == cnt) { + macs[i].fifo_cnt >>= 1; + fifo_cnt -= macs[i].fifo_cnt; + } + + if (fifo_cnt <= get_num_fifos()) + break; + } + + if (prio >= 3) { + prio = 1; + cnt >>= 1; + } else { + prio++; + } + + if (cnt == 0) + break; + } + + /* Assign left over fifos to dpi */ + if (get_num_fifos() - fifo_cnt > 0) { + if (get_num_fifos() - fifo_cnt >= 3) { + macs[1].fifo_cnt += 3; + fifo_cnt -= 3; + } else { + macs[1].fifo_cnt += 1; + fifo_cnt -= 1; + } + } + + return 0; +} + +static int get_73xx_fifos_required(int node, struct mac_info *macs) +{ + int bgx, cnt, i, index, num_lmacs, prio, qlm, fifo_cnt = 0; + enum port_mode mode; + u64 data; + + /* The loopback mac gets 1 fifo by default */ + macs[0].fifo_cnt = 1; + macs[0].speed = 1; + fifo_cnt += 1; + + /* The dpi mac gets 1 fifo by default */ + macs[1].fifo_cnt = 1; + macs[1].speed = 50; + fifo_cnt += 1; + + /* Assign fifos to the active bgx macs */ + for (i = 2; i < get_num_output_macs(); i += 4) { + bgx = (i - 2) / 4; + qlm = bgx_port_get_qlm(node, bgx, 0); + data = oct_csr_read(GSER_CFG(node, qlm)); + + /* Bgx2 can be connected to dlm 5, 6, or both */ + if (bgx == 2) { + if (!(data & GSER_CFG_BGX)) { + qlm = bgx_port_get_qlm(node, bgx, 2); + data = oct_csr_read(GSER_CFG(node, qlm)); + } + } + + if (data & GSER_CFG_BGX) { + data = oct_csr_read(BGX_CMR_TX_LMACS(node, bgx)); + num_lmacs = data & 7; + + for (index = 0; index < num_lmacs; index++) { + switch (num_lmacs) { + case 1: + macs[i + index].num_lmacs = 4; + break; + case 2: + macs[i + index].num_lmacs = 2; + break; + case 4: + default: + macs[i + index].num_lmacs = 1; + break; + } + + mode = bgx_port_get_mode(node, bgx, index); + switch (mode) { + case PORT_MODE_SGMII: + case PORT_MODE_RGMII: + macs[i + index].fifo_cnt = 1; + macs[i + index].prio = 1; + macs[i + index].speed = 1; + break; + + case PORT_MODE_XAUI: + case PORT_MODE_RXAUI: + macs[i + index].fifo_cnt = 4; + macs[i + index].prio = 2; + macs[i + index].speed = 20; + break; + + case PORT_MODE_10G_KR: + case PORT_MODE_XFI: + macs[i + index].fifo_cnt = 4; + macs[i + index].prio = 2; + macs[i + index].speed = 10; + break; + + case PORT_MODE_40G_KR4: + case PORT_MODE_XLAUI: + macs[i + index].fifo_cnt = 4; + macs[i + index].prio = 3; + macs[i + index].speed = 40; + break; + + default: + macs[i + index].fifo_cnt = 0; + macs[i + index].prio = 0; + macs[i + index].speed = 0; + break; + } + + fifo_cnt += macs[i + index].fifo_cnt; + } + } + } + + /* If more fifos than available were assigned, reduce the number of + * fifos until within limit. Start with the lowest priority macs with 4 + * fifos. + */ + prio = 1; + cnt = 4; + while (fifo_cnt > get_num_fifos()) { + for (i = 0; i < get_num_output_macs(); i++) { + if (macs[i].prio == prio && macs[i].fifo_cnt == cnt) { + macs[i].fifo_cnt >>= 1; + fifo_cnt -= macs[i].fifo_cnt; + } + + if (fifo_cnt <= get_num_fifos()) + break; + } + + if (prio >= 3) { + prio = 1; + cnt >>= 1; + } else { + prio++; + } + + if (cnt == 0) + break; + } + + /* Assign left over fifos to dpi */ + if (get_num_fifos() - fifo_cnt > 0) { + if (get_num_fifos() - fifo_cnt >= 3) { + macs[1].fifo_cnt += 3; + fifo_cnt -= 3; + } else { + macs[1].fifo_cnt += 1; + fifo_cnt -= 1; + } + } + + return 0; +} + +static int setup_macs(int node) +{ + struct fifo_grp_info fifo_grp[PKO_MAX_FIFO_GROUPS]; + struct mac_info macs[PKO_MAX_OUTPUT_MACS]; + int cnt, fifo, grp, i, size; + u64 data; + + memset(macs, 0, sizeof(macs)); + memset(fifo_grp, 0, sizeof(fifo_grp)); + + /* Get the number of fifos required by each mac */ + if (OCTEON_IS_MODEL(OCTEON_CN78XX)) { + get_78xx_fifos_required(node, macs); + } else if (OCTEON_IS_MODEL(OCTEON_CNF75XX)) { + get_75xx_fifos_required(node, macs); + } else if (OCTEON_IS_MODEL(OCTEON_CN73XX)) { + get_73xx_fifos_required(node, macs); + } else { + pr_err("%s: Unsupported board type\n", __FILE__); + return -1; + } + + /* Assign fifos to each mac. Start with macs requiring 4 fifos */ + fifo = 0; + for (cnt = 4; cnt > 0; cnt >>= 1) { + for (i = 0; i < get_num_output_macs(); i++) { + if (macs[i].fifo_cnt != cnt) + continue; + + macs[i].fifo = fifo; + grp = fifo / 4; + + fifo_grp[grp].speed += macs[i].speed; + + if (cnt == 4) { + /* 10, 0, 0, 0 */ + fifo_grp[grp].size = 4; + } else if (cnt == 2) { + /* 5, 0, 5, 0 */ + fifo_grp[grp].size = 3; + } else if (cnt == 1) { + if ((fifo & 0x2) && fifo_grp[grp].size == 3) { + /* 5, 0, 2.5, 2.5 */ + fifo_grp[grp].size = 1; + } else { + /* 2.5, 2.5, 2.5, 2.5 */ + fifo_grp[grp].size = 0; + } + } + + fifo += cnt; + } + } + + /* Configure the fifo groups */ + for (i = 0; i < get_num_fifo_groups(); i++) { + data = oct_csr_read(PKO_PTGF_CFG(node, i)); + size = data & PKO_PTGF_CFG_SIZE_MASK; + if (size != fifo_grp[i].size) + data |= PKO_PTGF_CFG_RESET; + data &= ~PKO_PTGF_CFG_SIZE_MASK; + data |= fifo_grp[i].size; + + data &= ~PKO_PTGF_CFG_RATE_MASK; + if (fifo_grp[i].speed >= 40) { + if (fifo_grp[i].size >= 3) { + /* 50 Gbps */ + data |= 0x3 << PKO_PTGF_CFG_RATE_SHIFT; + } else { + /* 25 Gbps */ + data |= 0x2 << PKO_PTGF_CFG_RATE_SHIFT; + } + } else if (fifo_grp[i].speed >= 20) { + /* 25 Gbps */ + data |= 0x2 << PKO_PTGF_CFG_RATE_SHIFT; + } else if (fifo_grp[i].speed >= 10) { + /* 12.5 Gbps */ + data |= 0x1 << PKO_PTGF_CFG_RATE_SHIFT; + } + oct_csr_write(data, PKO_PTGF_CFG(node, i)); + data &= ~PKO_PTGF_CFG_RESET; + oct_csr_write(data, PKO_PTGF_CFG(node, i)); + } + + /* Configure the macs with their assigned fifo */ + for (i = 0; i < get_num_output_macs(); i++) { + data = oct_csr_read(PKO_MAC_CFG(node, i)); + data &= ~PKO_MAC_CFG_FIFO_NUM_MASK; + if (!macs[i].fifo_cnt) + data |= PKO_MAC_CFG_FIFO_UNDEFINED; + else + data |= macs[i].fifo; + oct_csr_write(data, PKO_MAC_CFG(node, i)); + } + + /* Setup mci0/mci1/skid credits */ + for (i = 0; i < get_num_output_macs(); i++) { + int fifo_credit, mac_credit, skid_credit; + + if (!macs[i].fifo_cnt) + continue; + + if (i == 0) { + /* Loopback */ + mac_credit = 4 * 1024; + skid_credit = 0; + } else if (i == 1) { + /* Dpi */ + mac_credit = 2 * 1024; + skid_credit = 0; + } else if (OCTEON_IS_MODEL(OCTEON_CN78XX) && + ((i == 2 || i == 3))) { + /* ILK */ + mac_credit = 4 * 1024; + skid_credit = 0; + } else if (OCTEON_IS_MODEL(OCTEON_CNF75XX) && + ((i >= 6 && i <= 9))) { + /* Srio */ + mac_credit = 1024 / 2; + skid_credit = 0; + } else { + /* Bgx */ + mac_credit = macs[i].num_lmacs * 8 * 1024; + skid_credit = macs[i].num_lmacs * 256; + } + + if (OCTEON_IS_MODEL(OCTEON_CN78XX_PASS1_X)) { + fifo_credit = macs[i].fifo_cnt * PKO_FIFO_SIZE; + data = (fifo_credit + mac_credit) / 16; + oct_csr_write(data, PKO_MCI0_MAX_CRED(node, i)); + } + + data = mac_credit / 16; + oct_csr_write(data, PKO_MCI1_MAX_CRED(node, i)); + + data = oct_csr_read(PKO_MAC_CFG(node, i)); + data &= ~PKO_MAC_CFG_SKID_MAX_CNT_MASK; + data |= ((skid_credit / 256) >> 1) << 5; + oct_csr_write(data, PKO_MAC_CFG(node, i)); + } + + return 0; +} + +static int hw_init_global(int node, int aura) +{ + int timeout; + u64 data; + + data = oct_csr_read(PKO_ENABLE(node)); + if (data & PKO_ENABLE_ENABLE) { + pr_info("%s: Pko already enabled on node %d\n", __FILE__, node); + return 0; + } + + /* Enable color awareness */ + data = oct_csr_read(PKO_SHAPER_CFG(node)); + data |= PKO_SHAPER_CFG_COLOR_AWARE; + oct_csr_write(data, PKO_SHAPER_CFG(node)); + + /* Clear flush command */ + oct_csr_write(0, PKO_DPFI_FLUSH(node)); + + /* Set the aura number */ + data = (node << PKO_DPFI_FPA_AURA_NODE_SHIFT) | aura; + oct_csr_write(data, PKO_DPFI_FPA_AURA(node)); + + data = PKO_DPFI_ENA_ENABLE; + oct_csr_write(data, PKO_DPFI_ENA(node)); + + /* Wait until all pointers have been returned */ + timeout = 100000; + do { + data = oct_csr_read(PKO_STATUS(node)); + if (data & PKO_STATUS_PKO_RDY) + break; + udelay(1); + timeout--; + } while (timeout); + if (!timeout) { + pr_err("%s: Pko dfpi failed on node %d\n", __FILE__, node); + return -1; + } + + /* Set max outstanding requests in IOBP for any FIFO.*/ + data = oct_csr_read(PKO_PTF_IOBP_CFG(node)); + data &= ~PKO_PTF_IOBP_CFG_MAX_READ_SIZE_MASK; + if (OCTEON_IS_MODEL(OCTEON_CN78XX)) + data |= 16; + else + data |= 3; + oct_csr_write(data, PKO_PTF_IOBP_CFG(node)); + + /* Set minimum packet size per Ethernet standard */ + data = (PKO_PDM_PAD_MINLEN_TYPICAL << PKO_PDM_PAD_MINLEN_SHIFT); + oct_csr_write(data, PKO_PDM_CFG(node)); + + /* Initialize macs and fifos */ + setup_macs(node); + + /* Enable pko */ + data = PKO_ENABLE_ENABLE; + oct_csr_write(data, PKO_ENABLE(node)); + + /* Verify pko is ready */ + data = oct_csr_read(PKO_STATUS(node)); + if (!(data & PKO_STATUS_PKO_RDY)) { + pr_err("%s: pko is not ready\n", __FILE__); + return -1; + } + + return 0; +} + +static int hw_exit_global(int node) +{ + int i, timeout; + u64 data; + + /* Wait until there are no in-flight packets */ + for (i = 0; i < get_num_fifos(); i++) { + data = oct_csr_read(PKO_PTF_STATUS(node, i)); + if ((data & PKO_PTF_STATUS_MAC_NUM_MASK) == + PKO_MAC_CFG_FIFO_UNDEFINED) + continue; + + timeout = 10000; + do { + if (!(data & PKO_PTF_STATUS_IN_FLIGHT_CNT_MASK)) + break; + udelay(1); + timeout--; + data = oct_csr_read(PKO_PTF_STATUS(node, i)); + } while (timeout); + if (!timeout) { + pr_err("%s: Timeout in-flight fifo\n", __FILE__); + return -1; + } + } + + /* Disable pko */ + oct_csr_write(0, PKO_ENABLE(node)); + + /* Reset all port queues to the virtual mac */ + for (i = 0; i < get_num_port_queues(); i++) { + data = get_num_output_macs() << PKO_L1_SQ_TOPOLOGY_LINK_SHIFT; + oct_csr_write(data, PKO_L1_SQ_TOPOLOGY(node, i)); + + data = get_num_output_macs() << PKO_L1_SQ_SHAPE_LINK_SHIFT; + oct_csr_write(data, PKO_L1_SQ_SHAPE(node, i)); + + data = (u64)get_num_output_macs() << PKO_L1_SQ_LINK_LINK_SHIFT; + oct_csr_write(data, PKO_L1_SQ_LINK(node, i)); + } + + /* Reset all output macs */ + for (i = 0; i < get_num_output_macs(); i++) { + data = PKO_MAC_CFG_FIFO_NUM_UNDEFINED; + oct_csr_write(data, PKO_MAC_CFG(node, i)); + } + + /* Reset all fifo groups */ + for (i = 0; i < get_num_fifo_groups(); i++) { + data = oct_csr_read(PKO_PTGF_CFG(node, i)); + /* Simulator asserts if an unused group is reset */ + if (data == 0) + continue; + data = PKO_PTGF_CFG_RESET; + oct_csr_write(data, PKO_PTGF_CFG(node, i)); + } + + /* Return cache pointers to fpa */ + data = PKO_DPFI_FLUSH_FLUSH_EN; + oct_csr_write(data, PKO_DPFI_FLUSH(node)); + timeout = 10000; + do { + data = oct_csr_read(PKO_DPFI_STATUS(node)); + if (data & PKO_DPFI_STATUS_CACHE_FLUSHED) + break; + udelay(1); + timeout--; + } while (timeout); + if (!timeout) { + pr_err("%s: Timeout flushing cache\n", __FILE__); + return -1; + } + oct_csr_write(0, PKO_DPFI_ENA(node)); + oct_csr_write(0, PKO_DPFI_FLUSH(node)); + + return 0; +} + +static int virtual_mac_config(int node) +{ + int dq[8], i, num_dq, parent_q, pq, queue, rc, vmac; + enum queue_level level; + + /* The virtual mac is after the last output mac. Note: for the 73xx it + * might be 2 after the last output mac (15). + */ + vmac = get_num_output_macs(); + + /* Allocate a port queue */ + rc = allocate_queues(node, PQ, 1, &pq); + if (rc < 0) { + pr_err("%s: Failed to allocate port queue\n", __FILE__); + return rc; + } + + /* Connect the port queue to the output mac */ + port_queue_init(node, pq, vmac); + + parent_q = pq; + for (level = L2_SQ; level <= max_sq_level(); level++) { + rc = allocate_queues(node, level, 1, &queue); + if (rc < 0) { + pr_err("%s: Failed to allocate queue\n", __FILE__); + return rc; + } + + switch (level) { + case L2_SQ: + scheduler_queue_l2_init(node, queue, parent_q); + break; + case L3_SQ: + scheduler_queue_l3_init(node, queue, parent_q); + break; + case L4_SQ: + scheduler_queue_l4_init(node, queue, parent_q); + break; + case L5_SQ: + scheduler_queue_l5_init(node, queue, parent_q); + break; + default: + break; + } + + parent_q = queue; + } + + if (OCTEON_IS_MODEL(OCTEON_CN78XX_PASS1_0)) + num_dq = 8; + else + num_dq = 1; + + rc = allocate_queues(node, DQ, num_dq, dq); + if (rc < 0) { + pr_err("%s: Failed to allocate description queues\n", __FILE__); + return rc; + } + + /* By convention the dq must be zero */ + if (dq[0] != 0) { + pr_err("%s: Failed to reserve description queues\n", __FILE__); + return -1; + } + descriptor_queue_init(node, dq, parent_q, num_dq); + + /* Open the dqs */ + for (i = 0; i < num_dq; i++) + open_dq(node, dq[i]); + + return 0; +} + +static int drain_dq(int node, int dq) +{ + int timeout; + u64 data; + s64 rc; + + data = PKO_DQ_SW_XOFF_SC_RAM_FLIP | PKO_DQ_SW_XOFF_SC_RAM_CDIS; + oct_csr_write(data, PKO_DQ_SW_XOFF(node, dq)); + + usleep_range(1000, 2000); + + data = 0; + oct_csr_write(data, PKO_DQ_SW_XOFF(node, dq)); + + /* Wait for the dq to drain */ + timeout = 10000; + do { + rc = query_dq(node, dq); + if (!rc) + break; + else if (rc < 0) + return rc; + udelay(1); + timeout--; + } while (timeout); + if (!timeout) { + pr_err("%s: Timeout waiting for dq to drain\n", __FILE__); + return -1; + } + + /* Close the queue anf free internal buffers */ + close_dq(node, dq); + + return 0; +} + +int octeon3_pko_exit_global(int node) +{ + int dq[8], i, num_dq; + + if (OCTEON_IS_MODEL(OCTEON_CN78XX_PASS1_0)) + num_dq = 8; + else + num_dq = 1; + + /* Shutdown the virtual/null interface */ + for (i = 0; i < ARRAY_SIZE(dq); i++) + dq[i] = i; + octeon3_pko_interface_uninit(node, dq, num_dq); + + /* Shutdown pko */ + hw_exit_global(node); + + return 0; +} +EXPORT_SYMBOL(octeon3_pko_exit_global); + +int octeon3_pko_init_global(int node, int aura) +{ + int rc; + + rc = hw_init_global(node, aura); + if (rc) + return rc; + + /* Channel credit level at level 2 */ + oct_csr_write(0, PKO_CHANNEL_LEVEL(node)); + + /* Configure the null mac */ + rc = virtual_mac_config(node); + if (rc) + return rc; + + return 0; +} +EXPORT_SYMBOL(octeon3_pko_init_global); + +int octeon3_pko_set_mac_options(int node, int interface, int index, + enum octeon3_mac_type mac_type, bool fcs_en, + bool pad_en, int fcs_sop_off) +{ + int fifo_num, mac; + u64 data; + + mac = get_output_mac(interface, index, mac_type); + + data = oct_csr_read(PKO_MAC_CFG(node, mac)); + fifo_num = data & PKO_MAC_CFG_FIFO_NUM_MASK; + if (fifo_num == PKO_MAC_CFG_FIFO_UNDEFINED) { + pr_err("%s: mac not configured %d:%d:%d\n", __FILE__, node, + interface, index); + return -ENODEV; + } + + /* Some silicon requires fifo_num=0x1f to change padding, fcs */ + data &= ~PKO_MAC_CFG_FIFO_NUM_MASK; + data |= PKO_MAC_CFG_FIFO_UNDEFINED; + + data &= ~(PKO_MAC_CFG_MIN_PAD_ENA | PKO_MAC_CFG_FCS_ENA | + PKO_MAC_CFG_FCS_SOP_OFF_MASK); + if (pad_en) + data |= PKO_MAC_CFG_MIN_PAD_ENA; + if (fcs_en) + data |= PKO_MAC_CFG_FCS_ENA; + if (fcs_sop_off) + data |= fcs_sop_off << PKO_MAC_CFG_FCS_SOP_OFF_SHIFT; + + oct_csr_write(data, PKO_MAC_CFG(node, mac)); + + data &= ~PKO_MAC_CFG_FIFO_NUM_MASK; + data |= fifo_num; + oct_csr_write(data, PKO_MAC_CFG(node, mac)); + + return 0; +} +EXPORT_SYMBOL(octeon3_pko_set_mac_options); + +int octeon3_pko_get_fifo_size(int node, int interface, int index, + enum octeon3_mac_type mac_type) +{ + int fifo_grp, fifo_off, mac, size; + u64 data; + + /* Set fifo size to 2.4 KB */ + size = PKO_FIFO_SIZE; + + mac = get_output_mac(interface, index, mac_type); + + data = oct_csr_read(PKO_MAC_CFG(node, mac)); + if ((data & PKO_MAC_CFG_FIFO_NUM_MASK) == PKO_MAC_CFG_FIFO_UNDEFINED) { + pr_err("%s: mac not configured %d:%d:%d\n", __FILE__, node, + interface, index); + return -ENODEV; + } + fifo_grp = (data & PKO_MAC_CFG_FIFO_NUM_MASK) + >> PKO_MAC_CFG_FIFO_GRP_SHIFT; + fifo_off = data & PKO_MAC_CFG_FIFO_OFF_MASK; + + data = oct_csr_read(PKO_PTGF_CFG(node, fifo_grp)); + data &= PKO_PTGF_CFG_SIZE_MASK; + switch (data) { + case 0: + /* 2.5l, 2.5k, 2.5k, 2.5k */ + break; + case 1: + /* 5.0k, 0.0k, 2.5k, 2.5k */ + if (fifo_off == 0) + size *= 2; + if (fifo_off == 1) + size = 0; + break; + case 2: + /* 2.5k, 2.5k, 5.0k, 0.0k */ + if (fifo_off == 2) + size *= 2; + if (fifo_off == 3) + size = 0; + break; + case 3: + /* 5k, 0, 5k, 0 */ + if ((fifo_off & 1) != 0) + size = 0; + size *= 2; + break; + case 4: + /* 10k, 0, 0, 0 */ + if (fifo_off != 0) + size = 0; + size *= 4; + break; + default: + size = -1; + } + + return size; +} +EXPORT_SYMBOL(octeon3_pko_get_fifo_size); + +int octeon3_pko_activate_dq(int node, int dq, int cnt) +{ + int i, rc = 0; + u64 data; + + for (i = 0; i < cnt; i++) { + rc = open_dq(node, dq + i); + if (rc) + break; + + data = oct_csr_read(PKO_PDM_DQ_MINPAD(node, dq + i)); + data &= ~PKO_PDM_DQ_MINPAD_MINPAD; + oct_csr_write(data, PKO_PDM_DQ_MINPAD(node, dq + i)); + } + + return rc; +} +EXPORT_SYMBOL(octeon3_pko_activate_dq); + +int octeon3_pko_interface_init(int node, int interface, int index, + enum octeon3_mac_type mac_type, int ipd_port) +{ + int mac, pq, parent_q, queue, rc; + enum queue_level level; + + mac = get_output_mac(interface, index, mac_type); + + /* Allocate a port queue for this interface */ + rc = allocate_queues(node, PQ, 1, &pq); + if (rc < 0) { + pr_err("%s: Failed to allocate port queue\n", __FILE__); + return rc; + } + + /* Connect the port queue to the output mac */ + port_queue_init(node, pq, mac); + + /* Link scheduler queues to the port queue */ + parent_q = pq; + for (level = L2_SQ; level <= max_sq_level(); level++) { + rc = allocate_queues(node, level, 1, &queue); + if (rc < 0) { + pr_err("%s: Failed to allocate queue\n", __FILE__); + return rc; + } + + switch (level) { + case L2_SQ: + scheduler_queue_l2_init(node, queue, parent_q); + map_channel(node, pq, queue, ipd_port); + break; + case L3_SQ: + scheduler_queue_l3_init(node, queue, parent_q); + break; + case L4_SQ: + scheduler_queue_l4_init(node, queue, parent_q); + break; + case L5_SQ: + scheduler_queue_l5_init(node, queue, parent_q); + break; + default: + break; + } + + parent_q = queue; + } + + /* Link the descriptor queue */ + rc = allocate_queues(node, DQ, 1, &queue); + if (rc < 0) { + pr_err("%s: Failed to allocate descriptor queue\n", __FILE__); + return rc; + } + descriptor_queue_init(node, &queue, parent_q, 1); + + return queue; +} +EXPORT_SYMBOL(octeon3_pko_interface_init); + +int octeon3_pko_interface_uninit(int node, const int *dq, int num_dq) +{ + int i, parent_q, queue, rc; + enum queue_level level; + u64 addr, data, mask, shift = PKO_DQ_TOPOLOGY_PARENT_SHIFT; + + /* Drain all dqs */ + for (i = 0; i < num_dq; i++) { + rc = drain_dq(node, dq[i]); + if (rc) + return rc; + + /* Free the dq */ + data = oct_csr_read(PKO_DQ_TOPOLOGY(node, dq[i])); + + parent_q = (data & PKO_DQ_TOPOLOGY_PARENT_MASK) >> shift; + free_queues(node, DQ, 1, &dq[i]); + + /* Free all the scheduler queues */ + queue = parent_q; + for (level = max_sq_level(); (signed int)level >= PQ; level--) { + switch (level) { + case L5_SQ: + addr = PKO_L5_SQ_TOPOLOGY(node, queue); + data = oct_csr_read(addr); + mask = PKO_DQ_TOPOLOGY_PARENT_MASK; + parent_q = (data & mask) >> shift; + break; + + case L4_SQ: + addr = PKO_L4_SQ_TOPOLOGY(node, queue); + data = oct_csr_read(addr); + mask = PKO_L34_DQ_TOPOLOGY_PARENT_MASK; + parent_q = (data & mask) >> shift; + break; + + case L3_SQ: + addr = PKO_L3_SQ_TOPOLOGY(node, queue); + data = oct_csr_read(addr); + mask = PKO_L34_DQ_TOPOLOGY_PARENT_MASK; + parent_q = (data & mask) >> shift; + break; + + case L2_SQ: + addr = PKO_L2_SQ_TOPOLOGY(node, queue); + data = oct_csr_read(addr); + mask = PKO_L2_DQ_TOPOLOGY_PARENT_MASK; + parent_q = (data & mask) >> shift; + break; + + case PQ: + break; + + default: + pr_err("%s: Invalid lvl=%d\n", __FILE__, level); + return -1; + } + + free_queues(node, level, 1, &queue); + queue = parent_q; + } + } + + return 0; +} +EXPORT_SYMBOL(octeon3_pko_interface_uninit); diff --git a/drivers/net/ethernet/cavium/octeon/octeon3-pko.h b/drivers/net/ethernet/cavium/octeon/octeon3-pko.h new file mode 100644 index 0000000..f1053a8 --- /dev/null +++ b/drivers/net/ethernet/cavium/octeon/octeon3-pko.h @@ -0,0 +1,159 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Octeon III Packet-Output Processing Unit (PKO) + * + * Copyright (C) 2018 Cavium, Inc. + */ +#ifndef _OCTEON3_PKO_H_ +#define _OCTEON3_PKO_H_ + +#include + +#define PKO_DPFI_ENA_ENABLE BIT(0) +#define PKO_DPFI_FLUSH_FLUSH_EN BIT(0) +#define PKO_DPFI_FPA_AURA_NODE_SHIFT 10 +#define PKO_DPFI_STATUS_CACHE_FLUSHED BIT(0) +#define PKO_DQ_SCHEDULE_PRIO_SHIFT 24 +#define PKO_DQ_SW_XOFF_SC_RAM_FLIP BIT(2) +#define PKO_DQ_SW_XOFF_SC_RAM_CDIS BIT(1) +#define PKO_DQ_TOPOLOGY_PARENT_MASK GENMASK_ULL(25, 16) +#define PKO_DQ_TOPOLOGY_PARENT_SHIFT 16 +#define PKO_DQ_WM_CTL_KIND BIT(49) +#define PKO_ENABLE_ENABLE BIT(0) +#define PKO_L12_SQ_TOPOLOGY_PRIO_ANCHOR_MASK GENMASK_ULL(40, 32) +#define PKO_L1_SQ_LINK_LINK_SHIFT 44 +#define PKO_L1_SQ_SHAPE_LINK_SHIFT 13 +#define PKO_L2_DQ_TOPOLOGY_PARENT_MASK GENMASK_ULL(20, 16) +#define PKO_L1_SQ_TOPOLOGY_LINK_SHIFT 16 +#define PKO_L345_SQ_TOPOLOGY_PRIO_ANCHOR_MASK GENMASK_ULL(41, 32) +#define PKO_L34_DQ_TOPOLOGY_PARENT_MASK GENMASK_ULL(24, 16) +#define PKO_L3_L2_CHANNEL_CC_CHANNEL_MASK GENMASK_ULL(43, 32) +#define PKO_L3_L2_CHANNEL_CC_CHANNEL_SHIFT 32 +#define PKO_LUT_PORT_MASK 0xf00 +#define PKO_LUT_PORT_TO_INDEX 0x800 +#define PKO_LUT_PQ_IDX_SHIFT 9 +#define PKO_LUT_QUEUE_NUMBER_MASK GENMASK_ULL(7, 0) +#define PKO_LUT_QUEUE_NUMBER_SHIFT 8 +#define PKO_LUT_VALID BIT(15) +#define PKO_MAC_CFG_FCS_ENA BIT(15) +#define PKO_MAC_CFG_FCS_SOP_OFF_MASK GENMASK_ULL(14, 7) +#define PKO_MAC_CFG_FCS_SOP_OFF_SHIFT 7 +#define PKO_MAC_CFG_FIFO_GRP_SHIFT 2 +#define PKO_MAC_CFG_FIFO_NUM_MASK GENMASK_ULL(4, 0) +#define PKO_MAC_CFG_FIFO_NUM_UNDEFINED 0x1f +#define PKO_MAC_CFG_FIFO_OFF_MASK GENMASK_ULL(1, 0) +#define PKO_MAC_CFG_FIFO_UNDEFINED 0x1f +#define PKO_MAC_CFG_MIN_PAD_ENA BIT(16) +#define PKO_MAC_CFG_SKID_MAX_CNT_MASK GENMASK_ULL(6, 5) +#define PKO_PDM_DQ_MINPAD_MINPAD BIT(0) +#define PKO_PDM_PAD_MINLEN_SHIFT 3 +#define PKO_PDM_PAD_MINLEN_TYPICAL 60 +#define PKO_PTF_IOBP_CFG_MAX_READ_SIZE_MASK GENMASK_ULL(6, 0) +#define PKO_PTF_STATUS_IN_FLIGHT_CNT_MASK GENMASK_ULL(11, 5) +#define PKO_PTF_STATUS_MAC_NUM_MASK GENMASK_ULL(4, 0) +#define PKO_PTGF_CFG_RATE_MASK GENMASK_ULL(5, 3) +#define PKO_PTGF_CFG_RATE_SHIFT 3 +#define PKO_PTGF_CFG_RESET BIT(6) +#define PKO_PTGF_CFG_SIZE_MASK GENMASK_ULL(2, 0) +#define PKO_QUERY_DMA_SCRADDR_SHIFT 56 +#define PKO_QUERY_DMA_RTNLEN_SHIFT 48 +#define PKO_QUERY_DMA_DID_SHIFT 40 +#define PKO_QUERY_DMA_NODE_SHIFT 36 +#define PKO_QUERY_DMA_DQOP_SHIFT 32 +#define PKO_QUERY_DMA_DQ_SHIFT 16 +#define PKO_QUERY_RTN_DEPTH_MASK GENMASK_ULL(47, 0) +#define PKO_QUERY_RTN_DQSTATUS_MASK GENMASK_ULL(63, 60) +#define PKO_QUERY_RTN_DQSTATUS_SHIFT 60 + +/* PKO_DQ_STATUS_E */ +enum pko_dq_status_e { + PKO_DQSTATUS_PASS = 0, + PKO_DQSTATUS_BADSTATE = 8, + PKO_DQSTATUS_NOFPABUF, + PKO_DQSTATUS_NOPKOBUF, + PKO_DQSTATUS_FAILRTNPTR, + PKO_DQSTATUS_ALREADYCREATED, + PKO_DQSTATUS_NOTCREATED, + PKO_DQSTATUS_NOTEMPTY, + PKO_DQSTATUS_SENDPKTDROP +}; + +#define PKO_SHAPER_CFG_COLOR_AWARE BIT(1) +#define PKO_SQ_TOPOLOGY_LINK_SHIFT 16 +#define PKO_SQ_TOPOLOGY_PRIO_ANCHOR_SHIFT 32 +#define PKO_SQ_TOPOLOGY_RR_PRIO_MASK GENMASK_ULL(4, 1) +#define PKO_SQ_TOPOLOGY_RR_PRIO_SHAPER 0xf +#define PKO_SQ_TOPOLOGY_RR_PRIO_SHIFT 1 +#define PKO_STATUS_PKO_RDY BIT(63) + +#define PKO_SEND_HDR_AURA_SHIFT 48 +#define PKO_SEND_HDR_CKL4_SHIFT 46 +#define PKO_SEND_HDR_CKL3 BIT(45) +#define PKO_SEND_HDR_LE BIT(43) +#define PKO_SEND_HDR_N2 BIT(42) +#define PKO_SEND_HDR_DF BIT(40) +#define PKO_SEND_HDR_L4PTR_MASK GENMASK_ULL(31, 24) +#define PKO_SEND_HDR_L4PTR_SHIFT 24 +#define PKO_SEND_HDR_L3PTR_SHIFT 16 + +#define PKO_SEND_SUBDC4_SHIFT 44 +#define PKO_SEND_EXT_RA_SHIFT 40 +#define PKO_SEND_EXT_TSTMP BIT(39) +#define PKO_SEND_EXT_MARKPTR_SHIFT 16 + +/* PKO_REDALG_E */ +enum pko_redalg_e { + PKO_REDALG_E_STD, + PKO_REDALG_E_SEND, + PKO_REDALG_E_STALL, + PKO_REDALG_E_DISCARD +}; + +#define PKO_SEND_TSO_L2LEN_SHIFT 56 +#define PKO_SEND_TSO_SB_SHIFT 24 +#define PKO_SEND_TSO_MSS_SHIFT 8 + +#define PKO_SEND_MEM_WMEM_SHIFT 62 +#define PKO_SEND_MEM_DSZ_SHIFT 60 +#define PKO_SEND_MEM_ALG_SHIFT 56 +#define PKO_SEND_MEM_OFFSET_SHIFT 48 + +/* PKO_MEMALG_E */ +enum pko_memalg_e { + PKO_MEMALG_SET = 0, + PKO_MEMALG_SETTSTMP, + PKO_MEMALG_SETRSLT, + PKO_MEMALG_ADD = 8, + PKO_MEMALG_SUB, + PKO_MEMALG_ADDLEN, + PKO_MEMALG_SUBLEN, + PKO_MEMALG_ADDMBUF, + PKO_MEMALG_SUBMBUF +}; + +/* PKO_MEMDSZ_E */ +enum pko_memdsz_e { + PKO_MEMDSZ_B64, + PKO_MEMDSZ_B32, + PKO_MEMDSZ_B16, + PKO_MEMDSZ_B8 +}; + +#define PKO_SEND_WORK_GRP_SHIFT 52 +#define PKO_SEND_WORK_TT_SHIFT 50 + +#define PKO_SEND_GATHER_SIZE_MASK GENMASK_ULL(63, 48) +#define PKO_SEND_GATHER_SIZE_SHIFT 48 +#define PKO_SEND_GATHER_SUBDC_SHIFT 45 +#define PKO_SEND_GATHER_ADDR_MASK GENMASK_ULL(41, 0) + +/* Pko sub-command three bit codes (SUBDC3) */ +#define PKO_SENDSUBDC_GATHER 0x1 + +/* Pko sub-command four bit codes (SUBDC4) */ +#define PKO_SENDSUBDC_TSO 0x8 +#define PKO_SENDSUBDC_FREE 0x9 +#define PKO_SENDSUBDC_WORK 0xa +#define PKO_SENDSUBDC_MEM 0xc +#define PKO_SENDSUBDC_EXT 0xd + +#endif /* _OCTEON3_PKO_H_ */ -- 2.1.4