diff options
| author |   José Pekkarinen <jose.pekkarinen@nokia.com> | 2016-04-11 10:41:07 +0300 |
|---|---|---|
| committer | José Pekkarinen <jose.pekkarinen@nokia.com> | 2016-04-13 08:17:18 +0300 |
| commit | e09b41010ba33a20a87472ee821fa407a5b8da36 (patch) | |
| tree | d10dc367189862e7ca5c592f033dc3726e1df4e3 /kernel/drivers/net/ethernet/cavium | |
| parent | f93b97fd65072de626c074dbe099a1fff05ce060 (diff) | |
These changes are the raw update to linux-4.4.6-rt14. Kernel sources
are taken from kernel.org, and rt patch from the rt wiki download page.
During the rebasing, the following patch collided:
Force tick interrupt and get rid of softirq magic(I70131fb85).
Collisions have been removed because its logic was found on the
source already.
Change-Id: I7f57a4081d9deaa0d9ccfc41a6c8daccdee3b769
Signed-off-by: José Pekkarinen <jose.pekkarinen@nokia.com>
Diffstat (limited to 'kernel/drivers/net/ethernet/cavium')
40 files changed, 22628 insertions, 0 deletions
diff --git a/kernel/drivers/net/ethernet/cavium/Kconfig b/kernel/drivers/net/ethernet/cavium/Kconfig new file mode 100644 index 000000000..8fb84e69c --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/Kconfig @@ -0,0 +1,56 @@ +# +# Cavium ethernet device configuration +# + +config NET_VENDOR_CAVIUM + bool "Cavium ethernet drivers" + depends on PCI + default y + ---help--- + Select this option if you want enable Cavium network support. + + If you have a Cavium SoC or network adapter, say Y. + +if NET_VENDOR_CAVIUM + +config THUNDER_NIC_PF + tristate "Thunder Physical function driver" + depends on 64BIT + select THUNDER_NIC_BGX + ---help--- + This driver supports Thunder's NIC physical function. + The NIC provides the controller and DMA engines to + move network traffic to/from the memory. The NIC + works closely with TNS, BGX and SerDes to implement the + functions replacing and virtualizing those of a typical + standalone PCIe NIC chip. + +config THUNDER_NIC_VF + tristate "Thunder Virtual function driver" + depends on 64BIT + ---help--- + This driver supports Thunder's NIC virtual function + +config THUNDER_NIC_BGX + tristate "Thunder MAC interface driver (BGX)" + depends on 64BIT + select PHYLIB + select MDIO_OCTEON + ---help--- + This driver supports programming and controlling of MAC + interface from NIC physical function driver. + +config LIQUIDIO + tristate "Cavium LiquidIO support" + depends on 64BIT + select PTP_1588_CLOCK + select FW_LOADER + select LIBCRC32C + ---help--- + This driver supports Cavium LiquidIO Intelligent Server Adapters + based on CN66XX and CN68XX chips. + + To compile this driver as a module, choose M here: the module + will be called liquidio. This is recommended. + +endif # NET_VENDOR_CAVIUM diff --git a/kernel/drivers/net/ethernet/cavium/Makefile b/kernel/drivers/net/ethernet/cavium/Makefile new file mode 100644 index 000000000..d22f886ac --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/Makefile @@ -0,0 +1,5 @@ +# +# Makefile for the Cavium ethernet device drivers. +# +obj-$(CONFIG_NET_VENDOR_CAVIUM) += thunder/ +obj-$(CONFIG_NET_VENDOR_CAVIUM) += liquidio/ diff --git a/kernel/drivers/net/ethernet/cavium/liquidio/Makefile b/kernel/drivers/net/ethernet/cavium/liquidio/Makefile new file mode 100644 index 000000000..2f3668068 --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/liquidio/Makefile @@ -0,0 +1,16 @@ +# +# Cavium Liquidio ethernet device driver +# +obj-$(CONFIG_LIQUIDIO) += liquidio.o + +liquidio-objs := lio_main.o \ + lio_ethtool.o \ + request_manager.o \ + response_manager.o \ + octeon_device.o \ + cn66xx_device.o \ + cn68xx_device.o \ + octeon_mem_ops.o \ + octeon_droq.o \ + octeon_console.o \ + octeon_nic.o diff --git a/kernel/drivers/net/ethernet/cavium/liquidio/cn66xx_device.c b/kernel/drivers/net/ethernet/cavium/liquidio/cn66xx_device.c new file mode 100644 index 000000000..8ad7425f8 --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/liquidio/cn66xx_device.c @@ -0,0 +1,796 @@ +/********************************************************************** +* Author: Cavium, Inc. +* +* Contact: support@cavium.com +* Please include "LiquidIO" in the subject. +* +* Copyright (c) 2003-2015 Cavium, Inc. +* +* This file is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License, Version 2, as +* published by the Free Software Foundation. +* +* This file is distributed in the hope that it will be useful, but +* AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty +* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or +* NONINFRINGEMENT. See the GNU General Public License for more +* details. +* +* This file may also be available under a different license from Cavium. +* Contact Cavium, Inc. for more information +**********************************************************************/ +#include <linux/version.h> +#include <linux/types.h> +#include <linux/list.h> +#include <linux/interrupt.h> +#include <linux/pci.h> +#include <linux/kthread.h> +#include <linux/netdevice.h> +#include "octeon_config.h" +#include "liquidio_common.h" +#include "octeon_droq.h" +#include "octeon_iq.h" +#include "response_manager.h" +#include "octeon_device.h" +#include "octeon_nic.h" +#include "octeon_main.h" +#include "octeon_network.h" +#include "cn66xx_regs.h" +#include "cn66xx_device.h" +#include "liquidio_image.h" +#include "octeon_mem_ops.h" + +int lio_cn6xxx_soft_reset(struct octeon_device *oct) +{ + octeon_write_csr64(oct, CN6XXX_WIN_WR_MASK_REG, 0xFF); + + dev_dbg(&oct->pci_dev->dev, "BIST enabled for soft reset\n"); + + lio_pci_writeq(oct, 1, CN6XXX_CIU_SOFT_BIST); + octeon_write_csr64(oct, CN6XXX_SLI_SCRATCH1, 0x1234ULL); + + lio_pci_readq(oct, CN6XXX_CIU_SOFT_RST); + lio_pci_writeq(oct, 1, CN6XXX_CIU_SOFT_RST); + + /* make sure that the reset is written before starting timer */ + mmiowb(); + + /* Wait for 10ms as Octeon resets. */ + mdelay(100); + + if (octeon_read_csr64(oct, CN6XXX_SLI_SCRATCH1) == 0x1234ULL) { + dev_err(&oct->pci_dev->dev, "Soft reset failed\n"); + return 1; + } + + dev_dbg(&oct->pci_dev->dev, "Reset completed\n"); + octeon_write_csr64(oct, CN6XXX_WIN_WR_MASK_REG, 0xFF); + + return 0; +} + +void lio_cn6xxx_enable_error_reporting(struct octeon_device *oct) +{ + u32 val; + + pci_read_config_dword(oct->pci_dev, CN6XXX_PCIE_DEVCTL, &val); + if (val & 0x000f0000) { + dev_err(&oct->pci_dev->dev, "PCI-E Link error detected: 0x%08x\n", + val & 0x000f0000); + } + + val |= 0xf; /* Enable Link error reporting */ + + dev_dbg(&oct->pci_dev->dev, "Enabling PCI-E error reporting..\n"); + pci_write_config_dword(oct->pci_dev, CN6XXX_PCIE_DEVCTL, val); +} + +void lio_cn6xxx_setup_pcie_mps(struct octeon_device *oct, + enum octeon_pcie_mps mps) +{ + u32 val; + u64 r64; + + /* Read config register for MPS */ + pci_read_config_dword(oct->pci_dev, CN6XXX_PCIE_DEVCTL, &val); + + if (mps == PCIE_MPS_DEFAULT) { + mps = ((val & (0x7 << 5)) >> 5); + } else { + val &= ~(0x7 << 5); /* Turn off any MPS bits */ + val |= (mps << 5); /* Set MPS */ + pci_write_config_dword(oct->pci_dev, CN6XXX_PCIE_DEVCTL, val); + } + + /* Set MPS in DPI_SLI_PRT0_CFG to the same value. */ + r64 = lio_pci_readq(oct, CN6XXX_DPI_SLI_PRTX_CFG(oct->pcie_port)); + r64 |= (mps << 4); + lio_pci_writeq(oct, r64, CN6XXX_DPI_SLI_PRTX_CFG(oct->pcie_port)); +} + +void lio_cn6xxx_setup_pcie_mrrs(struct octeon_device *oct, + enum octeon_pcie_mrrs mrrs) +{ + u32 val; + u64 r64; + + /* Read config register for MRRS */ + pci_read_config_dword(oct->pci_dev, CN6XXX_PCIE_DEVCTL, &val); + + if (mrrs == PCIE_MRRS_DEFAULT) { + mrrs = ((val & (0x7 << 12)) >> 12); + } else { + val &= ~(0x7 << 12); /* Turn off any MRRS bits */ + val |= (mrrs << 12); /* Set MRRS */ + pci_write_config_dword(oct->pci_dev, CN6XXX_PCIE_DEVCTL, val); + } + + /* Set MRRS in SLI_S2M_PORT0_CTL to the same value. */ + r64 = octeon_read_csr64(oct, CN6XXX_SLI_S2M_PORTX_CTL(oct->pcie_port)); + r64 |= mrrs; + octeon_write_csr64(oct, CN6XXX_SLI_S2M_PORTX_CTL(oct->pcie_port), r64); + + /* Set MRRS in DPI_SLI_PRT0_CFG to the same value. */ + r64 = lio_pci_readq(oct, CN6XXX_DPI_SLI_PRTX_CFG(oct->pcie_port)); + r64 |= mrrs; + lio_pci_writeq(oct, r64, CN6XXX_DPI_SLI_PRTX_CFG(oct->pcie_port)); +} + +u32 lio_cn6xxx_coprocessor_clock(struct octeon_device *oct) +{ + /* Bits 29:24 of MIO_RST_BOOT holds the ref. clock multiplier + * for SLI. + */ + return ((lio_pci_readq(oct, CN6XXX_MIO_RST_BOOT) >> 24) & 0x3f) * 50; +} + +u32 lio_cn6xxx_get_oq_ticks(struct octeon_device *oct, + u32 time_intr_in_us) +{ + /* This gives the SLI clock per microsec */ + u32 oqticks_per_us = lio_cn6xxx_coprocessor_clock(oct); + + /* core clock per us / oq ticks will be fractional. TO avoid that + * we use the method below. + */ + + /* This gives the clock cycles per millisecond */ + oqticks_per_us *= 1000; + + /* This gives the oq ticks (1024 core clock cycles) per millisecond */ + oqticks_per_us /= 1024; + + /* time_intr is in microseconds. The next 2 steps gives the oq ticks + * corressponding to time_intr. + */ + oqticks_per_us *= time_intr_in_us; + oqticks_per_us /= 1000; + + return oqticks_per_us; +} + +void lio_cn6xxx_setup_global_input_regs(struct octeon_device *oct) +{ + /* Select Round-Robin Arb, ES, RO, NS for Input Queues */ + octeon_write_csr(oct, CN6XXX_SLI_PKT_INPUT_CONTROL, + CN6XXX_INPUT_CTL_MASK); + + /* Instruction Read Size - Max 4 instructions per PCIE Read */ + octeon_write_csr64(oct, CN6XXX_SLI_PKT_INSTR_RD_SIZE, + 0xFFFFFFFFFFFFFFFFULL); + + /* Select PCIE Port for all Input rings. */ + octeon_write_csr64(oct, CN6XXX_SLI_IN_PCIE_PORT, + (oct->pcie_port * 0x5555555555555555ULL)); +} + +static void lio_cn66xx_setup_pkt_ctl_regs(struct octeon_device *oct) +{ + u64 pktctl; + + struct octeon_cn6xxx *cn6xxx = (struct octeon_cn6xxx *)oct->chip; + + pktctl = octeon_read_csr64(oct, CN6XXX_SLI_PKT_CTL); + + /* 66XX SPECIFIC */ + if (CFG_GET_OQ_MAX_Q(cn6xxx->conf) <= 4) + /* Disable RING_EN if only upto 4 rings are used. */ + pktctl &= ~(1 << 4); + else + pktctl |= (1 << 4); + + if (CFG_GET_IS_SLI_BP_ON(cn6xxx->conf)) + pktctl |= 0xF; + else + /* Disable per-port backpressure. */ + pktctl &= ~0xF; + octeon_write_csr64(oct, CN6XXX_SLI_PKT_CTL, pktctl); +} + +void lio_cn6xxx_setup_global_output_regs(struct octeon_device *oct) +{ + u32 time_threshold; + struct octeon_cn6xxx *cn6xxx = (struct octeon_cn6xxx *)oct->chip; + + /* / Select PCI-E Port for all Output queues */ + octeon_write_csr64(oct, CN6XXX_SLI_PKT_PCIE_PORT64, + (oct->pcie_port * 0x5555555555555555ULL)); + + if (CFG_GET_IS_SLI_BP_ON(cn6xxx->conf)) { + octeon_write_csr64(oct, CN6XXX_SLI_OQ_WMARK, 32); + } else { + /* / Set Output queue watermark to 0 to disable backpressure */ + octeon_write_csr64(oct, CN6XXX_SLI_OQ_WMARK, 0); + } + + /* / Select Info Ptr for length & data */ + octeon_write_csr(oct, CN6XXX_SLI_PKT_IPTR, 0xFFFFFFFF); + + /* / Select Packet count instead of bytes for SLI_PKTi_CNTS[CNT] */ + octeon_write_csr(oct, CN6XXX_SLI_PKT_OUT_BMODE, 0); + + /* / Select ES,RO,NS setting from register for Output Queue Packet + * Address + */ + octeon_write_csr(oct, CN6XXX_SLI_PKT_DPADDR, 0xFFFFFFFF); + + /* No Relaxed Ordering, No Snoop, 64-bit swap for Output + * Queue ScatterList + */ + octeon_write_csr(oct, CN6XXX_SLI_PKT_SLIST_ROR, 0); + octeon_write_csr(oct, CN6XXX_SLI_PKT_SLIST_NS, 0); + + /* / ENDIAN_SPECIFIC CHANGES - 0 works for LE. */ +#ifdef __BIG_ENDIAN_BITFIELD + octeon_write_csr64(oct, CN6XXX_SLI_PKT_SLIST_ES64, + 0x5555555555555555ULL); +#else + octeon_write_csr64(oct, CN6XXX_SLI_PKT_SLIST_ES64, 0ULL); +#endif + + /* / No Relaxed Ordering, No Snoop, 64-bit swap for Output Queue Data */ + octeon_write_csr(oct, CN6XXX_SLI_PKT_DATA_OUT_ROR, 0); + octeon_write_csr(oct, CN6XXX_SLI_PKT_DATA_OUT_NS, 0); + octeon_write_csr64(oct, CN6XXX_SLI_PKT_DATA_OUT_ES64, + 0x5555555555555555ULL); + + /* / Set up interrupt packet and time threshold */ + octeon_write_csr(oct, CN6XXX_SLI_OQ_INT_LEVEL_PKTS, + (u32)CFG_GET_OQ_INTR_PKT(cn6xxx->conf)); + time_threshold = + lio_cn6xxx_get_oq_ticks(oct, (u32) + CFG_GET_OQ_INTR_TIME(cn6xxx->conf)); + + octeon_write_csr(oct, CN6XXX_SLI_OQ_INT_LEVEL_TIME, time_threshold); +} + +static int lio_cn6xxx_setup_device_regs(struct octeon_device *oct) +{ + lio_cn6xxx_setup_pcie_mps(oct, PCIE_MPS_DEFAULT); + lio_cn6xxx_setup_pcie_mrrs(oct, PCIE_MRRS_512B); + lio_cn6xxx_enable_error_reporting(oct); + + lio_cn6xxx_setup_global_input_regs(oct); + lio_cn66xx_setup_pkt_ctl_regs(oct); + lio_cn6xxx_setup_global_output_regs(oct); + + /* Default error timeout value should be 0x200000 to avoid host hang + * when reads invalid register + */ + octeon_write_csr64(oct, CN6XXX_SLI_WINDOW_CTL, 0x200000ULL); + return 0; +} + +void lio_cn6xxx_setup_iq_regs(struct octeon_device *oct, u32 iq_no) +{ + struct octeon_instr_queue *iq = oct->instr_queue[iq_no]; + + /* Disable Packet-by-Packet mode; No Parse Mode or Skip length */ + octeon_write_csr64(oct, CN6XXX_SLI_IQ_PKT_INSTR_HDR64(iq_no), 0); + + /* Write the start of the input queue's ring and its size */ + octeon_write_csr64(oct, CN6XXX_SLI_IQ_BASE_ADDR64(iq_no), + iq->base_addr_dma); + octeon_write_csr(oct, CN6XXX_SLI_IQ_SIZE(iq_no), iq->max_count); + + /* Remember the doorbell & instruction count register addr for this + * queue + */ + iq->doorbell_reg = oct->mmio[0].hw_addr + CN6XXX_SLI_IQ_DOORBELL(iq_no); + iq->inst_cnt_reg = oct->mmio[0].hw_addr + + CN6XXX_SLI_IQ_INSTR_COUNT(iq_no); + dev_dbg(&oct->pci_dev->dev, "InstQ[%d]:dbell reg @ 0x%p instcnt_reg @ 0x%p\n", + iq_no, iq->doorbell_reg, iq->inst_cnt_reg); + + /* Store the current instruction counter + * (used in flush_iq calculation) + */ + iq->reset_instr_cnt = readl(iq->inst_cnt_reg); +} + +static void lio_cn66xx_setup_iq_regs(struct octeon_device *oct, u32 iq_no) +{ + lio_cn6xxx_setup_iq_regs(oct, iq_no); + + /* Backpressure for this queue - WMARK set to all F's. This effectively + * disables the backpressure mechanism. + */ + octeon_write_csr64(oct, CN66XX_SLI_IQ_BP64(iq_no), + (0xFFFFFFFFULL << 32)); +} + +void lio_cn6xxx_setup_oq_regs(struct octeon_device *oct, u32 oq_no) +{ + u32 intr; + struct octeon_droq *droq = oct->droq[oq_no]; + + octeon_write_csr64(oct, CN6XXX_SLI_OQ_BASE_ADDR64(oq_no), + droq->desc_ring_dma); + octeon_write_csr(oct, CN6XXX_SLI_OQ_SIZE(oq_no), droq->max_count); + + octeon_write_csr(oct, CN6XXX_SLI_OQ_BUFF_INFO_SIZE(oq_no), + (droq->buffer_size | (OCT_RH_SIZE << 16))); + + /* Get the mapped address of the pkt_sent and pkts_credit regs */ + droq->pkts_sent_reg = + oct->mmio[0].hw_addr + CN6XXX_SLI_OQ_PKTS_SENT(oq_no); + droq->pkts_credit_reg = + oct->mmio[0].hw_addr + CN6XXX_SLI_OQ_PKTS_CREDIT(oq_no); + + /* Enable this output queue to generate Packet Timer Interrupt */ + intr = octeon_read_csr(oct, CN6XXX_SLI_PKT_TIME_INT_ENB); + intr |= (1 << oq_no); + octeon_write_csr(oct, CN6XXX_SLI_PKT_TIME_INT_ENB, intr); + + /* Enable this output queue to generate Packet Timer Interrupt */ + intr = octeon_read_csr(oct, CN6XXX_SLI_PKT_CNT_INT_ENB); + intr |= (1 << oq_no); + octeon_write_csr(oct, CN6XXX_SLI_PKT_CNT_INT_ENB, intr); +} + +void lio_cn6xxx_enable_io_queues(struct octeon_device *oct) +{ + u32 mask; + + mask = octeon_read_csr(oct, CN6XXX_SLI_PKT_INSTR_SIZE); + mask |= oct->io_qmask.iq64B; + octeon_write_csr(oct, CN6XXX_SLI_PKT_INSTR_SIZE, mask); + + mask = octeon_read_csr(oct, CN6XXX_SLI_PKT_INSTR_ENB); + mask |= oct->io_qmask.iq; + octeon_write_csr(oct, CN6XXX_SLI_PKT_INSTR_ENB, mask); + + mask = octeon_read_csr(oct, CN6XXX_SLI_PKT_OUT_ENB); + mask |= oct->io_qmask.oq; + octeon_write_csr(oct, CN6XXX_SLI_PKT_OUT_ENB, mask); +} + +void lio_cn6xxx_disable_io_queues(struct octeon_device *oct) +{ + u32 mask, i, loop = HZ; + u32 d32; + + /* Reset the Enable bits for Input Queues. */ + mask = octeon_read_csr(oct, CN6XXX_SLI_PKT_INSTR_ENB); + mask ^= oct->io_qmask.iq; + octeon_write_csr(oct, CN6XXX_SLI_PKT_INSTR_ENB, mask); + + /* Wait until hardware indicates that the queues are out of reset. */ + mask = oct->io_qmask.iq; + d32 = octeon_read_csr(oct, CN6XXX_SLI_PORT_IN_RST_IQ); + while (((d32 & mask) != mask) && loop--) { + d32 = octeon_read_csr(oct, CN6XXX_SLI_PORT_IN_RST_IQ); + schedule_timeout_uninterruptible(1); + } + + /* Reset the doorbell register for each Input queue. */ + for (i = 0; i < MAX_OCTEON_INSTR_QUEUES; i++) { + if (!(oct->io_qmask.iq & (1UL << i))) + continue; + octeon_write_csr(oct, CN6XXX_SLI_IQ_DOORBELL(i), 0xFFFFFFFF); + d32 = octeon_read_csr(oct, CN6XXX_SLI_IQ_DOORBELL(i)); + } + + /* Reset the Enable bits for Output Queues. */ + mask = octeon_read_csr(oct, CN6XXX_SLI_PKT_OUT_ENB); + mask ^= oct->io_qmask.oq; + octeon_write_csr(oct, CN6XXX_SLI_PKT_OUT_ENB, mask); + + /* Wait until hardware indicates that the queues are out of reset. */ + loop = HZ; + mask = oct->io_qmask.oq; + d32 = octeon_read_csr(oct, CN6XXX_SLI_PORT_IN_RST_OQ); + while (((d32 & mask) != mask) && loop--) { + d32 = octeon_read_csr(oct, CN6XXX_SLI_PORT_IN_RST_OQ); + schedule_timeout_uninterruptible(1); + } + ; + + /* Reset the doorbell register for each Output queue. */ + /* for (i = 0; i < oct->num_oqs; i++) { */ + for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES; i++) { + if (!(oct->io_qmask.oq & (1UL << i))) + continue; + octeon_write_csr(oct, CN6XXX_SLI_OQ_PKTS_CREDIT(i), 0xFFFFFFFF); + d32 = octeon_read_csr(oct, CN6XXX_SLI_OQ_PKTS_CREDIT(i)); + + d32 = octeon_read_csr(oct, CN6XXX_SLI_OQ_PKTS_SENT(i)); + octeon_write_csr(oct, CN6XXX_SLI_OQ_PKTS_SENT(i), d32); + } + + d32 = octeon_read_csr(oct, CN6XXX_SLI_PKT_CNT_INT); + if (d32) + octeon_write_csr(oct, CN6XXX_SLI_PKT_CNT_INT, d32); + + d32 = octeon_read_csr(oct, CN6XXX_SLI_PKT_TIME_INT); + if (d32) + octeon_write_csr(oct, CN6XXX_SLI_PKT_TIME_INT, d32); +} + +void lio_cn6xxx_reinit_regs(struct octeon_device *oct) +{ + u32 i; + + for (i = 0; i < MAX_OCTEON_INSTR_QUEUES; i++) { + if (!(oct->io_qmask.iq & (1UL << i))) + continue; + oct->fn_list.setup_iq_regs(oct, i); + } + + for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES; i++) { + if (!(oct->io_qmask.oq & (1UL << i))) + continue; + oct->fn_list.setup_oq_regs(oct, i); + } + + oct->fn_list.setup_device_regs(oct); + + oct->fn_list.enable_interrupt(oct->chip); + + oct->fn_list.enable_io_queues(oct); + + /* for (i = 0; i < oct->num_oqs; i++) { */ + for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES; i++) { + if (!(oct->io_qmask.oq & (1UL << i))) + continue; + writel(oct->droq[i]->max_count, oct->droq[i]->pkts_credit_reg); + } +} + +void +lio_cn6xxx_bar1_idx_setup(struct octeon_device *oct, + u64 core_addr, + u32 idx, + int valid) +{ + u64 bar1; + + if (valid == 0) { + bar1 = lio_pci_readq(oct, CN6XXX_BAR1_REG(idx, oct->pcie_port)); + lio_pci_writeq(oct, (bar1 & 0xFFFFFFFEULL), + CN6XXX_BAR1_REG(idx, oct->pcie_port)); + bar1 = lio_pci_readq(oct, CN6XXX_BAR1_REG(idx, oct->pcie_port)); + return; + } + + /* Bits 17:4 of the PCI_BAR1_INDEXx stores bits 35:22 of + * the Core Addr + */ + lio_pci_writeq(oct, (((core_addr >> 22) << 4) | PCI_BAR1_MASK), + CN6XXX_BAR1_REG(idx, oct->pcie_port)); + + bar1 = lio_pci_readq(oct, CN6XXX_BAR1_REG(idx, oct->pcie_port)); +} + +void lio_cn6xxx_bar1_idx_write(struct octeon_device *oct, + u32 idx, + u32 mask) +{ + lio_pci_writeq(oct, mask, CN6XXX_BAR1_REG(idx, oct->pcie_port)); +} + +u32 lio_cn6xxx_bar1_idx_read(struct octeon_device *oct, u32 idx) +{ + return (u32)lio_pci_readq(oct, CN6XXX_BAR1_REG(idx, oct->pcie_port)); +} + +u32 +lio_cn6xxx_update_read_index(struct octeon_device *oct __attribute__((unused)), + struct octeon_instr_queue *iq) +{ + u32 new_idx = readl(iq->inst_cnt_reg); + + /* The new instr cnt reg is a 32-bit counter that can roll over. We have + * noted the counter's initial value at init time into + * reset_instr_cnt + */ + if (iq->reset_instr_cnt < new_idx) + new_idx -= iq->reset_instr_cnt; + else + new_idx += (0xffffffff - iq->reset_instr_cnt) + 1; + + /* Modulo of the new index with the IQ size will give us + * the new index. + */ + new_idx %= iq->max_count; + + return new_idx; +} + +void lio_cn6xxx_enable_interrupt(void *chip) +{ + struct octeon_cn6xxx *cn6xxx = (struct octeon_cn6xxx *)chip; + u64 mask = cn6xxx->intr_mask64 | CN6XXX_INTR_DMA0_FORCE; + + /* Enable Interrupt */ + writeq(mask, cn6xxx->intr_enb_reg64); +} + +void lio_cn6xxx_disable_interrupt(void *chip) +{ + struct octeon_cn6xxx *cn6xxx = (struct octeon_cn6xxx *)chip; + + /* Disable Interrupts */ + writeq(0, cn6xxx->intr_enb_reg64); + + /* make sure interrupts are really disabled */ + mmiowb(); +} + +static void lio_cn6xxx_get_pcie_qlmport(struct octeon_device *oct) +{ + /* CN63xx Pass2 and newer parts implements the SLI_MAC_NUMBER register + * to determine the PCIE port # + */ + oct->pcie_port = octeon_read_csr(oct, CN6XXX_SLI_MAC_NUMBER) & 0xff; + + dev_dbg(&oct->pci_dev->dev, "Using PCIE Port %d\n", oct->pcie_port); +} + +void +lio_cn6xxx_process_pcie_error_intr(struct octeon_device *oct, u64 intr64) +{ + dev_err(&oct->pci_dev->dev, "Error Intr: 0x%016llx\n", + CVM_CAST64(intr64)); +} + +int lio_cn6xxx_process_droq_intr_regs(struct octeon_device *oct) +{ + struct octeon_droq *droq; + u32 oq_no, pkt_count, droq_time_mask, droq_mask, droq_int_enb; + u32 droq_cnt_enb, droq_cnt_mask; + + droq_cnt_enb = octeon_read_csr(oct, CN6XXX_SLI_PKT_CNT_INT_ENB); + droq_cnt_mask = octeon_read_csr(oct, CN6XXX_SLI_PKT_CNT_INT); + droq_mask = droq_cnt_mask & droq_cnt_enb; + + droq_time_mask = octeon_read_csr(oct, CN6XXX_SLI_PKT_TIME_INT); + droq_int_enb = octeon_read_csr(oct, CN6XXX_SLI_PKT_TIME_INT_ENB); + droq_mask |= (droq_time_mask & droq_int_enb); + + droq_mask &= oct->io_qmask.oq; + + oct->droq_intr = 0; + + /* for (oq_no = 0; oq_no < oct->num_oqs; oq_no++) { */ + for (oq_no = 0; oq_no < MAX_OCTEON_OUTPUT_QUEUES; oq_no++) { + if (!(droq_mask & (1 << oq_no))) + continue; + + droq = oct->droq[oq_no]; + pkt_count = octeon_droq_check_hw_for_pkts(oct, droq); + if (pkt_count) { + oct->droq_intr |= (1ULL << oq_no); + if (droq->ops.poll_mode) { + u32 value; + u32 reg; + + struct octeon_cn6xxx *cn6xxx = + (struct octeon_cn6xxx *)oct->chip; + + /* disable interrupts for this droq */ + spin_lock + (&cn6xxx->lock_for_droq_int_enb_reg); + reg = CN6XXX_SLI_PKT_TIME_INT_ENB; + value = octeon_read_csr(oct, reg); + value &= ~(1 << oq_no); + octeon_write_csr(oct, reg, value); + reg = CN6XXX_SLI_PKT_CNT_INT_ENB; + value = octeon_read_csr(oct, reg); + value &= ~(1 << oq_no); + octeon_write_csr(oct, reg, value); + + /* Ensure that the enable register is written. + */ + mmiowb(); + + spin_unlock(&cn6xxx->lock_for_droq_int_enb_reg); + } + } + } + + droq_time_mask &= oct->io_qmask.oq; + droq_cnt_mask &= oct->io_qmask.oq; + + /* Reset the PKT_CNT/TIME_INT registers. */ + if (droq_time_mask) + octeon_write_csr(oct, CN6XXX_SLI_PKT_TIME_INT, droq_time_mask); + + if (droq_cnt_mask) /* reset PKT_CNT register:66xx */ + octeon_write_csr(oct, CN6XXX_SLI_PKT_CNT_INT, droq_cnt_mask); + + return 0; +} + +irqreturn_t lio_cn6xxx_process_interrupt_regs(void *dev) +{ + struct octeon_device *oct = (struct octeon_device *)dev; + struct octeon_cn6xxx *cn6xxx = (struct octeon_cn6xxx *)oct->chip; + u64 intr64; + + intr64 = readq(cn6xxx->intr_sum_reg64); + + /* If our device has interrupted, then proceed. + * Also check for all f's if interrupt was triggered on an error + * and the PCI read fails. + */ + if (!intr64 || (intr64 == 0xFFFFFFFFFFFFFFFFULL)) + return IRQ_NONE; + + oct->int_status = 0; + + if (intr64 & CN6XXX_INTR_ERR) + lio_cn6xxx_process_pcie_error_intr(oct, intr64); + + if (intr64 & CN6XXX_INTR_PKT_DATA) { + lio_cn6xxx_process_droq_intr_regs(oct); + oct->int_status |= OCT_DEV_INTR_PKT_DATA; + } + + if (intr64 & CN6XXX_INTR_DMA0_FORCE) + oct->int_status |= OCT_DEV_INTR_DMA0_FORCE; + + if (intr64 & CN6XXX_INTR_DMA1_FORCE) + oct->int_status |= OCT_DEV_INTR_DMA1_FORCE; + + /* Clear the current interrupts */ + writeq(intr64, cn6xxx->intr_sum_reg64); + + return IRQ_HANDLED; +} + +void lio_cn6xxx_setup_reg_address(struct octeon_device *oct, + void *chip, + struct octeon_reg_list *reg_list) +{ + u8 __iomem *bar0_pciaddr = oct->mmio[0].hw_addr; + struct octeon_cn6xxx *cn6xxx = (struct octeon_cn6xxx *)chip; + + reg_list->pci_win_wr_addr_hi = + (u32 __iomem *)(bar0_pciaddr + CN6XXX_WIN_WR_ADDR_HI); + reg_list->pci_win_wr_addr_lo = + (u32 __iomem *)(bar0_pciaddr + CN6XXX_WIN_WR_ADDR_LO); + reg_list->pci_win_wr_addr = + (u64 __iomem *)(bar0_pciaddr + CN6XXX_WIN_WR_ADDR64); + + reg_list->pci_win_rd_addr_hi = + (u32 __iomem *)(bar0_pciaddr + CN6XXX_WIN_RD_ADDR_HI); + reg_list->pci_win_rd_addr_lo = + (u32 __iomem *)(bar0_pciaddr + CN6XXX_WIN_RD_ADDR_LO); + reg_list->pci_win_rd_addr = + (u64 __iomem *)(bar0_pciaddr + CN6XXX_WIN_RD_ADDR64); + + reg_list->pci_win_wr_data_hi = + (u32 __iomem *)(bar0_pciaddr + CN6XXX_WIN_WR_DATA_HI); + reg_list->pci_win_wr_data_lo = + (u32 __iomem *)(bar0_pciaddr + CN6XXX_WIN_WR_DATA_LO); + reg_list->pci_win_wr_data = + (u64 __iomem *)(bar0_pciaddr + CN6XXX_WIN_WR_DATA64); + + reg_list->pci_win_rd_data_hi = + (u32 __iomem *)(bar0_pciaddr + CN6XXX_WIN_RD_DATA_HI); + reg_list->pci_win_rd_data_lo = + (u32 __iomem *)(bar0_pciaddr + CN6XXX_WIN_RD_DATA_LO); + reg_list->pci_win_rd_data = + (u64 __iomem *)(bar0_pciaddr + CN6XXX_WIN_RD_DATA64); + + lio_cn6xxx_get_pcie_qlmport(oct); + + cn6xxx->intr_sum_reg64 = bar0_pciaddr + CN6XXX_SLI_INT_SUM64; + cn6xxx->intr_mask64 = CN6XXX_INTR_MASK; + cn6xxx->intr_enb_reg64 = + bar0_pciaddr + CN6XXX_SLI_INT_ENB64(oct->pcie_port); +} + +int lio_setup_cn66xx_octeon_device(struct octeon_device *oct) +{ + struct octeon_cn6xxx *cn6xxx = (struct octeon_cn6xxx *)oct->chip; + + if (octeon_map_pci_barx(oct, 0, 0)) + return 1; + + if (octeon_map_pci_barx(oct, 1, MAX_BAR1_IOREMAP_SIZE)) { + dev_err(&oct->pci_dev->dev, "%s CN66XX BAR1 map failed\n", + __func__); + octeon_unmap_pci_barx(oct, 0); + return 1; + } + + spin_lock_init(&cn6xxx->lock_for_droq_int_enb_reg); + + oct->fn_list.setup_iq_regs = lio_cn66xx_setup_iq_regs; + oct->fn_list.setup_oq_regs = lio_cn6xxx_setup_oq_regs; + + oct->fn_list.soft_reset = lio_cn6xxx_soft_reset; + oct->fn_list.setup_device_regs = lio_cn6xxx_setup_device_regs; + oct->fn_list.reinit_regs = lio_cn6xxx_reinit_regs; + oct->fn_list.update_iq_read_idx = lio_cn6xxx_update_read_index; + + oct->fn_list.bar1_idx_setup = lio_cn6xxx_bar1_idx_setup; + oct->fn_list.bar1_idx_write = lio_cn6xxx_bar1_idx_write; + oct->fn_list.bar1_idx_read = lio_cn6xxx_bar1_idx_read; + + oct->fn_list.process_interrupt_regs = lio_cn6xxx_process_interrupt_regs; + oct->fn_list.enable_interrupt = lio_cn6xxx_enable_interrupt; + oct->fn_list.disable_interrupt = lio_cn6xxx_disable_interrupt; + + oct->fn_list.enable_io_queues = lio_cn6xxx_enable_io_queues; + oct->fn_list.disable_io_queues = lio_cn6xxx_disable_io_queues; + + lio_cn6xxx_setup_reg_address(oct, oct->chip, &oct->reg_list); + + cn6xxx->conf = (struct octeon_config *) + oct_get_config_info(oct, LIO_210SV); + if (!cn6xxx->conf) { + dev_err(&oct->pci_dev->dev, "%s No Config found for CN66XX\n", + __func__); + octeon_unmap_pci_barx(oct, 0); + octeon_unmap_pci_barx(oct, 1); + return 1; + } + + oct->coproc_clock_rate = 1000000ULL * lio_cn6xxx_coprocessor_clock(oct); + + return 0; +} + +int lio_validate_cn6xxx_config_info(struct octeon_device *oct, + struct octeon_config *conf6xxx) +{ + /* int total_instrs = 0; */ + + if (CFG_GET_IQ_MAX_Q(conf6xxx) > CN6XXX_MAX_INPUT_QUEUES) { + dev_err(&oct->pci_dev->dev, "%s: Num IQ (%d) exceeds Max (%d)\n", + __func__, CFG_GET_IQ_MAX_Q(conf6xxx), + CN6XXX_MAX_INPUT_QUEUES); + return 1; + } + + if (CFG_GET_OQ_MAX_Q(conf6xxx) > CN6XXX_MAX_OUTPUT_QUEUES) { + dev_err(&oct->pci_dev->dev, "%s: Num OQ (%d) exceeds Max (%d)\n", + __func__, CFG_GET_OQ_MAX_Q(conf6xxx), + CN6XXX_MAX_OUTPUT_QUEUES); + return 1; + } + + if (CFG_GET_IQ_INSTR_TYPE(conf6xxx) != OCTEON_32BYTE_INSTR && + CFG_GET_IQ_INSTR_TYPE(conf6xxx) != OCTEON_64BYTE_INSTR) { + dev_err(&oct->pci_dev->dev, "%s: Invalid instr type for IQ\n", + __func__); + return 1; + } + if (!(CFG_GET_OQ_INFO_PTR(conf6xxx)) || + !(CFG_GET_OQ_REFILL_THRESHOLD(conf6xxx))) { + dev_err(&oct->pci_dev->dev, "%s: Invalid parameter for OQ\n", + __func__); + return 1; + } + + if (!(CFG_GET_OQ_INTR_TIME(conf6xxx))) { + dev_err(&oct->pci_dev->dev, "%s: No Time Interrupt for OQ\n", + __func__); + return 1; + } + + return 0; +} diff --git a/kernel/drivers/net/ethernet/cavium/liquidio/cn66xx_device.h b/kernel/drivers/net/ethernet/cavium/liquidio/cn66xx_device.h new file mode 100644 index 000000000..f77918779 --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/liquidio/cn66xx_device.h @@ -0,0 +1,107 @@ +/********************************************************************** +* Author: Cavium, Inc. +* +* Contact: support@cavium.com +* Please include "LiquidIO" in the subject. +* +* Copyright (c) 2003-2015 Cavium, Inc. +* +* This file is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License, Version 2, as +* published by the Free Software Foundation. +* +* This file is distributed in the hope that it will be useful, but +* AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty +* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or +* NONINFRINGEMENT. See the GNU General Public License for more +* details. +* +* This file may also be available under a different license from Cavium. +* Contact Cavium, Inc. for more information +**********************************************************************/ + +/*! \file cn66xx_device.h + * \brief Host Driver: Routines that perform CN66XX specific operations. + */ + +#ifndef __CN66XX_DEVICE_H__ +#define __CN66XX_DEVICE_H__ + +/* Register address and configuration for a CN6XXX devices. + * If device specific changes need to be made then add a struct to include + * device specific fields as shown in the commented section + */ +struct octeon_cn6xxx { + /** PCI interrupt summary register */ + u8 __iomem *intr_sum_reg64; + + /** PCI interrupt enable register */ + u8 __iomem *intr_enb_reg64; + + /** The PCI interrupt mask used by interrupt handler */ + u64 intr_mask64; + + struct octeon_config *conf; + + /* Example additional fields - not used currently + * struct { + * }cn6xyz; + */ + + /* For the purpose of atomic access to interrupt enable reg */ + spinlock_t lock_for_droq_int_enb_reg; + +}; + +enum octeon_pcie_mps { + PCIE_MPS_DEFAULT = -1, /* Use the default setup by BIOS */ + PCIE_MPS_128B = 0, + PCIE_MPS_256B = 1 +}; + +enum octeon_pcie_mrrs { + PCIE_MRRS_DEFAULT = -1, /* Use the default setup by BIOS */ + PCIE_MRRS_128B = 0, + PCIE_MRRS_256B = 1, + PCIE_MRRS_512B = 2, + PCIE_MRRS_1024B = 3, + PCIE_MRRS_2048B = 4, + PCIE_MRRS_4096B = 5 +}; + +/* Common functions for 66xx and 68xx */ +int lio_cn6xxx_soft_reset(struct octeon_device *oct); +void lio_cn6xxx_enable_error_reporting(struct octeon_device *oct); +void lio_cn6xxx_setup_pcie_mps(struct octeon_device *oct, + enum octeon_pcie_mps mps); +void lio_cn6xxx_setup_pcie_mrrs(struct octeon_device *oct, + enum octeon_pcie_mrrs mrrs); +void lio_cn6xxx_setup_global_input_regs(struct octeon_device *oct); +void lio_cn6xxx_setup_global_output_regs(struct octeon_device *oct); +void lio_cn6xxx_setup_iq_regs(struct octeon_device *oct, u32 iq_no); +void lio_cn6xxx_setup_oq_regs(struct octeon_device *oct, u32 oq_no); +void lio_cn6xxx_enable_io_queues(struct octeon_device *oct); +void lio_cn6xxx_disable_io_queues(struct octeon_device *oct); +void lio_cn6xxx_process_pcie_error_intr(struct octeon_device *oct, u64 intr64); +int lio_cn6xxx_process_droq_intr_regs(struct octeon_device *oct); +irqreturn_t lio_cn6xxx_process_interrupt_regs(void *dev); +void lio_cn6xxx_reinit_regs(struct octeon_device *oct); +void lio_cn6xxx_bar1_idx_setup(struct octeon_device *oct, u64 core_addr, + u32 idx, int valid); +void lio_cn6xxx_bar1_idx_write(struct octeon_device *oct, u32 idx, u32 mask); +u32 lio_cn6xxx_bar1_idx_read(struct octeon_device *oct, u32 idx); +u32 +lio_cn6xxx_update_read_index(struct octeon_device *oct __attribute__((unused)), + struct octeon_instr_queue *iq); +void lio_cn6xxx_enable_interrupt(void *chip); +void lio_cn6xxx_disable_interrupt(void *chip); +void cn6xxx_get_pcie_qlmport(struct octeon_device *oct); +void lio_cn6xxx_setup_reg_address(struct octeon_device *oct, void *chip, + struct octeon_reg_list *reg_list); +u32 lio_cn6xxx_coprocessor_clock(struct octeon_device *oct); +u32 lio_cn6xxx_get_oq_ticks(struct octeon_device *oct, u32 time_intr_in_us); +int lio_setup_cn66xx_octeon_device(struct octeon_device *); +int lio_validate_cn6xxx_config_info(struct octeon_device *oct, + struct octeon_config *); + +#endif diff --git a/kernel/drivers/net/ethernet/cavium/liquidio/cn66xx_regs.h b/kernel/drivers/net/ethernet/cavium/liquidio/cn66xx_regs.h new file mode 100644 index 000000000..5e3aff242 --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/liquidio/cn66xx_regs.h @@ -0,0 +1,535 @@ +/********************************************************************** +* Author: Cavium, Inc. +* +* Contact: support@cavium.com +* Please include "LiquidIO" in the subject. +* +* Copyright (c) 2003-2015 Cavium, Inc. +* +* This file is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License, Version 2, as +* published by the Free Software Foundation. +* +* This file is distributed in the hope that it will be useful, but +* AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty +* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or +* NONINFRINGEMENT. See the GNU General Public License for more +* details. +* +* This file may also be available under a different license from Cavium. +* Contact Cavium, Inc. for more information +**********************************************************************/ + +/*! \file cn66xx_regs.h + * \brief Host Driver: Register Address and Register Mask values for + * Octeon CN66XX devices. + */ + +#ifndef __CN66XX_REGS_H__ +#define __CN66XX_REGS_H__ + +#define CN6XXX_XPANSION_BAR 0x30 + +#define CN6XXX_MSI_CAP 0x50 +#define CN6XXX_MSI_ADDR_LO 0x54 +#define CN6XXX_MSI_ADDR_HI 0x58 +#define CN6XXX_MSI_DATA 0x5C + +#define CN6XXX_PCIE_CAP 0x70 +#define CN6XXX_PCIE_DEVCAP 0x74 +#define CN6XXX_PCIE_DEVCTL 0x78 +#define CN6XXX_PCIE_LINKCAP 0x7C +#define CN6XXX_PCIE_LINKCTL 0x80 +#define CN6XXX_PCIE_SLOTCAP 0x84 +#define CN6XXX_PCIE_SLOTCTL 0x88 + +#define CN6XXX_PCIE_ENH_CAP 0x100 +#define CN6XXX_PCIE_UNCORR_ERR_STATUS 0x104 +#define CN6XXX_PCIE_UNCORR_ERR_MASK 0x108 +#define CN6XXX_PCIE_UNCORR_ERR 0x10C +#define CN6XXX_PCIE_CORR_ERR_STATUS 0x110 +#define CN6XXX_PCIE_CORR_ERR_MASK 0x114 +#define CN6XXX_PCIE_ADV_ERR_CAP 0x118 + +#define CN6XXX_PCIE_ACK_REPLAY_TIMER 0x700 +#define CN6XXX_PCIE_OTHER_MSG 0x704 +#define CN6XXX_PCIE_PORT_FORCE_LINK 0x708 +#define CN6XXX_PCIE_ACK_FREQ 0x70C +#define CN6XXX_PCIE_PORT_LINK_CTL 0x710 +#define CN6XXX_PCIE_LANE_SKEW 0x714 +#define CN6XXX_PCIE_SYM_NUM 0x718 +#define CN6XXX_PCIE_FLTMSK 0x720 + +/* ############## BAR0 Registers ################ */ + +#define CN6XXX_SLI_CTL_PORT0 0x0050 +#define CN6XXX_SLI_CTL_PORT1 0x0060 + +#define CN6XXX_SLI_WINDOW_CTL 0x02E0 +#define CN6XXX_SLI_DBG_DATA 0x0310 +#define CN6XXX_SLI_SCRATCH1 0x03C0 +#define CN6XXX_SLI_SCRATCH2 0x03D0 +#define CN6XXX_SLI_CTL_STATUS 0x0570 + +#define CN6XXX_WIN_WR_ADDR_LO 0x0000 +#define CN6XXX_WIN_WR_ADDR_HI 0x0004 +#define CN6XXX_WIN_WR_ADDR64 CN6XXX_WIN_WR_ADDR_LO + +#define CN6XXX_WIN_RD_ADDR_LO 0x0010 +#define CN6XXX_WIN_RD_ADDR_HI 0x0014 +#define CN6XXX_WIN_RD_ADDR64 CN6XXX_WIN_RD_ADDR_LO + +#define CN6XXX_WIN_WR_DATA_LO 0x0020 +#define CN6XXX_WIN_WR_DATA_HI 0x0024 +#define CN6XXX_WIN_WR_DATA64 CN6XXX_WIN_WR_DATA_LO + +#define CN6XXX_WIN_RD_DATA_LO 0x0040 +#define CN6XXX_WIN_RD_DATA_HI 0x0044 +#define CN6XXX_WIN_RD_DATA64 CN6XXX_WIN_RD_DATA_LO + +#define CN6XXX_WIN_WR_MASK_LO 0x0030 +#define CN6XXX_WIN_WR_MASK_HI 0x0034 +#define CN6XXX_WIN_WR_MASK_REG CN6XXX_WIN_WR_MASK_LO + +/* 1 register (32-bit) to enable Input queues */ +#define CN6XXX_SLI_PKT_INSTR_ENB 0x1000 + +/* 1 register (32-bit) to enable Output queues */ +#define CN6XXX_SLI_PKT_OUT_ENB 0x1010 + +/* 1 register (32-bit) to determine whether Output queues are in reset. */ +#define CN6XXX_SLI_PORT_IN_RST_OQ 0x11F0 + +/* 1 register (32-bit) to determine whether Input queues are in reset. */ +#define CN6XXX_SLI_PORT_IN_RST_IQ 0x11F4 + +/*###################### REQUEST QUEUE #########################*/ + +/* 1 register (32-bit) - instr. size of each input queue. */ +#define CN6XXX_SLI_PKT_INSTR_SIZE 0x1020 + +/* 32 registers for Input Queue Instr Count - SLI_PKT_IN_DONE0_CNTS */ +#define CN6XXX_SLI_IQ_INSTR_COUNT_START 0x2000 + +/* 32 registers for Input Queue Start Addr - SLI_PKT0_INSTR_BADDR */ +#define CN6XXX_SLI_IQ_BASE_ADDR_START64 0x2800 + +/* 32 registers for Input Doorbell - SLI_PKT0_INSTR_BAOFF_DBELL */ +#define CN6XXX_SLI_IQ_DOORBELL_START 0x2C00 + +/* 32 registers for Input Queue size - SLI_PKT0_INSTR_FIFO_RSIZE */ +#define CN6XXX_SLI_IQ_SIZE_START 0x3000 + +/* 32 registers for Instruction Header Options - SLI_PKT0_INSTR_HEADER */ +#define CN6XXX_SLI_IQ_PKT_INSTR_HDR_START64 0x3400 + +/* 1 register (64-bit) - Back Pressure for each input queue - SLI_PKT0_IN_BP */ +#define CN66XX_SLI_INPUT_BP_START64 0x3800 + +/* Each Input Queue register is at a 16-byte Offset in BAR0 */ +#define CN6XXX_IQ_OFFSET 0x10 + +/* 1 register (32-bit) - ES, RO, NS, Arbitration for Input Queue Data & + * gather list fetches. SLI_PKT_INPUT_CONTROL. + */ +#define CN6XXX_SLI_PKT_INPUT_CONTROL 0x1170 + +/* 1 register (64-bit) - Number of instructions to read at one time + * - 2 bits for each input ring. SLI_PKT_INSTR_RD_SIZE. + */ +#define CN6XXX_SLI_PKT_INSTR_RD_SIZE 0x11A0 + +/* 1 register (64-bit) - Assign Input ring to MAC port + * - 2 bits for each input ring. SLI_PKT_IN_PCIE_PORT. + */ +#define CN6XXX_SLI_IN_PCIE_PORT 0x11B0 + +/*------- Request Queue Macros ---------*/ +#define CN6XXX_SLI_IQ_BASE_ADDR64(iq) \ + (CN6XXX_SLI_IQ_BASE_ADDR_START64 + ((iq) * CN6XXX_IQ_OFFSET)) + +#define CN6XXX_SLI_IQ_SIZE(iq) \ + (CN6XXX_SLI_IQ_SIZE_START + ((iq) * CN6XXX_IQ_OFFSET)) + +#define CN6XXX_SLI_IQ_PKT_INSTR_HDR64(iq) \ + (CN6XXX_SLI_IQ_PKT_INSTR_HDR_START64 + ((iq) * CN6XXX_IQ_OFFSET)) + +#define CN6XXX_SLI_IQ_DOORBELL(iq) \ + (CN6XXX_SLI_IQ_DOORBELL_START + ((iq) * CN6XXX_IQ_OFFSET)) + +#define CN6XXX_SLI_IQ_INSTR_COUNT(iq) \ + (CN6XXX_SLI_IQ_INSTR_COUNT_START + ((iq) * CN6XXX_IQ_OFFSET)) + +#define CN66XX_SLI_IQ_BP64(iq) \ + (CN66XX_SLI_INPUT_BP_START64 + ((iq) * CN6XXX_IQ_OFFSET)) + +/*------------------ Masks ----------------*/ +#define CN6XXX_INPUT_CTL_ROUND_ROBIN_ARB BIT(22) +#define CN6XXX_INPUT_CTL_DATA_NS BIT(8) +#define CN6XXX_INPUT_CTL_DATA_ES_64B_SWAP BIT(6) +#define CN6XXX_INPUT_CTL_DATA_RO BIT(5) +#define CN6XXX_INPUT_CTL_USE_CSR BIT(4) +#define CN6XXX_INPUT_CTL_GATHER_NS BIT(3) +#define CN6XXX_INPUT_CTL_GATHER_ES_64B_SWAP BIT(2) +#define CN6XXX_INPUT_CTL_GATHER_RO BIT(1) + +#ifdef __BIG_ENDIAN_BITFIELD +#define CN6XXX_INPUT_CTL_MASK \ + (CN6XXX_INPUT_CTL_DATA_ES_64B_SWAP \ + | CN6XXX_INPUT_CTL_USE_CSR \ + | CN6XXX_INPUT_CTL_GATHER_ES_64B_SWAP) +#else +#define CN6XXX_INPUT_CTL_MASK \ + (CN6XXX_INPUT_CTL_DATA_ES_64B_SWAP \ + | CN6XXX_INPUT_CTL_USE_CSR) +#endif + +/*############################ OUTPUT QUEUE #########################*/ + +/* 32 registers for Output queue buffer and info size - SLI_PKT0_OUT_SIZE */ +#define CN6XXX_SLI_OQ0_BUFF_INFO_SIZE 0x0C00 + +/* 32 registers for Output Queue Start Addr - SLI_PKT0_SLIST_BADDR */ +#define CN6XXX_SLI_OQ_BASE_ADDR_START64 0x1400 + +/* 32 registers for Output Queue Packet Credits - SLI_PKT0_SLIST_BAOFF_DBELL */ +#define CN6XXX_SLI_OQ_PKT_CREDITS_START 0x1800 + +/* 32 registers for Output Queue size - SLI_PKT0_SLIST_FIFO_RSIZE */ +#define CN6XXX_SLI_OQ_SIZE_START 0x1C00 + +/* 32 registers for Output Queue Packet Count - SLI_PKT0_CNTS */ +#define CN6XXX_SLI_OQ_PKT_SENT_START 0x2400 + +/* Each Output Queue register is at a 16-byte Offset in BAR0 */ +#define CN6XXX_OQ_OFFSET 0x10 + +/* 1 register (32-bit) - 1 bit for each output queue + * - Relaxed Ordering setting for reading Output Queues descriptors + * - SLI_PKT_SLIST_ROR + */ +#define CN6XXX_SLI_PKT_SLIST_ROR 0x1030 + +/* 1 register (32-bit) - 1 bit for each output queue + * - No Snoop mode for reading Output Queues descriptors + * - SLI_PKT_SLIST_NS + */ +#define CN6XXX_SLI_PKT_SLIST_NS 0x1040 + +/* 1 register (64-bit) - 2 bits for each output queue + * - Endian-Swap mode for reading Output Queue descriptors + * - SLI_PKT_SLIST_ES + */ +#define CN6XXX_SLI_PKT_SLIST_ES64 0x1050 + +/* 1 register (32-bit) - 1 bit for each output queue + * - InfoPtr mode for Output Queues. + * - SLI_PKT_IPTR + */ +#define CN6XXX_SLI_PKT_IPTR 0x1070 + +/* 1 register (32-bit) - 1 bit for each output queue + * - DPTR format selector for Output queues. + * - SLI_PKT_DPADDR + */ +#define CN6XXX_SLI_PKT_DPADDR 0x1080 + +/* 1 register (32-bit) - 1 bit for each output queue + * - Relaxed Ordering setting for reading Output Queues data + * - SLI_PKT_DATA_OUT_ROR + */ +#define CN6XXX_SLI_PKT_DATA_OUT_ROR 0x1090 + +/* 1 register (32-bit) - 1 bit for each output queue + * - No Snoop mode for reading Output Queues data + * - SLI_PKT_DATA_OUT_NS + */ +#define CN6XXX_SLI_PKT_DATA_OUT_NS 0x10A0 + +/* 1 register (64-bit) - 2 bits for each output queue + * - Endian-Swap mode for reading Output Queue data + * - SLI_PKT_DATA_OUT_ES + */ +#define CN6XXX_SLI_PKT_DATA_OUT_ES64 0x10B0 + +/* 1 register (32-bit) - 1 bit for each output queue + * - Controls whether SLI_PKTn_CNTS is incremented for bytes or for packets. + * - SLI_PKT_OUT_BMODE + */ +#define CN6XXX_SLI_PKT_OUT_BMODE 0x10D0 + +/* 1 register (64-bit) - 2 bits for each output queue + * - Assign PCIE port for Output queues + * - SLI_PKT_PCIE_PORT. + */ +#define CN6XXX_SLI_PKT_PCIE_PORT64 0x10E0 + +/* 1 (64-bit) register for Output Queue Packet Count Interrupt Threshold + * & Time Threshold. The same setting applies to all 32 queues. + * The register is defined as a 64-bit registers, but we use the + * 32-bit offsets to define distinct addresses. + */ +#define CN6XXX_SLI_OQ_INT_LEVEL_PKTS 0x1120 +#define CN6XXX_SLI_OQ_INT_LEVEL_TIME 0x1124 + +/* 1 (64-bit register) for Output Queue backpressure across all rings. */ +#define CN6XXX_SLI_OQ_WMARK 0x1180 + +/* 1 register to control output queue global backpressure & ring enable. */ +#define CN6XXX_SLI_PKT_CTL 0x1220 + +/*------- Output Queue Macros ---------*/ +#define CN6XXX_SLI_OQ_BASE_ADDR64(oq) \ + (CN6XXX_SLI_OQ_BASE_ADDR_START64 + ((oq) * CN6XXX_OQ_OFFSET)) + +#define CN6XXX_SLI_OQ_SIZE(oq) \ + (CN6XXX_SLI_OQ_SIZE_START + ((oq) * CN6XXX_OQ_OFFSET)) + +#define CN6XXX_SLI_OQ_BUFF_INFO_SIZE(oq) \ + (CN6XXX_SLI_OQ0_BUFF_INFO_SIZE + ((oq) * CN6XXX_OQ_OFFSET)) + +#define CN6XXX_SLI_OQ_PKTS_SENT(oq) \ + (CN6XXX_SLI_OQ_PKT_SENT_START + ((oq) * CN6XXX_OQ_OFFSET)) + +#define CN6XXX_SLI_OQ_PKTS_CREDIT(oq) \ + (CN6XXX_SLI_OQ_PKT_CREDITS_START + ((oq) * CN6XXX_OQ_OFFSET)) + +/*######################### DMA Counters #########################*/ + +/* 2 registers (64-bit) - DMA Count - 1 for each DMA counter 0/1. */ +#define CN6XXX_DMA_CNT_START 0x0400 + +/* 2 registers (64-bit) - DMA Timer 0/1, contains DMA timer values + * SLI_DMA_0_TIM + */ +#define CN6XXX_DMA_TIM_START 0x0420 + +/* 2 registers (64-bit) - DMA count & Time Interrupt threshold - + * SLI_DMA_0_INT_LEVEL + */ +#define CN6XXX_DMA_INT_LEVEL_START 0x03E0 + +/* Each DMA register is at a 16-byte Offset in BAR0 */ +#define CN6XXX_DMA_OFFSET 0x10 + +/*---------- DMA Counter Macros ---------*/ +#define CN6XXX_DMA_CNT(dq) \ + (CN6XXX_DMA_CNT_START + ((dq) * CN6XXX_DMA_OFFSET)) + +#define CN6XXX_DMA_INT_LEVEL(dq) \ + (CN6XXX_DMA_INT_LEVEL_START + ((dq) * CN6XXX_DMA_OFFSET)) + +#define CN6XXX_DMA_PKT_INT_LEVEL(dq) \ + (CN6XXX_DMA_INT_LEVEL_START + ((dq) * CN6XXX_DMA_OFFSET)) + +#define CN6XXX_DMA_TIME_INT_LEVEL(dq) \ + (CN6XXX_DMA_INT_LEVEL_START + 4 + ((dq) * CN6XXX_DMA_OFFSET)) + +#define CN6XXX_DMA_TIM(dq) \ + (CN6XXX_DMA_TIM_START + ((dq) * CN6XXX_DMA_OFFSET)) + +/*######################## INTERRUPTS #########################*/ + +/* 1 register (64-bit) for Interrupt Summary */ +#define CN6XXX_SLI_INT_SUM64 0x0330 + +/* 1 register (64-bit) for Interrupt Enable */ +#define CN6XXX_SLI_INT_ENB64_PORT0 0x0340 +#define CN6XXX_SLI_INT_ENB64_PORT1 0x0350 + +/* 1 register (32-bit) to enable Output Queue Packet/Byte Count Interrupt */ +#define CN6XXX_SLI_PKT_CNT_INT_ENB 0x1150 + +/* 1 register (32-bit) to enable Output Queue Packet Timer Interrupt */ +#define CN6XXX_SLI_PKT_TIME_INT_ENB 0x1160 + +/* 1 register (32-bit) to indicate which Output Queue reached pkt threshold */ +#define CN6XXX_SLI_PKT_CNT_INT 0x1130 + +/* 1 register (32-bit) to indicate which Output Queue reached time threshold */ +#define CN6XXX_SLI_PKT_TIME_INT 0x1140 + +/*------------------ Interrupt Masks ----------------*/ + +#define CN6XXX_INTR_RML_TIMEOUT_ERR BIT(1) +#define CN6XXX_INTR_BAR0_RW_TIMEOUT_ERR BIT(2) +#define CN6XXX_INTR_IO2BIG_ERR BIT(3) +#define CN6XXX_INTR_PKT_COUNT BIT(4) +#define CN6XXX_INTR_PKT_TIME BIT(5) +#define CN6XXX_INTR_M0UPB0_ERR BIT(8) +#define CN6XXX_INTR_M0UPWI_ERR BIT(9) +#define CN6XXX_INTR_M0UNB0_ERR BIT(10) +#define CN6XXX_INTR_M0UNWI_ERR BIT(11) +#define CN6XXX_INTR_M1UPB0_ERR BIT(12) +#define CN6XXX_INTR_M1UPWI_ERR BIT(13) +#define CN6XXX_INTR_M1UNB0_ERR BIT(14) +#define CN6XXX_INTR_M1UNWI_ERR BIT(15) +#define CN6XXX_INTR_MIO_INT0 BIT(16) +#define CN6XXX_INTR_MIO_INT1 BIT(17) +#define CN6XXX_INTR_MAC_INT0 BIT(18) +#define CN6XXX_INTR_MAC_INT1 BIT(19) + +#define CN6XXX_INTR_DMA0_FORCE BIT_ULL(32) +#define CN6XXX_INTR_DMA1_FORCE BIT_ULL(33) +#define CN6XXX_INTR_DMA0_COUNT BIT_ULL(34) +#define CN6XXX_INTR_DMA1_COUNT BIT_ULL(35) +#define CN6XXX_INTR_DMA0_TIME BIT_ULL(36) +#define CN6XXX_INTR_DMA1_TIME BIT_ULL(37) +#define CN6XXX_INTR_INSTR_DB_OF_ERR BIT_ULL(48) +#define CN6XXX_INTR_SLIST_DB_OF_ERR BIT_ULL(49) +#define CN6XXX_INTR_POUT_ERR BIT_ULL(50) +#define CN6XXX_INTR_PIN_BP_ERR BIT_ULL(51) +#define CN6XXX_INTR_PGL_ERR BIT_ULL(52) +#define CN6XXX_INTR_PDI_ERR BIT_ULL(53) +#define CN6XXX_INTR_POP_ERR BIT_ULL(54) +#define CN6XXX_INTR_PINS_ERR BIT_ULL(55) +#define CN6XXX_INTR_SPRT0_ERR BIT_ULL(56) +#define CN6XXX_INTR_SPRT1_ERR BIT_ULL(57) +#define CN6XXX_INTR_ILL_PAD_ERR BIT_ULL(60) + +#define CN6XXX_INTR_DMA0_DATA (CN6XXX_INTR_DMA0_TIME) + +#define CN6XXX_INTR_DMA1_DATA (CN6XXX_INTR_DMA1_TIME) + +#define CN6XXX_INTR_DMA_DATA \ + (CN6XXX_INTR_DMA0_DATA | CN6XXX_INTR_DMA1_DATA) + +#define CN6XXX_INTR_PKT_DATA (CN6XXX_INTR_PKT_TIME | \ + CN6XXX_INTR_PKT_COUNT) + +/* Sum of interrupts for all PCI-Express Data Interrupts */ +#define CN6XXX_INTR_PCIE_DATA \ + (CN6XXX_INTR_DMA_DATA | CN6XXX_INTR_PKT_DATA) + +#define CN6XXX_INTR_MIO \ + (CN6XXX_INTR_MIO_INT0 | CN6XXX_INTR_MIO_INT1) + +#define CN6XXX_INTR_MAC \ + (CN6XXX_INTR_MAC_INT0 | CN6XXX_INTR_MAC_INT1) + +/* Sum of interrupts for error events */ +#define CN6XXX_INTR_ERR \ + (CN6XXX_INTR_BAR0_RW_TIMEOUT_ERR \ + | CN6XXX_INTR_IO2BIG_ERR \ + | CN6XXX_INTR_M0UPB0_ERR \ + | CN6XXX_INTR_M0UPWI_ERR \ + | CN6XXX_INTR_M0UNB0_ERR \ + | CN6XXX_INTR_M0UNWI_ERR \ + | CN6XXX_INTR_M1UPB0_ERR \ + | CN6XXX_INTR_M1UPWI_ERR \ + | CN6XXX_INTR_M1UPB0_ERR \ + | CN6XXX_INTR_M1UNWI_ERR \ + | CN6XXX_INTR_INSTR_DB_OF_ERR \ + | CN6XXX_INTR_SLIST_DB_OF_ERR \ + | CN6XXX_INTR_POUT_ERR \ + | CN6XXX_INTR_PIN_BP_ERR \ + | CN6XXX_INTR_PGL_ERR \ + | CN6XXX_INTR_PDI_ERR \ + | CN6XXX_INTR_POP_ERR \ + | CN6XXX_INTR_PINS_ERR \ + | CN6XXX_INTR_SPRT0_ERR \ + | CN6XXX_INTR_SPRT1_ERR \ + | CN6XXX_INTR_ILL_PAD_ERR) + +/* Programmed Mask for Interrupt Sum */ +#define CN6XXX_INTR_MASK \ + (CN6XXX_INTR_PCIE_DATA \ + | CN6XXX_INTR_DMA0_FORCE \ + | CN6XXX_INTR_DMA1_FORCE \ + | CN6XXX_INTR_MIO \ + | CN6XXX_INTR_MAC \ + | CN6XXX_INTR_ERR) + +#define CN6XXX_SLI_S2M_PORT0_CTL 0x3D80 +#define CN6XXX_SLI_S2M_PORT1_CTL 0x3D90 +#define CN6XXX_SLI_S2M_PORTX_CTL(port) \ + (CN6XXX_SLI_S2M_PORT0_CTL + (port * 0x10)) + +#define CN6XXX_SLI_INT_ENB64(port) \ + (CN6XXX_SLI_INT_ENB64_PORT0 + (port * 0x10)) + +#define CN6XXX_SLI_MAC_NUMBER 0x3E00 + +/* CN6XXX BAR1 Index registers. */ +#define CN6XXX_PEM_BAR1_INDEX000 0x00011800C00000A8ULL +#define CN6XXX_PEM_OFFSET 0x0000000001000000ULL + +#define CN6XXX_BAR1_INDEX_START CN6XXX_PEM_BAR1_INDEX000 +#define CN6XXX_PCI_BAR1_OFFSET 0x8 + +#define CN6XXX_BAR1_REG(idx, port) \ + (CN6XXX_BAR1_INDEX_START + (port * CN6XXX_PEM_OFFSET) + \ + (CN6XXX_PCI_BAR1_OFFSET * (idx))) + +/*############################ DPI #########################*/ + +#define CN6XXX_DPI_CTL 0x0001df0000000040ULL + +#define CN6XXX_DPI_DMA_CONTROL 0x0001df0000000048ULL + +#define CN6XXX_DPI_REQ_GBL_ENB 0x0001df0000000050ULL + +#define CN6XXX_DPI_REQ_ERR_RSP 0x0001df0000000058ULL + +#define CN6XXX_DPI_REQ_ERR_RST 0x0001df0000000060ULL + +#define CN6XXX_DPI_DMA_ENG0_ENB 0x0001df0000000080ULL + +#define CN6XXX_DPI_DMA_ENG_ENB(q_no) \ + (CN6XXX_DPI_DMA_ENG0_ENB + (q_no * 8)) + +#define CN6XXX_DPI_DMA_ENG0_BUF 0x0001df0000000880ULL + +#define CN6XXX_DPI_DMA_ENG_BUF(q_no) \ + (CN6XXX_DPI_DMA_ENG0_BUF + (q_no * 8)) + +#define CN6XXX_DPI_SLI_PRT0_CFG 0x0001df0000000900ULL +#define CN6XXX_DPI_SLI_PRT1_CFG 0x0001df0000000908ULL +#define CN6XXX_DPI_SLI_PRTX_CFG(port) \ + (CN6XXX_DPI_SLI_PRT0_CFG + (port * 0x10)) + +#define CN6XXX_DPI_DMA_COMMIT_MODE BIT_ULL(58) +#define CN6XXX_DPI_DMA_PKT_HP BIT_ULL(57) +#define CN6XXX_DPI_DMA_PKT_EN BIT_ULL(56) +#define CN6XXX_DPI_DMA_O_ES BIT_ULL(15) +#define CN6XXX_DPI_DMA_O_MODE BIT_ULL(14) + +#define CN6XXX_DPI_DMA_CTL_MASK \ + (CN6XXX_DPI_DMA_COMMIT_MODE | \ + CN6XXX_DPI_DMA_PKT_HP | \ + CN6XXX_DPI_DMA_PKT_EN | \ + CN6XXX_DPI_DMA_O_ES | \ + CN6XXX_DPI_DMA_O_MODE) + +/*############################ CIU #########################*/ + +#define CN6XXX_CIU_SOFT_BIST 0x0001070000000738ULL +#define CN6XXX_CIU_SOFT_RST 0x0001070000000740ULL + +/*############################ MIO #########################*/ +#define CN6XXX_MIO_PTP_CLOCK_CFG 0x0001070000000f00ULL +#define CN6XXX_MIO_PTP_CLOCK_LO 0x0001070000000f08ULL +#define CN6XXX_MIO_PTP_CLOCK_HI 0x0001070000000f10ULL +#define CN6XXX_MIO_PTP_CLOCK_COMP 0x0001070000000f18ULL +#define CN6XXX_MIO_PTP_TIMESTAMP 0x0001070000000f20ULL +#define CN6XXX_MIO_PTP_EVT_CNT 0x0001070000000f28ULL +#define CN6XXX_MIO_PTP_CKOUT_THRESH_LO 0x0001070000000f30ULL +#define CN6XXX_MIO_PTP_CKOUT_THRESH_HI 0x0001070000000f38ULL +#define CN6XXX_MIO_PTP_CKOUT_HI_INCR 0x0001070000000f40ULL +#define CN6XXX_MIO_PTP_CKOUT_LO_INCR 0x0001070000000f48ULL +#define CN6XXX_MIO_PTP_PPS_THRESH_LO 0x0001070000000f50ULL +#define CN6XXX_MIO_PTP_PPS_THRESH_HI 0x0001070000000f58ULL +#define CN6XXX_MIO_PTP_PPS_HI_INCR 0x0001070000000f60ULL +#define CN6XXX_MIO_PTP_PPS_LO_INCR 0x0001070000000f68ULL + +#define CN6XXX_MIO_QLM4_CFG 0x00011800000015B0ULL +#define CN6XXX_MIO_RST_BOOT 0x0001180000001600ULL + +#define CN6XXX_MIO_QLM_CFG_MASK 0x7 + +/*############################ LMC #########################*/ + +#define CN6XXX_LMC0_RESET_CTL 0x0001180088000180ULL +#define CN6XXX_LMC0_RESET_CTL_DDR3RST_MASK 0x0000000000000001ULL + +#endif diff --git a/kernel/drivers/net/ethernet/cavium/liquidio/cn68xx_device.c b/kernel/drivers/net/ethernet/cavium/liquidio/cn68xx_device.c new file mode 100644 index 000000000..8e830d0c0 --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/liquidio/cn68xx_device.c @@ -0,0 +1,198 @@ +/********************************************************************** +* Author: Cavium, Inc. +* +* Contact: support@cavium.com +* Please include "LiquidIO" in the subject. +* +* Copyright (c) 2003-2015 Cavium, Inc. +* +* This file is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License, Version 2, as +* published by the Free Software Foundation. +* +* This file is distributed in the hope that it will be useful, but +* AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty +* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or +* NONINFRINGEMENT. See the GNU General Public License for more +* details. +* +* This file may also be available under a different license from Cavium. +* Contact Cavium, Inc. for more information +**********************************************************************/ +#include <linux/version.h> +#include <linux/types.h> +#include <linux/list.h> +#include <linux/interrupt.h> +#include <linux/pci.h> +#include <linux/kthread.h> +#include <linux/netdevice.h> +#include "octeon_config.h" +#include "liquidio_common.h" +#include "octeon_droq.h" +#include "octeon_iq.h" +#include "response_manager.h" +#include "octeon_device.h" +#include "octeon_nic.h" +#include "octeon_main.h" +#include "octeon_network.h" +#include "cn66xx_regs.h" +#include "cn66xx_device.h" +#include "cn68xx_regs.h" +#include "cn68xx_device.h" +#include "liquidio_image.h" +#include "octeon_mem_ops.h" + +static void lio_cn68xx_set_dpi_regs(struct octeon_device *oct) +{ + u32 i; + u32 fifo_sizes[6] = { 3, 3, 1, 1, 1, 8 }; + + lio_pci_writeq(oct, CN6XXX_DPI_DMA_CTL_MASK, CN6XXX_DPI_DMA_CONTROL); + dev_dbg(&oct->pci_dev->dev, "DPI_DMA_CONTROL: 0x%016llx\n", + lio_pci_readq(oct, CN6XXX_DPI_DMA_CONTROL)); + + for (i = 0; i < 6; i++) { + /* Prevent service of instruction queue for all DMA engines + * Engine 5 will remain 0. Engines 0 - 4 will be setup by + * core. + */ + lio_pci_writeq(oct, 0, CN6XXX_DPI_DMA_ENG_ENB(i)); + lio_pci_writeq(oct, fifo_sizes[i], CN6XXX_DPI_DMA_ENG_BUF(i)); + dev_dbg(&oct->pci_dev->dev, "DPI_ENG_BUF%d: 0x%016llx\n", i, + lio_pci_readq(oct, CN6XXX_DPI_DMA_ENG_BUF(i))); + } + + /* DPI_SLI_PRT_CFG has MPS and MRRS settings that will be set + * separately. + */ + + lio_pci_writeq(oct, 1, CN6XXX_DPI_CTL); + dev_dbg(&oct->pci_dev->dev, "DPI_CTL: 0x%016llx\n", + lio_pci_readq(oct, CN6XXX_DPI_CTL)); +} + +static int lio_cn68xx_soft_reset(struct octeon_device *oct) +{ + lio_cn6xxx_soft_reset(oct); + lio_cn68xx_set_dpi_regs(oct); + + return 0; +} + +static void lio_cn68xx_setup_pkt_ctl_regs(struct octeon_device *oct) +{ + struct octeon_cn6xxx *cn68xx = (struct octeon_cn6xxx *)oct->chip; + u64 pktctl, tx_pipe, max_oqs; + + pktctl = octeon_read_csr64(oct, CN6XXX_SLI_PKT_CTL); + + /* 68XX specific */ + max_oqs = CFG_GET_OQ_MAX_Q(CHIP_FIELD(oct, cn6xxx, conf)); + tx_pipe = octeon_read_csr64(oct, CN68XX_SLI_TX_PIPE); + tx_pipe &= 0xffffffffff00ffffULL; /* clear out NUMP field */ + tx_pipe |= max_oqs << 16; /* put max_oqs in NUMP field */ + octeon_write_csr64(oct, CN68XX_SLI_TX_PIPE, tx_pipe); + + if (CFG_GET_IS_SLI_BP_ON(cn68xx->conf)) + pktctl |= 0xF; + else + /* Disable per-port backpressure. */ + pktctl &= ~0xF; + octeon_write_csr64(oct, CN6XXX_SLI_PKT_CTL, pktctl); +} + +static int lio_cn68xx_setup_device_regs(struct octeon_device *oct) +{ + lio_cn6xxx_setup_pcie_mps(oct, PCIE_MPS_DEFAULT); + lio_cn6xxx_setup_pcie_mrrs(oct, PCIE_MRRS_256B); + lio_cn6xxx_enable_error_reporting(oct); + + lio_cn6xxx_setup_global_input_regs(oct); + lio_cn68xx_setup_pkt_ctl_regs(oct); + lio_cn6xxx_setup_global_output_regs(oct); + + /* Default error timeout value should be 0x200000 to avoid host hang + * when reads invalid register + */ + octeon_write_csr64(oct, CN6XXX_SLI_WINDOW_CTL, 0x200000ULL); + + return 0; +} + +static inline void lio_cn68xx_vendor_message_fix(struct octeon_device *oct) +{ + u32 val = 0; + + /* Set M_VEND1_DRP and M_VEND0_DRP bits */ + pci_read_config_dword(oct->pci_dev, CN6XXX_PCIE_FLTMSK, &val); + val |= 0x3; + pci_write_config_dword(oct->pci_dev, CN6XXX_PCIE_FLTMSK, val); +} + +int lio_is_210nv(struct octeon_device *oct) +{ + u64 mio_qlm4_cfg = lio_pci_readq(oct, CN6XXX_MIO_QLM4_CFG); + + return ((mio_qlm4_cfg & CN6XXX_MIO_QLM_CFG_MASK) == 0); +} + +int lio_setup_cn68xx_octeon_device(struct octeon_device *oct) +{ + struct octeon_cn6xxx *cn68xx = (struct octeon_cn6xxx *)oct->chip; + u16 card_type = LIO_410NV; + + if (octeon_map_pci_barx(oct, 0, 0)) + return 1; + + if (octeon_map_pci_barx(oct, 1, MAX_BAR1_IOREMAP_SIZE)) { + dev_err(&oct->pci_dev->dev, "%s CN68XX BAR1 map failed\n", + __func__); + octeon_unmap_pci_barx(oct, 0); + return 1; + } + + spin_lock_init(&cn68xx->lock_for_droq_int_enb_reg); + + oct->fn_list.setup_iq_regs = lio_cn6xxx_setup_iq_regs; + oct->fn_list.setup_oq_regs = lio_cn6xxx_setup_oq_regs; + + oct->fn_list.process_interrupt_regs = lio_cn6xxx_process_interrupt_regs; + oct->fn_list.soft_reset = lio_cn68xx_soft_reset; + oct->fn_list.setup_device_regs = lio_cn68xx_setup_device_regs; + oct->fn_list.reinit_regs = lio_cn6xxx_reinit_regs; + oct->fn_list.update_iq_read_idx = lio_cn6xxx_update_read_index; + + oct->fn_list.bar1_idx_setup = lio_cn6xxx_bar1_idx_setup; + oct->fn_list.bar1_idx_write = lio_cn6xxx_bar1_idx_write; + oct->fn_list.bar1_idx_read = lio_cn6xxx_bar1_idx_read; + + oct->fn_list.enable_interrupt = lio_cn6xxx_enable_interrupt; + oct->fn_list.disable_interrupt = lio_cn6xxx_disable_interrupt; + + oct->fn_list.enable_io_queues = lio_cn6xxx_enable_io_queues; + oct->fn_list.disable_io_queues = lio_cn6xxx_disable_io_queues; + + lio_cn6xxx_setup_reg_address(oct, oct->chip, &oct->reg_list); + + /* Determine variant of card */ + if (lio_is_210nv(oct)) + card_type = LIO_210NV; + + cn68xx->conf = (struct octeon_config *) + oct_get_config_info(oct, card_type); + if (!cn68xx->conf) { + dev_err(&oct->pci_dev->dev, "%s No Config found for CN68XX %s\n", + __func__, + (card_type == LIO_410NV) ? LIO_410NV_NAME : + LIO_210NV_NAME); + octeon_unmap_pci_barx(oct, 0); + octeon_unmap_pci_barx(oct, 1); + return 1; + } + + oct->coproc_clock_rate = 1000000ULL * lio_cn6xxx_coprocessor_clock(oct); + + lio_cn68xx_vendor_message_fix(oct); + + return 0; +} diff --git a/kernel/drivers/net/ethernet/cavium/liquidio/cn68xx_device.h b/kernel/drivers/net/ethernet/cavium/liquidio/cn68xx_device.h new file mode 100644 index 000000000..d4e1c9fb0 --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/liquidio/cn68xx_device.h @@ -0,0 +1,33 @@ +/********************************************************************** +* Author: Cavium, Inc. +* +* Contact: support@cavium.com +* Please include "LiquidIO" in the subject. +* +* Copyright (c) 2003-2015 Cavium, Inc. +* +* This file is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License, Version 2, as +* published by the Free Software Foundation. +* +* This file is distributed in the hope that it will be useful, but +* AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty +* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or +* NONINFRINGEMENT. See the GNU General Public License for more +* details. +* +* This file may also be available under a different license from Cavium. +* Contact Cavium, Inc. for more information +**********************************************************************/ + +/*! \file cn68xx_device.h + * \brief Host Driver: Routines that perform CN68XX specific operations. + */ + +#ifndef __CN68XX_DEVICE_H__ +#define __CN68XX_DEVICE_H__ + +int lio_setup_cn68xx_octeon_device(struct octeon_device *oct); +int lio_is_210nv(struct octeon_device *oct); + +#endif diff --git a/kernel/drivers/net/ethernet/cavium/liquidio/cn68xx_regs.h b/kernel/drivers/net/ethernet/cavium/liquidio/cn68xx_regs.h new file mode 100644 index 000000000..38cddbd10 --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/liquidio/cn68xx_regs.h @@ -0,0 +1,51 @@ +/********************************************************************** +* Author: Cavium, Inc. +* +* Contact: support@cavium.com +* Please include "LiquidIO" in the subject. +* +* Copyright (c) 2003-2015 Cavium, Inc. +* +* This file is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License, Version 2, as +* published by the Free Software Foundation. +* +* This file is distributed in the hope that it will be useful, but +* AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty +* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or +* NONINFRINGEMENT. See the GNU General Public License for more +* details. +* +* This file may also be available under a different license from Cavium. +* Contact Cavium, Inc. for more information +**********************************************************************/ + +/*! \file cn68xx_regs.h + * \brief Host Driver: Register Address and Register Mask values for + * Octeon CN68XX devices. The register map for CN66XX is the same + * for most registers. This file has the other registers that are + * 68XX-specific. + */ + +#ifndef __CN68XX_REGS_H__ +#define __CN68XX_REGS_H__ +#include "cn66xx_regs.h" + +/*###################### REQUEST QUEUE #########################*/ + +#define CN68XX_SLI_IQ_PORT0_PKIND 0x0800 + +#define CN68XX_SLI_IQ_PORT_PKIND(iq) \ + (CN68XX_SLI_IQ_PORT0_PKIND + ((iq) * CN6XXX_IQ_OFFSET)) + +/*############################ OUTPUT QUEUE #########################*/ + +/* Starting pipe number and number of pipes used by the SLI packet output. */ +#define CN68XX_SLI_TX_PIPE 0x1230 + +/*######################## INTERRUPTS #########################*/ + +/*------------------ Interrupt Masks ----------------*/ +#define CN68XX_INTR_PIPE_ERR BIT_ULL(61) + +#endif diff --git a/kernel/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c b/kernel/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c new file mode 100644 index 000000000..245c063ed --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c @@ -0,0 +1,1216 @@ +/********************************************************************** +* Author: Cavium, Inc. +* +* Contact: support@cavium.com +* Please include "LiquidIO" in the subject. +* +* Copyright (c) 2003-2015 Cavium, Inc. +* +* This file is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License, Version 2, as +* published by the Free Software Foundation. +* +* This file is distributed in the hope that it will be useful, but +* AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty +* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or +* NONINFRINGEMENT. See the GNU General Public License for more +* details. +* +* This file may also be available under a different license from Cavium. +* Contact Cavium, Inc. for more information +**********************************************************************/ +#include <linux/version.h> +#include <linux/netdevice.h> +#include <linux/net_tstamp.h> +#include <linux/ethtool.h> +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include "octeon_config.h" +#include "liquidio_common.h" +#include "octeon_droq.h" +#include "octeon_iq.h" +#include "response_manager.h" +#include "octeon_device.h" +#include "octeon_nic.h" +#include "octeon_main.h" +#include "octeon_network.h" +#include "cn66xx_regs.h" +#include "cn66xx_device.h" +#include "cn68xx_regs.h" +#include "cn68xx_device.h" +#include "liquidio_image.h" + +struct oct_mdio_cmd_context { + int octeon_id; + wait_queue_head_t wc; + int cond; +}; + +struct oct_mdio_cmd_resp { + u64 rh; + struct oct_mdio_cmd resp; + u64 status; +}; + +#define OCT_MDIO45_RESP_SIZE (sizeof(struct oct_mdio_cmd_resp)) + +/* Octeon's interface mode of operation */ +enum { + INTERFACE_MODE_DISABLED, + INTERFACE_MODE_RGMII, + INTERFACE_MODE_GMII, + INTERFACE_MODE_SPI, + INTERFACE_MODE_PCIE, + INTERFACE_MODE_XAUI, + INTERFACE_MODE_SGMII, + INTERFACE_MODE_PICMG, + INTERFACE_MODE_NPI, + INTERFACE_MODE_LOOP, + INTERFACE_MODE_SRIO, + INTERFACE_MODE_ILK, + INTERFACE_MODE_RXAUI, + INTERFACE_MODE_QSGMII, + INTERFACE_MODE_AGL, +}; + +#define ARRAY_LENGTH(a) (sizeof(a) / sizeof((a)[0])) +#define OCT_ETHTOOL_REGDUMP_LEN 4096 +#define OCT_ETHTOOL_REGSVER 1 + +static const char oct_iq_stats_strings[][ETH_GSTRING_LEN] = { + "Instr posted", + "Instr processed", + "Instr dropped", + "Bytes Sent", + "Sgentry_sent", + "Inst cntreg", + "Tx done", + "Tx Iq busy", + "Tx dropped", + "Tx bytes", +}; + +static const char oct_droq_stats_strings[][ETH_GSTRING_LEN] = { + "OQ Pkts Received", + "OQ Bytes Received", + "Dropped no dispatch", + "Dropped nomem", + "Dropped toomany", + "Stack RX cnt", + "Stack RX Bytes", + "RX dropped", +}; + +#define OCTNIC_NCMD_AUTONEG_ON 0x1 +#define OCTNIC_NCMD_PHY_ON 0x2 + +static int lio_get_settings(struct net_device *netdev, struct ethtool_cmd *ecmd) +{ + struct lio *lio = GET_LIO(netdev); + struct octeon_device *oct = lio->oct_dev; + struct oct_link_info *linfo; + + linfo = &lio->linfo; + + if (linfo->link.s.interface == INTERFACE_MODE_XAUI || + linfo->link.s.interface == INTERFACE_MODE_RXAUI) { + ecmd->port = PORT_FIBRE; + ecmd->supported = + (SUPPORTED_10000baseT_Full | SUPPORTED_FIBRE | + SUPPORTED_Pause); + ecmd->advertising = + (ADVERTISED_10000baseT_Full | ADVERTISED_Pause); + ecmd->transceiver = XCVR_EXTERNAL; + ecmd->autoneg = AUTONEG_DISABLE; + + } else { + dev_err(&oct->pci_dev->dev, "Unknown link interface reported\n"); + } + + if (linfo->link.s.status) { + ethtool_cmd_speed_set(ecmd, linfo->link.s.speed); + ecmd->duplex = linfo->link.s.duplex; + } else { + ethtool_cmd_speed_set(ecmd, SPEED_UNKNOWN); + ecmd->duplex = DUPLEX_UNKNOWN; + } + + return 0; +} + +static void +lio_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *drvinfo) +{ + struct lio *lio; + struct octeon_device *oct; + + lio = GET_LIO(netdev); + oct = lio->oct_dev; + + memset(drvinfo, 0, sizeof(struct ethtool_drvinfo)); + strcpy(drvinfo->driver, "liquidio"); + strcpy(drvinfo->version, LIQUIDIO_VERSION); + strncpy(drvinfo->fw_version, oct->fw_info.liquidio_firmware_version, + ETHTOOL_FWVERS_LEN); + strncpy(drvinfo->bus_info, pci_name(oct->pci_dev), 32); +} + +static void +lio_ethtool_get_channels(struct net_device *dev, + struct ethtool_channels *channel) +{ + struct lio *lio = GET_LIO(dev); + struct octeon_device *oct = lio->oct_dev; + u32 max_rx = 0, max_tx = 0, tx_count = 0, rx_count = 0; + + if (OCTEON_CN6XXX(oct)) { + struct octeon_config *conf6x = CHIP_FIELD(oct, cn6xxx, conf); + + max_rx = CFG_GET_OQ_MAX_Q(conf6x); + max_tx = CFG_GET_IQ_MAX_Q(conf6x); + rx_count = CFG_GET_NUM_RXQS_NIC_IF(conf6x, lio->ifidx); + tx_count = CFG_GET_NUM_TXQS_NIC_IF(conf6x, lio->ifidx); + } + + channel->max_rx = max_rx; + channel->max_tx = max_tx; + channel->rx_count = rx_count; + channel->tx_count = tx_count; +} + +static int lio_get_eeprom_len(struct net_device *netdev) +{ + u8 buf[128]; + struct lio *lio = GET_LIO(netdev); + struct octeon_device *oct_dev = lio->oct_dev; + struct octeon_board_info *board_info; + int len; + + board_info = (struct octeon_board_info *)(&oct_dev->boardinfo); + len = sprintf(buf, "boardname:%s serialnum:%s maj:%lld min:%lld\n", + board_info->name, board_info->serial_number, + board_info->major, board_info->minor); + + return len; +} + +static int +lio_get_eeprom(struct net_device *netdev, struct ethtool_eeprom *eeprom, + u8 *bytes) +{ + struct lio *lio = GET_LIO(netdev); + struct octeon_device *oct_dev = lio->oct_dev; + struct octeon_board_info *board_info; + int len; + + if (eeprom->offset != 0) + return -EINVAL; + + eeprom->magic = oct_dev->pci_dev->vendor; + board_info = (struct octeon_board_info *)(&oct_dev->boardinfo); + len = + sprintf((char *)bytes, + "boardname:%s serialnum:%s maj:%lld min:%lld\n", + board_info->name, board_info->serial_number, + board_info->major, board_info->minor); + + return 0; +} + +static int octnet_gpio_access(struct net_device *netdev, int addr, int val) +{ + struct lio *lio = GET_LIO(netdev); + struct octeon_device *oct = lio->oct_dev; + struct octnic_ctrl_pkt nctrl; + struct octnic_ctrl_params nparams; + int ret = 0; + + memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt)); + + nctrl.ncmd.u64 = 0; + nctrl.ncmd.s.cmd = OCTNET_CMD_GPIO_ACCESS; + nctrl.ncmd.s.param1 = lio->linfo.ifidx; + nctrl.ncmd.s.param2 = addr; + nctrl.ncmd.s.param3 = val; + nctrl.wait_time = 100; + nctrl.netpndev = (u64)netdev; + nctrl.cb_fn = liquidio_link_ctrl_cmd_completion; + + nparams.resp_order = OCTEON_RESP_ORDERED; + + ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl, nparams); + if (ret < 0) { + dev_err(&oct->pci_dev->dev, "Failed to configure gpio value\n"); + return -EINVAL; + } + + return 0; +} + +/* Callback for when mdio command response arrives + */ +static void octnet_mdio_resp_callback(struct octeon_device *oct, + u32 status, + void *buf) +{ + struct oct_mdio_cmd_resp *mdio_cmd_rsp; + struct oct_mdio_cmd_context *mdio_cmd_ctx; + struct octeon_soft_command *sc = (struct octeon_soft_command *)buf; + + mdio_cmd_rsp = (struct oct_mdio_cmd_resp *)sc->virtrptr; + mdio_cmd_ctx = (struct oct_mdio_cmd_context *)sc->ctxptr; + + oct = lio_get_device(mdio_cmd_ctx->octeon_id); + if (status) { + dev_err(&oct->pci_dev->dev, "MIDO instruction failed. Status: %llx\n", + CVM_CAST64(status)); + ACCESS_ONCE(mdio_cmd_ctx->cond) = -1; + } else { + ACCESS_ONCE(mdio_cmd_ctx->cond) = 1; + } + wake_up_interruptible(&mdio_cmd_ctx->wc); +} + +/* This routine provides PHY access routines for + * mdio clause45 . + */ +static int +octnet_mdio45_access(struct lio *lio, int op, int loc, int *value) +{ + struct octeon_device *oct_dev = lio->oct_dev; + struct octeon_soft_command *sc; + struct oct_mdio_cmd_resp *mdio_cmd_rsp; + struct oct_mdio_cmd_context *mdio_cmd_ctx; + struct oct_mdio_cmd *mdio_cmd; + int retval = 0; + + sc = (struct octeon_soft_command *) + octeon_alloc_soft_command(oct_dev, + sizeof(struct oct_mdio_cmd), + sizeof(struct oct_mdio_cmd_resp), + sizeof(struct oct_mdio_cmd_context)); + + if (!sc) + return -ENOMEM; + + mdio_cmd_ctx = (struct oct_mdio_cmd_context *)sc->ctxptr; + mdio_cmd_rsp = (struct oct_mdio_cmd_resp *)sc->virtrptr; + mdio_cmd = (struct oct_mdio_cmd *)sc->virtdptr; + + ACCESS_ONCE(mdio_cmd_ctx->cond) = 0; + mdio_cmd_ctx->octeon_id = lio_get_device_id(oct_dev); + mdio_cmd->op = op; + mdio_cmd->mdio_addr = loc; + if (op) + mdio_cmd->value1 = *value; + mdio_cmd->value2 = lio->linfo.ifidx; + octeon_swap_8B_data((u64 *)mdio_cmd, sizeof(struct oct_mdio_cmd) / 8); + + octeon_prepare_soft_command(oct_dev, sc, OPCODE_NIC, OPCODE_NIC_MDIO45, + 0, 0, 0); + + sc->wait_time = 1000; + sc->callback = octnet_mdio_resp_callback; + sc->callback_arg = sc; + + init_waitqueue_head(&mdio_cmd_ctx->wc); + + retval = octeon_send_soft_command(oct_dev, sc); + + if (retval) { + dev_err(&oct_dev->pci_dev->dev, + "octnet_mdio45_access instruction failed status: %x\n", + retval); + retval = -EBUSY; + } else { + /* Sleep on a wait queue till the cond flag indicates that the + * response arrived + */ + sleep_cond(&mdio_cmd_ctx->wc, &mdio_cmd_ctx->cond); + retval = mdio_cmd_rsp->status; + if (retval) { + dev_err(&oct_dev->pci_dev->dev, "octnet mdio45 access failed\n"); + retval = -EBUSY; + } else { + octeon_swap_8B_data((u64 *)(&mdio_cmd_rsp->resp), + sizeof(struct oct_mdio_cmd) / 8); + + if (ACCESS_ONCE(mdio_cmd_ctx->cond) == 1) { + if (!op) + *value = mdio_cmd_rsp->resp.value1; + } else { + retval = -EINVAL; + } + } + } + + octeon_free_soft_command(oct_dev, sc); + + return retval; +} + +static int lio_set_phys_id(struct net_device *netdev, + enum ethtool_phys_id_state state) +{ + struct lio *lio = GET_LIO(netdev); + struct octeon_device *oct = lio->oct_dev; + int value, ret; + + switch (state) { + case ETHTOOL_ID_ACTIVE: + if (oct->chip_id == OCTEON_CN66XX) { + octnet_gpio_access(netdev, VITESSE_PHY_GPIO_CFG, + VITESSE_PHY_GPIO_DRIVEON); + return 2; + + } else if (oct->chip_id == OCTEON_CN68XX) { + /* Save the current LED settings */ + ret = octnet_mdio45_access(lio, 0, + LIO68XX_LED_BEACON_ADDR, + &lio->phy_beacon_val); + if (ret) + return ret; + + ret = octnet_mdio45_access(lio, 0, + LIO68XX_LED_CTRL_ADDR, + &lio->led_ctrl_val); + if (ret) + return ret; + + /* Configure Beacon values */ + value = LIO68XX_LED_BEACON_CFGON; + ret = + octnet_mdio45_access(lio, 1, + LIO68XX_LED_BEACON_ADDR, + &value); + if (ret) + return ret; + + value = LIO68XX_LED_CTRL_CFGON; + ret = + octnet_mdio45_access(lio, 1, + LIO68XX_LED_CTRL_ADDR, + &value); + if (ret) + return ret; + } else { + return -EINVAL; + } + break; + + case ETHTOOL_ID_ON: + if (oct->chip_id == OCTEON_CN66XX) { + octnet_gpio_access(netdev, VITESSE_PHY_GPIO_CFG, + VITESSE_PHY_GPIO_HIGH); + + } else if (oct->chip_id == OCTEON_CN68XX) { + return -EINVAL; + } else { + return -EINVAL; + } + break; + + case ETHTOOL_ID_OFF: + if (oct->chip_id == OCTEON_CN66XX) + octnet_gpio_access(netdev, VITESSE_PHY_GPIO_CFG, + VITESSE_PHY_GPIO_LOW); + else if (oct->chip_id == OCTEON_CN68XX) + return -EINVAL; + else + return -EINVAL; + + break; + + case ETHTOOL_ID_INACTIVE: + if (oct->chip_id == OCTEON_CN66XX) { + octnet_gpio_access(netdev, VITESSE_PHY_GPIO_CFG, + VITESSE_PHY_GPIO_DRIVEOFF); + } else if (oct->chip_id == OCTEON_CN68XX) { + /* Restore LED settings */ + ret = octnet_mdio45_access(lio, 1, + LIO68XX_LED_CTRL_ADDR, + &lio->led_ctrl_val); + if (ret) + return ret; + + ret = octnet_mdio45_access(lio, 1, + LIO68XX_LED_BEACON_ADDR, + &lio->phy_beacon_val); + if (ret) + return ret; + + } else { + return -EINVAL; + } + break; + + default: + return -EINVAL; + } + + return 0; +} + +static void +lio_ethtool_get_ringparam(struct net_device *netdev, + struct ethtool_ringparam *ering) +{ + struct lio *lio = GET_LIO(netdev); + struct octeon_device *oct = lio->oct_dev; + u32 tx_max_pending = 0, rx_max_pending = 0, tx_pending = 0, + rx_pending = 0; + + if (OCTEON_CN6XXX(oct)) { + struct octeon_config *conf6x = CHIP_FIELD(oct, cn6xxx, conf); + + tx_max_pending = CN6XXX_MAX_IQ_DESCRIPTORS; + rx_max_pending = CN6XXX_MAX_OQ_DESCRIPTORS; + rx_pending = CFG_GET_NUM_RX_DESCS_NIC_IF(conf6x, lio->ifidx); + tx_pending = CFG_GET_NUM_TX_DESCS_NIC_IF(conf6x, lio->ifidx); + } + + if (lio->mtu > OCTNET_DEFAULT_FRM_SIZE) { + ering->rx_pending = 0; + ering->rx_max_pending = 0; + ering->rx_mini_pending = 0; + ering->rx_jumbo_pending = rx_pending; + ering->rx_mini_max_pending = 0; + ering->rx_jumbo_max_pending = rx_max_pending; + } else { + ering->rx_pending = rx_pending; + ering->rx_max_pending = rx_max_pending; + ering->rx_mini_pending = 0; + ering->rx_jumbo_pending = 0; + ering->rx_mini_max_pending = 0; + ering->rx_jumbo_max_pending = 0; + } + + ering->tx_pending = tx_pending; + ering->tx_max_pending = tx_max_pending; +} + +static u32 lio_get_msglevel(struct net_device *netdev) +{ + struct lio *lio = GET_LIO(netdev); + + return lio->msg_enable; +} + +static void lio_set_msglevel(struct net_device *netdev, u32 msglvl) +{ + struct lio *lio = GET_LIO(netdev); + + if ((msglvl ^ lio->msg_enable) & NETIF_MSG_HW) { + if (msglvl & NETIF_MSG_HW) + liquidio_set_feature(netdev, + OCTNET_CMD_VERBOSE_ENABLE); + else + liquidio_set_feature(netdev, + OCTNET_CMD_VERBOSE_DISABLE); + } + + lio->msg_enable = msglvl; +} + +static void +lio_get_pauseparam(struct net_device *netdev, struct ethtool_pauseparam *pause) +{ + /* Notes: Not supporting any auto negotiation in these + * drivers. Just report pause frame support. + */ + pause->tx_pause = 1; + pause->rx_pause = 1; /* TODO: Need to support RX pause frame!!. */ +} + +static void +lio_get_ethtool_stats(struct net_device *netdev, + struct ethtool_stats *stats, u64 *data) +{ + struct lio *lio = GET_LIO(netdev); + struct octeon_device *oct_dev = lio->oct_dev; + int i = 0, j; + + for (j = 0; j < MAX_OCTEON_INSTR_QUEUES; j++) { + if (!(oct_dev->io_qmask.iq & (1UL << j))) + continue; + data[i++] = + CVM_CAST64(oct_dev->instr_queue[j]->stats.instr_posted); + data[i++] = + CVM_CAST64( + oct_dev->instr_queue[j]->stats.instr_processed); + data[i++] = + CVM_CAST64( + oct_dev->instr_queue[j]->stats.instr_dropped); + data[i++] = + CVM_CAST64(oct_dev->instr_queue[j]->stats.bytes_sent); + data[i++] = + CVM_CAST64(oct_dev->instr_queue[j]->stats.sgentry_sent); + data[i++] = + readl(oct_dev->instr_queue[j]->inst_cnt_reg); + data[i++] = + CVM_CAST64(oct_dev->instr_queue[j]->stats.tx_done); + data[i++] = + CVM_CAST64(oct_dev->instr_queue[j]->stats.tx_iq_busy); + data[i++] = + CVM_CAST64(oct_dev->instr_queue[j]->stats.tx_dropped); + data[i++] = + CVM_CAST64(oct_dev->instr_queue[j]->stats.tx_tot_bytes); + } + + /* for (j = 0; j < oct_dev->num_oqs; j++){ */ + for (j = 0; j < MAX_OCTEON_OUTPUT_QUEUES; j++) { + if (!(oct_dev->io_qmask.oq & (1UL << j))) + continue; + data[i++] = CVM_CAST64(oct_dev->droq[j]->stats.pkts_received); + data[i++] = CVM_CAST64(oct_dev->droq[j]->stats.bytes_received); + data[i++] = + CVM_CAST64(oct_dev->droq[j]->stats.dropped_nodispatch); + data[i++] = CVM_CAST64(oct_dev->droq[j]->stats.dropped_nomem); + data[i++] = CVM_CAST64(oct_dev->droq[j]->stats.dropped_toomany); + data[i++] = + CVM_CAST64(oct_dev->droq[j]->stats.rx_pkts_received); + data[i++] = + CVM_CAST64(oct_dev->droq[j]->stats.rx_bytes_received); + data[i++] = + CVM_CAST64(oct_dev->droq[j]->stats.rx_dropped); + } +} + +static void lio_get_strings(struct net_device *netdev, u32 stringset, u8 *data) +{ + struct lio *lio = GET_LIO(netdev); + struct octeon_device *oct_dev = lio->oct_dev; + int num_iq_stats, num_oq_stats, i, j; + + num_iq_stats = ARRAY_SIZE(oct_iq_stats_strings); + for (i = 0; i < MAX_OCTEON_INSTR_QUEUES; i++) { + if (!(oct_dev->io_qmask.iq & (1UL << i))) + continue; + for (j = 0; j < num_iq_stats; j++) { + sprintf(data, "IQ%d %s", i, oct_iq_stats_strings[j]); + data += ETH_GSTRING_LEN; + } + } + + num_oq_stats = ARRAY_SIZE(oct_droq_stats_strings); + /* for (i = 0; i < oct_dev->num_oqs; i++) { */ + for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES; i++) { + if (!(oct_dev->io_qmask.oq & (1UL << i))) + continue; + for (j = 0; j < num_oq_stats; j++) { + sprintf(data, "OQ%d %s", i, oct_droq_stats_strings[j]); + data += ETH_GSTRING_LEN; + } + } +} + +static int lio_get_sset_count(struct net_device *netdev, int sset) +{ + struct lio *lio = GET_LIO(netdev); + struct octeon_device *oct_dev = lio->oct_dev; + + return (ARRAY_SIZE(oct_iq_stats_strings) * oct_dev->num_iqs) + + (ARRAY_SIZE(oct_droq_stats_strings) * oct_dev->num_oqs); +} + +static int lio_get_intr_coalesce(struct net_device *netdev, + struct ethtool_coalesce *intr_coal) +{ + struct lio *lio = GET_LIO(netdev); + struct octeon_device *oct = lio->oct_dev; + struct octeon_cn6xxx *cn6xxx = (struct octeon_cn6xxx *)oct->chip; + struct octeon_instr_queue *iq; + struct oct_intrmod_cfg *intrmod_cfg; + + intrmod_cfg = &oct->intrmod; + + switch (oct->chip_id) { + /* case OCTEON_CN73XX: Todo */ + /* break; */ + case OCTEON_CN68XX: + case OCTEON_CN66XX: + if (!intrmod_cfg->intrmod_enable) { + intr_coal->rx_coalesce_usecs = + CFG_GET_OQ_INTR_TIME(cn6xxx->conf); + intr_coal->rx_max_coalesced_frames = + CFG_GET_OQ_INTR_PKT(cn6xxx->conf); + } else { + intr_coal->use_adaptive_rx_coalesce = + intrmod_cfg->intrmod_enable; + intr_coal->rate_sample_interval = + intrmod_cfg->intrmod_check_intrvl; + intr_coal->pkt_rate_high = + intrmod_cfg->intrmod_maxpkt_ratethr; + intr_coal->pkt_rate_low = + intrmod_cfg->intrmod_minpkt_ratethr; + intr_coal->rx_max_coalesced_frames_high = + intrmod_cfg->intrmod_maxcnt_trigger; + intr_coal->rx_coalesce_usecs_high = + intrmod_cfg->intrmod_maxtmr_trigger; + intr_coal->rx_coalesce_usecs_low = + intrmod_cfg->intrmod_mintmr_trigger; + intr_coal->rx_max_coalesced_frames_low = + intrmod_cfg->intrmod_mincnt_trigger; + } + + iq = oct->instr_queue[lio->linfo.txpciq[0]]; + intr_coal->tx_max_coalesced_frames = iq->fill_threshold; + break; + + default: + netif_info(lio, drv, lio->netdev, "Unknown Chip !!\n"); + return -EINVAL; + } + + return 0; +} + +/* Callback function for intrmod */ +static void octnet_intrmod_callback(struct octeon_device *oct_dev, + u32 status, + void *ptr) +{ + struct oct_intrmod_cmd *cmd = ptr; + struct octeon_soft_command *sc = cmd->sc; + + oct_dev = cmd->oct_dev; + + if (status) + dev_err(&oct_dev->pci_dev->dev, "intrmod config failed. Status: %llx\n", + CVM_CAST64(status)); + else + dev_info(&oct_dev->pci_dev->dev, + "Rx-Adaptive Interrupt moderation enabled:%llx\n", + oct_dev->intrmod.intrmod_enable); + + octeon_free_soft_command(oct_dev, sc); +} + +/* Configure interrupt moderation parameters */ +static int octnet_set_intrmod_cfg(void *oct, struct oct_intrmod_cfg *intr_cfg) +{ + struct octeon_soft_command *sc; + struct oct_intrmod_cmd *cmd; + struct oct_intrmod_cfg *cfg; + int retval; + struct octeon_device *oct_dev = (struct octeon_device *)oct; + + /* Alloc soft command */ + sc = (struct octeon_soft_command *) + octeon_alloc_soft_command(oct_dev, + sizeof(struct oct_intrmod_cfg), + 0, + sizeof(struct oct_intrmod_cmd)); + + if (!sc) + return -ENOMEM; + + cmd = (struct oct_intrmod_cmd *)sc->ctxptr; + cfg = (struct oct_intrmod_cfg *)sc->virtdptr; + + memcpy(cfg, intr_cfg, sizeof(struct oct_intrmod_cfg)); + octeon_swap_8B_data((u64 *)cfg, (sizeof(struct oct_intrmod_cfg)) / 8); + cmd->sc = sc; + cmd->cfg = cfg; + cmd->oct_dev = oct_dev; + + octeon_prepare_soft_command(oct_dev, sc, OPCODE_NIC, + OPCODE_NIC_INTRMOD_CFG, 0, 0, 0); + + sc->callback = octnet_intrmod_callback; + sc->callback_arg = cmd; + sc->wait_time = 1000; + + retval = octeon_send_soft_command(oct_dev, sc); + if (retval) { + octeon_free_soft_command(oct_dev, sc); + return -EINVAL; + } + + return 0; +} + +/* Enable/Disable auto interrupt Moderation */ +static int oct_cfg_adaptive_intr(struct lio *lio, struct ethtool_coalesce + *intr_coal, int adaptive) +{ + int ret = 0; + struct octeon_device *oct = lio->oct_dev; + struct oct_intrmod_cfg *intrmod_cfg; + + intrmod_cfg = &oct->intrmod; + + if (adaptive) { + if (intr_coal->rate_sample_interval) + intrmod_cfg->intrmod_check_intrvl = + intr_coal->rate_sample_interval; + else + intrmod_cfg->intrmod_check_intrvl = + LIO_INTRMOD_CHECK_INTERVAL; + + if (intr_coal->pkt_rate_high) + intrmod_cfg->intrmod_maxpkt_ratethr = + intr_coal->pkt_rate_high; + else + intrmod_cfg->intrmod_maxpkt_ratethr = + LIO_INTRMOD_MAXPKT_RATETHR; + + if (intr_coal->pkt_rate_low) + intrmod_cfg->intrmod_minpkt_ratethr = + intr_coal->pkt_rate_low; + else + intrmod_cfg->intrmod_minpkt_ratethr = + LIO_INTRMOD_MINPKT_RATETHR; + + if (intr_coal->rx_max_coalesced_frames_high) + intrmod_cfg->intrmod_maxcnt_trigger = + intr_coal->rx_max_coalesced_frames_high; + else + intrmod_cfg->intrmod_maxcnt_trigger = + LIO_INTRMOD_MAXCNT_TRIGGER; + + if (intr_coal->rx_coalesce_usecs_high) + intrmod_cfg->intrmod_maxtmr_trigger = + intr_coal->rx_coalesce_usecs_high; + else + intrmod_cfg->intrmod_maxtmr_trigger = + LIO_INTRMOD_MAXTMR_TRIGGER; + + if (intr_coal->rx_coalesce_usecs_low) + intrmod_cfg->intrmod_mintmr_trigger = + intr_coal->rx_coalesce_usecs_low; + else + intrmod_cfg->intrmod_mintmr_trigger = + LIO_INTRMOD_MINTMR_TRIGGER; + + if (intr_coal->rx_max_coalesced_frames_low) + intrmod_cfg->intrmod_mincnt_trigger = + intr_coal->rx_max_coalesced_frames_low; + else + intrmod_cfg->intrmod_mincnt_trigger = + LIO_INTRMOD_MINCNT_TRIGGER; + } + + intrmod_cfg->intrmod_enable = adaptive; + ret = octnet_set_intrmod_cfg(oct, intrmod_cfg); + + return ret; +} + +static int +oct_cfg_rx_intrcnt(struct lio *lio, struct ethtool_coalesce *intr_coal) +{ + int ret; + struct octeon_device *oct = lio->oct_dev; + struct octeon_cn6xxx *cn6xxx = (struct octeon_cn6xxx *)oct->chip; + u32 rx_max_coalesced_frames; + + if (!intr_coal->rx_max_coalesced_frames) + rx_max_coalesced_frames = CN6XXX_OQ_INTR_PKT; + else + rx_max_coalesced_frames = intr_coal->rx_max_coalesced_frames; + + /* Disable adaptive interrupt modulation */ + ret = oct_cfg_adaptive_intr(lio, intr_coal, 0); + if (ret) + return ret; + + /* Config Cnt based interrupt values */ + octeon_write_csr(oct, CN6XXX_SLI_OQ_INT_LEVEL_PKTS, + rx_max_coalesced_frames); + CFG_SET_OQ_INTR_PKT(cn6xxx->conf, rx_max_coalesced_frames); + return 0; +} + +static int oct_cfg_rx_intrtime(struct lio *lio, struct ethtool_coalesce + *intr_coal) +{ + int ret; + struct octeon_device *oct = lio->oct_dev; + struct octeon_cn6xxx *cn6xxx = (struct octeon_cn6xxx *)oct->chip; + u32 time_threshold, rx_coalesce_usecs; + + if (!intr_coal->rx_coalesce_usecs) + rx_coalesce_usecs = CN6XXX_OQ_INTR_TIME; + else + rx_coalesce_usecs = intr_coal->rx_coalesce_usecs; + + /* Disable adaptive interrupt modulation */ + ret = oct_cfg_adaptive_intr(lio, intr_coal, 0); + if (ret) + return ret; + + /* Config Time based interrupt values */ + time_threshold = lio_cn6xxx_get_oq_ticks(oct, rx_coalesce_usecs); + octeon_write_csr(oct, CN6XXX_SLI_OQ_INT_LEVEL_TIME, time_threshold); + CFG_SET_OQ_INTR_TIME(cn6xxx->conf, rx_coalesce_usecs); + + return 0; +} + +static int lio_set_intr_coalesce(struct net_device *netdev, + struct ethtool_coalesce *intr_coal) +{ + struct lio *lio = GET_LIO(netdev); + int ret; + struct octeon_device *oct = lio->oct_dev; + u32 j, q_no; + + if ((intr_coal->tx_max_coalesced_frames >= CN6XXX_DB_MIN) && + (intr_coal->tx_max_coalesced_frames <= CN6XXX_DB_MAX)) { + for (j = 0; j < lio->linfo.num_txpciq; j++) { + q_no = lio->linfo.txpciq[j]; + oct->instr_queue[q_no]->fill_threshold = + intr_coal->tx_max_coalesced_frames; + } + } else { + dev_err(&oct->pci_dev->dev, + "LIQUIDIO: Invalid tx-frames:%d. Range is min:%d max:%d\n", + intr_coal->tx_max_coalesced_frames, CN6XXX_DB_MIN, + CN6XXX_DB_MAX); + return -EINVAL; + } + + /* User requested adaptive-rx on */ + if (intr_coal->use_adaptive_rx_coalesce) { + ret = oct_cfg_adaptive_intr(lio, intr_coal, 1); + if (ret) + goto ret_intrmod; + } + + /* User requested adaptive-rx off and rx coalesce */ + if ((intr_coal->rx_coalesce_usecs) && + (!intr_coal->use_adaptive_rx_coalesce)) { + ret = oct_cfg_rx_intrtime(lio, intr_coal); + if (ret) + goto ret_intrmod; + } + + /* User requested adaptive-rx off and rx coalesce */ + if ((intr_coal->rx_max_coalesced_frames) && + (!intr_coal->use_adaptive_rx_coalesce)) { + ret = oct_cfg_rx_intrcnt(lio, intr_coal); + if (ret) + goto ret_intrmod; + } + + /* User requested adaptive-rx off, so use default coalesce params */ + if ((!intr_coal->rx_max_coalesced_frames) && + (!intr_coal->use_adaptive_rx_coalesce) && + (!intr_coal->rx_coalesce_usecs)) { + dev_info(&oct->pci_dev->dev, + "Turning off adaptive-rx interrupt moderation\n"); + dev_info(&oct->pci_dev->dev, + "Using RX Coalesce Default values rx_coalesce_usecs:%d rx_max_coalesced_frames:%d\n", + CN6XXX_OQ_INTR_TIME, CN6XXX_OQ_INTR_PKT); + ret = oct_cfg_rx_intrtime(lio, intr_coal); + if (ret) + goto ret_intrmod; + + ret = oct_cfg_rx_intrcnt(lio, intr_coal); + if (ret) + goto ret_intrmod; + } + + return 0; +ret_intrmod: + return ret; +} + +static int lio_get_ts_info(struct net_device *netdev, + struct ethtool_ts_info *info) +{ + struct lio *lio = GET_LIO(netdev); + + info->so_timestamping = + SOF_TIMESTAMPING_TX_HARDWARE | + SOF_TIMESTAMPING_TX_SOFTWARE | + SOF_TIMESTAMPING_RX_HARDWARE | + SOF_TIMESTAMPING_RX_SOFTWARE | + SOF_TIMESTAMPING_SOFTWARE | SOF_TIMESTAMPING_RAW_HARDWARE; + + if (lio->ptp_clock) + info->phc_index = ptp_clock_index(lio->ptp_clock); + else + info->phc_index = -1; + + info->tx_types = (1 << HWTSTAMP_TX_OFF) | (1 << HWTSTAMP_TX_ON); + + info->rx_filters = (1 << HWTSTAMP_FILTER_NONE) | + (1 << HWTSTAMP_FILTER_PTP_V1_L4_EVENT) | + (1 << HWTSTAMP_FILTER_PTP_V2_L2_EVENT) | + (1 << HWTSTAMP_FILTER_PTP_V2_L4_EVENT); + + return 0; +} + +static int lio_set_settings(struct net_device *netdev, struct ethtool_cmd *ecmd) +{ + struct lio *lio = GET_LIO(netdev); + struct octeon_device *oct = lio->oct_dev; + struct oct_link_info *linfo; + struct octnic_ctrl_pkt nctrl; + struct octnic_ctrl_params nparams; + int ret = 0; + + /* get the link info */ + linfo = &lio->linfo; + + if (ecmd->autoneg != AUTONEG_ENABLE && ecmd->autoneg != AUTONEG_DISABLE) + return -EINVAL; + + if (ecmd->autoneg == AUTONEG_DISABLE && ((ecmd->speed != SPEED_100 && + ecmd->speed != SPEED_10) || + (ecmd->duplex != DUPLEX_HALF && + ecmd->duplex != DUPLEX_FULL))) + return -EINVAL; + + /* Ethtool Support is not provided for XAUI and RXAUI Interfaces + * as they operate at fixed Speed and Duplex settings + */ + if (linfo->link.s.interface == INTERFACE_MODE_XAUI || + linfo->link.s.interface == INTERFACE_MODE_RXAUI) { + dev_info(&oct->pci_dev->dev, "XAUI IFs settings cannot be modified.\n"); + return -EINVAL; + } + + memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt)); + + nctrl.ncmd.u64 = 0; + nctrl.ncmd.s.cmd = OCTNET_CMD_SET_SETTINGS; + nctrl.wait_time = 1000; + nctrl.netpndev = (u64)netdev; + nctrl.ncmd.s.param1 = lio->linfo.ifidx; + nctrl.cb_fn = liquidio_link_ctrl_cmd_completion; + + /* Passing the parameters sent by ethtool like Speed, Autoneg & Duplex + * to SE core application using ncmd.s.more & ncmd.s.param + */ + if (ecmd->autoneg == AUTONEG_ENABLE) { + /* Autoneg ON */ + nctrl.ncmd.s.more = OCTNIC_NCMD_PHY_ON | + OCTNIC_NCMD_AUTONEG_ON; + nctrl.ncmd.s.param2 = ecmd->advertising; + } else { + /* Autoneg OFF */ + nctrl.ncmd.s.more = OCTNIC_NCMD_PHY_ON; + + nctrl.ncmd.s.param3 = ecmd->duplex; + + nctrl.ncmd.s.param2 = ecmd->speed; + } + + nparams.resp_order = OCTEON_RESP_ORDERED; + + ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl, nparams); + if (ret < 0) { + dev_err(&oct->pci_dev->dev, "Failed to set settings\n"); + return -1; + } + + return 0; +} + +static int lio_nway_reset(struct net_device *netdev) +{ + if (netif_running(netdev)) { + struct ethtool_cmd ecmd; + + memset(&ecmd, 0, sizeof(struct ethtool_cmd)); + ecmd.autoneg = 0; + ecmd.speed = 0; + ecmd.duplex = 0; + lio_set_settings(netdev, &ecmd); + } + return 0; +} + +/* Return register dump len. */ +static int lio_get_regs_len(struct net_device *dev) +{ + return OCT_ETHTOOL_REGDUMP_LEN; +} + +static int cn6xxx_read_csr_reg(char *s, struct octeon_device *oct) +{ + u32 reg; + int i, len = 0; + + /* PCI Window Registers */ + + len += sprintf(s + len, "\n\t Octeon CSR Registers\n\n"); + reg = CN6XXX_WIN_WR_ADDR_LO; + len += sprintf(s + len, "\n[%02x] (WIN_WR_ADDR_LO): %08x\n", + CN6XXX_WIN_WR_ADDR_LO, octeon_read_csr(oct, reg)); + reg = CN6XXX_WIN_WR_ADDR_HI; + len += sprintf(s + len, "[%02x] (WIN_WR_ADDR_HI): %08x\n", + CN6XXX_WIN_WR_ADDR_HI, octeon_read_csr(oct, reg)); + reg = CN6XXX_WIN_RD_ADDR_LO; + len += sprintf(s + len, "[%02x] (WIN_RD_ADDR_LO): %08x\n", + CN6XXX_WIN_RD_ADDR_LO, octeon_read_csr(oct, reg)); + reg = CN6XXX_WIN_RD_ADDR_HI; + len += sprintf(s + len, "[%02x] (WIN_RD_ADDR_HI): %08x\n", + CN6XXX_WIN_RD_ADDR_HI, octeon_read_csr(oct, reg)); + reg = CN6XXX_WIN_WR_DATA_LO; + len += sprintf(s + len, "[%02x] (WIN_WR_DATA_LO): %08x\n", + CN6XXX_WIN_WR_DATA_LO, octeon_read_csr(oct, reg)); + reg = CN6XXX_WIN_WR_DATA_HI; + len += sprintf(s + len, "[%02x] (WIN_WR_DATA_HI): %08x\n", + CN6XXX_WIN_WR_DATA_HI, octeon_read_csr(oct, reg)); + len += sprintf(s + len, "[%02x] (WIN_WR_MASK_REG): %08x\n", + CN6XXX_WIN_WR_MASK_REG, + octeon_read_csr(oct, CN6XXX_WIN_WR_MASK_REG)); + + /* PCI Interrupt Register */ + len += sprintf(s + len, "\n[%x] (INT_ENABLE PORT 0): %08x\n", + CN6XXX_SLI_INT_ENB64_PORT0, octeon_read_csr(oct, + CN6XXX_SLI_INT_ENB64_PORT0)); + len += sprintf(s + len, "\n[%x] (INT_ENABLE PORT 1): %08x\n", + CN6XXX_SLI_INT_ENB64_PORT1, + octeon_read_csr(oct, CN6XXX_SLI_INT_ENB64_PORT1)); + len += sprintf(s + len, "[%x] (INT_SUM): %08x\n", CN6XXX_SLI_INT_SUM64, + octeon_read_csr(oct, CN6XXX_SLI_INT_SUM64)); + + /* PCI Output queue registers */ + for (i = 0; i < oct->num_oqs; i++) { + reg = CN6XXX_SLI_OQ_PKTS_SENT(i); + len += sprintf(s + len, "\n[%x] (PKTS_SENT_%d): %08x\n", + reg, i, octeon_read_csr(oct, reg)); + reg = CN6XXX_SLI_OQ_PKTS_CREDIT(i); + len += sprintf(s + len, "[%x] (PKT_CREDITS_%d): %08x\n", + reg, i, octeon_read_csr(oct, reg)); + } + reg = CN6XXX_SLI_OQ_INT_LEVEL_PKTS; + len += sprintf(s + len, "\n[%x] (PKTS_SENT_INT_LEVEL): %08x\n", + reg, octeon_read_csr(oct, reg)); + reg = CN6XXX_SLI_OQ_INT_LEVEL_TIME; + len += sprintf(s + len, "[%x] (PKTS_SENT_TIME): %08x\n", + reg, octeon_read_csr(oct, reg)); + + /* PCI Input queue registers */ + for (i = 0; i <= 3; i++) { + u32 reg; + + reg = CN6XXX_SLI_IQ_DOORBELL(i); + len += sprintf(s + len, "\n[%x] (INSTR_DOORBELL_%d): %08x\n", + reg, i, octeon_read_csr(oct, reg)); + reg = CN6XXX_SLI_IQ_INSTR_COUNT(i); + len += sprintf(s + len, "[%x] (INSTR_COUNT_%d): %08x\n", + reg, i, octeon_read_csr(oct, reg)); + } + + /* PCI DMA registers */ + + len += sprintf(s + len, "\n[%x] (DMA_CNT_0): %08x\n", + CN6XXX_DMA_CNT(0), + octeon_read_csr(oct, CN6XXX_DMA_CNT(0))); + reg = CN6XXX_DMA_PKT_INT_LEVEL(0); + len += sprintf(s + len, "[%x] (DMA_INT_LEV_0): %08x\n", + CN6XXX_DMA_PKT_INT_LEVEL(0), octeon_read_csr(oct, reg)); + reg = CN6XXX_DMA_TIME_INT_LEVEL(0); + len += sprintf(s + len, "[%x] (DMA_TIME_0): %08x\n", + CN6XXX_DMA_TIME_INT_LEVEL(0), + octeon_read_csr(oct, reg)); + + len += sprintf(s + len, "\n[%x] (DMA_CNT_1): %08x\n", + CN6XXX_DMA_CNT(1), + octeon_read_csr(oct, CN6XXX_DMA_CNT(1))); + reg = CN6XXX_DMA_PKT_INT_LEVEL(1); + len += sprintf(s + len, "[%x] (DMA_INT_LEV_1): %08x\n", + CN6XXX_DMA_PKT_INT_LEVEL(1), + octeon_read_csr(oct, reg)); + reg = CN6XXX_DMA_PKT_INT_LEVEL(1); + len += sprintf(s + len, "[%x] (DMA_TIME_1): %08x\n", + CN6XXX_DMA_TIME_INT_LEVEL(1), + octeon_read_csr(oct, reg)); + + /* PCI Index registers */ + + len += sprintf(s + len, "\n"); + + for (i = 0; i < 16; i++) { + reg = lio_pci_readq(oct, CN6XXX_BAR1_REG(i, oct->pcie_port)); + len += sprintf(s + len, "[%llx] (BAR1_INDEX_%02d): %08x\n", + CN6XXX_BAR1_REG(i, oct->pcie_port), i, reg); + } + + return len; +} + +static int cn6xxx_read_config_reg(char *s, struct octeon_device *oct) +{ + u32 val; + int i, len = 0; + + /* PCI CONFIG Registers */ + + len += sprintf(s + len, + "\n\t Octeon Config space Registers\n\n"); + + for (i = 0; i <= 13; i++) { + pci_read_config_dword(oct->pci_dev, (i * 4), &val); + len += sprintf(s + len, "[0x%x] (Config[%d]): 0x%08x\n", + (i * 4), i, val); + } + + for (i = 30; i <= 34; i++) { + pci_read_config_dword(oct->pci_dev, (i * 4), &val); + len += sprintf(s + len, "[0x%x] (Config[%d]): 0x%08x\n", + (i * 4), i, val); + } + + return len; +} + +/* Return register dump user app. */ +static void lio_get_regs(struct net_device *dev, + struct ethtool_regs *regs, void *regbuf) +{ + struct lio *lio = GET_LIO(dev); + int len = 0; + struct octeon_device *oct = lio->oct_dev; + + memset(regbuf, 0, OCT_ETHTOOL_REGDUMP_LEN); + regs->version = OCT_ETHTOOL_REGSVER; + + switch (oct->chip_id) { + /* case OCTEON_CN73XX: Todo */ + case OCTEON_CN68XX: + case OCTEON_CN66XX: + len += cn6xxx_read_csr_reg(regbuf + len, oct); + len += cn6xxx_read_config_reg(regbuf + len, oct); + break; + default: + dev_err(&oct->pci_dev->dev, "%s Unknown chipid: %d\n", + __func__, oct->chip_id); + } +} + +static const struct ethtool_ops lio_ethtool_ops = { + .get_settings = lio_get_settings, + .get_link = ethtool_op_get_link, + .get_drvinfo = lio_get_drvinfo, + .get_ringparam = lio_ethtool_get_ringparam, + .get_channels = lio_ethtool_get_channels, + .set_phys_id = lio_set_phys_id, + .get_eeprom_len = lio_get_eeprom_len, + .get_eeprom = lio_get_eeprom, + .get_strings = lio_get_strings, + .get_ethtool_stats = lio_get_ethtool_stats, + .get_pauseparam = lio_get_pauseparam, + .get_regs_len = lio_get_regs_len, + .get_regs = lio_get_regs, + .get_msglevel = lio_get_msglevel, + .set_msglevel = lio_set_msglevel, + .get_sset_count = lio_get_sset_count, + .nway_reset = lio_nway_reset, + .set_settings = lio_set_settings, + .get_coalesce = lio_get_intr_coalesce, + .set_coalesce = lio_set_intr_coalesce, + .get_ts_info = lio_get_ts_info, +}; + +void liquidio_set_ethtool_ops(struct net_device *netdev) +{ + netdev->ethtool_ops = &lio_ethtool_ops; +} diff --git a/kernel/drivers/net/ethernet/cavium/liquidio/lio_main.c b/kernel/drivers/net/ethernet/cavium/liquidio/lio_main.c new file mode 100644 index 000000000..b89504405 --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/liquidio/lio_main.c @@ -0,0 +1,3667 @@ +/********************************************************************** +* Author: Cavium, Inc. +* +* Contact: support@cavium.com +* Please include "LiquidIO" in the subject. +* +* Copyright (c) 2003-2015 Cavium, Inc. +* +* This file is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License, Version 2, as +* published by the Free Software Foundation. +* +* This file is distributed in the hope that it will be useful, but +* AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty +* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or +* NONINFRINGEMENT. See the GNU General Public License for more +* details. +* +* This file may also be available under a different license from Cavium. +* Contact Cavium, Inc. for more information +**********************************************************************/ +#include <linux/version.h> +#include <linux/module.h> +#include <linux/crc32.h> +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include <linux/pci_ids.h> +#include <linux/ip.h> +#include <net/ip.h> +#include <linux/ipv6.h> +#include <linux/net_tstamp.h> +#include <linux/if_vlan.h> +#include <linux/firmware.h> +#include <linux/ethtool.h> +#include <linux/ptp_clock_kernel.h> +#include <linux/types.h> +#include <linux/list.h> +#include <linux/workqueue.h> +#include <linux/interrupt.h> +#include "octeon_config.h" +#include "liquidio_common.h" +#include "octeon_droq.h" +#include "octeon_iq.h" +#include "response_manager.h" +#include "octeon_device.h" +#include "octeon_nic.h" +#include "octeon_main.h" +#include "octeon_network.h" +#include "cn66xx_regs.h" +#include "cn66xx_device.h" +#include "cn68xx_regs.h" +#include "cn68xx_device.h" +#include "liquidio_image.h" + +MODULE_AUTHOR("Cavium Networks, <support@cavium.com>"); +MODULE_DESCRIPTION("Cavium LiquidIO Intelligent Server Adapter Driver"); +MODULE_LICENSE("GPL"); +MODULE_VERSION(LIQUIDIO_VERSION); +MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210SV_NAME LIO_FW_NAME_SUFFIX); +MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210NV_NAME LIO_FW_NAME_SUFFIX); +MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_410NV_NAME LIO_FW_NAME_SUFFIX); + +static int ddr_timeout = 10000; +module_param(ddr_timeout, int, 0644); +MODULE_PARM_DESC(ddr_timeout, + "Number of milliseconds to wait for DDR initialization. 0 waits for ddr_timeout to be set to non-zero value before starting to check"); + +static u32 console_bitmask; +module_param(console_bitmask, int, 0644); +MODULE_PARM_DESC(console_bitmask, + "Bitmask indicating which consoles have debug output redirected to syslog."); + +#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK) + +static int debug = -1; +module_param(debug, int, 0644); +MODULE_PARM_DESC(debug, "NETIF_MSG debug bits"); + +static char fw_type[LIO_MAX_FW_TYPE_LEN]; +module_param_string(fw_type, fw_type, sizeof(fw_type), 0000); +MODULE_PARM_DESC(fw_type, "Type of firmware to be loaded. Default \"nic\""); + +static int conf_type; +module_param(conf_type, int, 0); +MODULE_PARM_DESC(conf_type, "select octeon configuration 0 default 1 ovs"); + +/* Bit mask values for lio->ifstate */ +#define LIO_IFSTATE_DROQ_OPS 0x01 +#define LIO_IFSTATE_REGISTERED 0x02 +#define LIO_IFSTATE_RUNNING 0x04 +#define LIO_IFSTATE_RX_TIMESTAMP_ENABLED 0x08 + +/* Polling interval for determining when NIC application is alive */ +#define LIQUIDIO_STARTER_POLL_INTERVAL_MS 100 + +/* runtime link query interval */ +#define LIQUIDIO_LINK_QUERY_INTERVAL_MS 1000 + +struct liquidio_if_cfg_context { + int octeon_id; + + wait_queue_head_t wc; + + int cond; +}; + +struct liquidio_if_cfg_resp { + u64 rh; + struct liquidio_if_cfg_info cfg_info; + u64 status; +}; + +struct oct_link_status_resp { + u64 rh; + struct oct_link_info link_info; + u64 status; +}; + +struct oct_timestamp_resp { + u64 rh; + u64 timestamp; + u64 status; +}; + +#define OCT_TIMESTAMP_RESP_SIZE (sizeof(struct oct_timestamp_resp)) + +union tx_info { + u64 u64; + struct { +#ifdef __BIG_ENDIAN_BITFIELD + u16 gso_size; + u16 gso_segs; + u32 reserved; +#else + u32 reserved; + u16 gso_segs; + u16 gso_size; +#endif + } s; +}; + +/** Octeon device properties to be used by the NIC module. + * Each octeon device in the system will be represented + * by this structure in the NIC module. + */ + +#define OCTNIC_MAX_SG (MAX_SKB_FRAGS) + +#define OCTNIC_GSO_MAX_HEADER_SIZE 128 +#define OCTNIC_GSO_MAX_SIZE (GSO_MAX_SIZE - OCTNIC_GSO_MAX_HEADER_SIZE) + +/** Structure of a node in list of gather components maintained by + * NIC driver for each network device. + */ +struct octnic_gather { + /** List manipulation. Next and prev pointers. */ + struct list_head list; + + /** Size of the gather component at sg in bytes. */ + int sg_size; + + /** Number of bytes that sg was adjusted to make it 8B-aligned. */ + int adjust; + + /** Gather component that can accommodate max sized fragment list + * received from the IP layer. + */ + struct octeon_sg_entry *sg; +}; + +/** This structure is used by NIC driver to store information required + * to free the sk_buff when the packet has been fetched by Octeon. + * Bytes offset below assume worst-case of a 64-bit system. + */ +struct octnet_buf_free_info { + /** Bytes 1-8. Pointer to network device private structure. */ + struct lio *lio; + + /** Bytes 9-16. Pointer to sk_buff. */ + struct sk_buff *skb; + + /** Bytes 17-24. Pointer to gather list. */ + struct octnic_gather *g; + + /** Bytes 25-32. Physical address of skb->data or gather list. */ + u64 dptr; + + /** Bytes 33-47. Piggybacked soft command, if any */ + struct octeon_soft_command *sc; +}; + +struct handshake { + struct completion init; + struct completion started; + struct pci_dev *pci_dev; + int init_ok; + int started_ok; +}; + +struct octeon_device_priv { + /** Tasklet structures for this device. */ + struct tasklet_struct droq_tasklet; + unsigned long napi_mask; +}; + +static int octeon_device_init(struct octeon_device *); +static void liquidio_remove(struct pci_dev *pdev); +static int liquidio_probe(struct pci_dev *pdev, + const struct pci_device_id *ent); + +static struct handshake handshake[MAX_OCTEON_DEVICES]; +static struct completion first_stage; + +static void octeon_droq_bh(unsigned long pdev) +{ + int q_no; + int reschedule = 0; + struct octeon_device *oct = (struct octeon_device *)pdev; + struct octeon_device_priv *oct_priv = + (struct octeon_device_priv *)oct->priv; + + /* for (q_no = 0; q_no < oct->num_oqs; q_no++) { */ + for (q_no = 0; q_no < MAX_OCTEON_OUTPUT_QUEUES; q_no++) { + if (!(oct->io_qmask.oq & (1UL << q_no))) + continue; + reschedule |= octeon_droq_process_packets(oct, oct->droq[q_no], + MAX_PACKET_BUDGET); + } + + if (reschedule) + tasklet_schedule(&oct_priv->droq_tasklet); +} + +static int lio_wait_for_oq_pkts(struct octeon_device *oct) +{ + struct octeon_device_priv *oct_priv = + (struct octeon_device_priv *)oct->priv; + int retry = 100, pkt_cnt = 0, pending_pkts = 0; + int i; + + do { + pending_pkts = 0; + + for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES; i++) { + if (!(oct->io_qmask.oq & (1UL << i))) + continue; + pkt_cnt += octeon_droq_check_hw_for_pkts(oct, + oct->droq[i]); + } + if (pkt_cnt > 0) { + pending_pkts += pkt_cnt; + tasklet_schedule(&oct_priv->droq_tasklet); + } + pkt_cnt = 0; + schedule_timeout_uninterruptible(1); + + } while (retry-- && pending_pkts); + + return pkt_cnt; +} + +void octeon_report_tx_completion_to_bql(void *txq, unsigned int pkts_compl, + unsigned int bytes_compl) +{ + struct netdev_queue *netdev_queue = txq; + + netdev_tx_completed_queue(netdev_queue, pkts_compl, bytes_compl); +} + +void octeon_update_tx_completion_counters(void *buf, int reqtype, + unsigned int *pkts_compl, + unsigned int *bytes_compl) +{ + struct octnet_buf_free_info *finfo; + struct sk_buff *skb = NULL; + struct octeon_soft_command *sc; + + switch (reqtype) { + case REQTYPE_NORESP_NET: + case REQTYPE_NORESP_NET_SG: + finfo = buf; + skb = finfo->skb; + break; + + case REQTYPE_RESP_NET_SG: + case REQTYPE_RESP_NET: + sc = buf; + skb = sc->callback_arg; + break; + + default: + return; + } + + (*pkts_compl)++; + *bytes_compl += skb->len; +} + +void octeon_report_sent_bytes_to_bql(void *buf, int reqtype) +{ + struct octnet_buf_free_info *finfo; + struct sk_buff *skb; + struct octeon_soft_command *sc; + struct netdev_queue *txq; + + switch (reqtype) { + case REQTYPE_NORESP_NET: + case REQTYPE_NORESP_NET_SG: + finfo = buf; + skb = finfo->skb; + break; + + case REQTYPE_RESP_NET_SG: + case REQTYPE_RESP_NET: + sc = buf; + skb = sc->callback_arg; + break; + + default: + return; + } + + txq = netdev_get_tx_queue(skb->dev, skb_get_queue_mapping(skb)); + netdev_tx_sent_queue(txq, skb->len); +} + +int octeon_console_debug_enabled(u32 console) +{ + return (console_bitmask >> (console)) & 0x1; +} + +/** + * \brief Forces all IO queues off on a given device + * @param oct Pointer to Octeon device + */ +static void force_io_queues_off(struct octeon_device *oct) +{ + if ((oct->chip_id == OCTEON_CN66XX) || + (oct->chip_id == OCTEON_CN68XX)) { + /* Reset the Enable bits for Input Queues. */ + octeon_write_csr(oct, CN6XXX_SLI_PKT_INSTR_ENB, 0); + + /* Reset the Enable bits for Output Queues. */ + octeon_write_csr(oct, CN6XXX_SLI_PKT_OUT_ENB, 0); + } +} + +/** + * \brief wait for all pending requests to complete + * @param oct Pointer to Octeon device + * + * Called during shutdown sequence + */ +static int wait_for_pending_requests(struct octeon_device *oct) +{ + int i, pcount = 0; + + for (i = 0; i < 100; i++) { + pcount = + atomic_read(&oct->response_list + [OCTEON_ORDERED_SC_LIST].pending_req_count); + if (pcount) + schedule_timeout_uninterruptible(HZ / 10); + else + break; + } + + if (pcount) + return 1; + + return 0; +} + +/** + * \brief Cause device to go quiet so it can be safely removed/reset/etc + * @param oct Pointer to Octeon device + */ +static inline void pcierror_quiesce_device(struct octeon_device *oct) +{ + int i; + + /* Disable the input and output queues now. No more packets will + * arrive from Octeon, but we should wait for all packet processing + * to finish. + */ + force_io_queues_off(oct); + + /* To allow for in-flight requests */ + schedule_timeout_uninterruptible(100); + + if (wait_for_pending_requests(oct)) + dev_err(&oct->pci_dev->dev, "There were pending requests\n"); + + /* Force all requests waiting to be fetched by OCTEON to complete. */ + for (i = 0; i < MAX_OCTEON_INSTR_QUEUES; i++) { + struct octeon_instr_queue *iq; + + if (!(oct->io_qmask.iq & (1UL << i))) + continue; + iq = oct->instr_queue[i]; + + if (atomic_read(&iq->instr_pending)) { + spin_lock_bh(&iq->lock); + iq->fill_cnt = 0; + iq->octeon_read_index = iq->host_write_index; + iq->stats.instr_processed += + atomic_read(&iq->instr_pending); + lio_process_iq_request_list(oct, iq); + spin_unlock_bh(&iq->lock); + } + } + + /* Force all pending ordered list requests to time out. */ + lio_process_ordered_list(oct, 1); + + /* We do not need to wait for output queue packets to be processed. */ +} + +/** + * \brief Cleanup PCI AER uncorrectable error status + * @param dev Pointer to PCI device + */ +static void cleanup_aer_uncorrect_error_status(struct pci_dev *dev) +{ + int pos = 0x100; + u32 status, mask; + + pr_info("%s :\n", __func__); + + pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status); + pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_SEVER, &mask); + if (dev->error_state == pci_channel_io_normal) + status &= ~mask; /* Clear corresponding nonfatal bits */ + else + status &= mask; /* Clear corresponding fatal bits */ + pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, status); +} + +/** + * \brief Stop all PCI IO to a given device + * @param dev Pointer to Octeon device + */ +static void stop_pci_io(struct octeon_device *oct) +{ + /* No more instructions will be forwarded. */ + atomic_set(&oct->status, OCT_DEV_IN_RESET); + + pci_disable_device(oct->pci_dev); + + /* Disable interrupts */ + oct->fn_list.disable_interrupt(oct->chip); + + pcierror_quiesce_device(oct); + + /* Release the interrupt line */ + free_irq(oct->pci_dev->irq, oct); + + if (oct->flags & LIO_FLAG_MSI_ENABLED) + pci_disable_msi(oct->pci_dev); + + dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n", + lio_get_state_string(&oct->status)); + + /* cn63xx_cleanup_aer_uncorrect_error_status(oct->pci_dev); */ + /* making it a common function for all OCTEON models */ + cleanup_aer_uncorrect_error_status(oct->pci_dev); +} + +/** + * \brief called when PCI error is detected + * @param pdev Pointer to PCI device + * @param state The current pci connection state + * + * This function is called after a PCI bus error affecting + * this device has been detected. + */ +static pci_ers_result_t liquidio_pcie_error_detected(struct pci_dev *pdev, + pci_channel_state_t state) +{ + struct octeon_device *oct = pci_get_drvdata(pdev); + + /* Non-correctable Non-fatal errors */ + if (state == pci_channel_io_normal) { + dev_err(&oct->pci_dev->dev, "Non-correctable non-fatal error reported:\n"); + cleanup_aer_uncorrect_error_status(oct->pci_dev); + return PCI_ERS_RESULT_CAN_RECOVER; + } + + /* Non-correctable Fatal errors */ + dev_err(&oct->pci_dev->dev, "Non-correctable FATAL reported by PCI AER driver\n"); + stop_pci_io(oct); + + /* Always return a DISCONNECT. There is no support for recovery but only + * for a clean shutdown. + */ + return PCI_ERS_RESULT_DISCONNECT; +} + +/** + * \brief mmio handler + * @param pdev Pointer to PCI device + */ +static pci_ers_result_t liquidio_pcie_mmio_enabled(struct pci_dev *pdev) +{ + /* We should never hit this since we never ask for a reset for a Fatal + * Error. We always return DISCONNECT in io_error above. + * But play safe and return RECOVERED for now. + */ + return PCI_ERS_RESULT_RECOVERED; +} + +/** + * \brief called after the pci bus has been reset. + * @param pdev Pointer to PCI device + * + * Restart the card from scratch, as if from a cold-boot. Implementation + * resembles the first-half of the octeon_resume routine. + */ +static pci_ers_result_t liquidio_pcie_slot_reset(struct pci_dev *pdev) +{ + /* We should never hit this since we never ask for a reset for a Fatal + * Error. We always return DISCONNECT in io_error above. + * But play safe and return RECOVERED for now. + */ + return PCI_ERS_RESULT_RECOVERED; +} + +/** + * \brief called when traffic can start flowing again. + * @param pdev Pointer to PCI device + * + * This callback is called when the error recovery driver tells us that + * its OK to resume normal operation. Implementation resembles the + * second-half of the octeon_resume routine. + */ +static void liquidio_pcie_resume(struct pci_dev *pdev) +{ + /* Nothing to be done here. */ +} + +#ifdef CONFIG_PM +/** + * \brief called when suspending + * @param pdev Pointer to PCI device + * @param state state to suspend to + */ +static int liquidio_suspend(struct pci_dev *pdev, pm_message_t state) +{ + return 0; +} + +/** + * \brief called when resuming + * @param pdev Pointer to PCI device + */ +static int liquidio_resume(struct pci_dev *pdev) +{ + return 0; +} +#endif + +/* For PCI-E Advanced Error Recovery (AER) Interface */ +static const struct pci_error_handlers liquidio_err_handler = { + .error_detected = liquidio_pcie_error_detected, + .mmio_enabled = liquidio_pcie_mmio_enabled, + .slot_reset = liquidio_pcie_slot_reset, + .resume = liquidio_pcie_resume, +}; + +static const struct pci_device_id liquidio_pci_tbl[] = { + { /* 68xx */ + PCI_VENDOR_ID_CAVIUM, 0x91, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 + }, + { /* 66xx */ + PCI_VENDOR_ID_CAVIUM, 0x92, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 + }, + { + 0, 0, 0, 0, 0, 0, 0 + } +}; +MODULE_DEVICE_TABLE(pci, liquidio_pci_tbl); + +static struct pci_driver liquidio_pci_driver = { + .name = "LiquidIO", + .id_table = liquidio_pci_tbl, + .probe = liquidio_probe, + .remove = liquidio_remove, + .err_handler = &liquidio_err_handler, /* For AER */ + +#ifdef CONFIG_PM + .suspend = liquidio_suspend, + .resume = liquidio_resume, +#endif + +}; + +/** + * \brief register PCI driver + */ +static int liquidio_init_pci(void) +{ + return pci_register_driver(&liquidio_pci_driver); +} + +/** + * \brief unregister PCI driver + */ +static void liquidio_deinit_pci(void) +{ + pci_unregister_driver(&liquidio_pci_driver); +} + +/** + * \brief check interface state + * @param lio per-network private data + * @param state_flag flag state to check + */ +static inline int ifstate_check(struct lio *lio, int state_flag) +{ + return atomic_read(&lio->ifstate) & state_flag; +} + +/** + * \brief set interface state + * @param lio per-network private data + * @param state_flag flag state to set + */ +static inline void ifstate_set(struct lio *lio, int state_flag) +{ + atomic_set(&lio->ifstate, (atomic_read(&lio->ifstate) | state_flag)); +} + +/** + * \brief clear interface state + * @param lio per-network private data + * @param state_flag flag state to clear + */ +static inline void ifstate_reset(struct lio *lio, int state_flag) +{ + atomic_set(&lio->ifstate, (atomic_read(&lio->ifstate) & ~(state_flag))); +} + +/** + * \brief Stop Tx queues + * @param netdev network device + */ +static inline void txqs_stop(struct net_device *netdev) +{ + if (netif_is_multiqueue(netdev)) { + int i; + + for (i = 0; i < netdev->num_tx_queues; i++) + netif_stop_subqueue(netdev, i); + } else { + netif_stop_queue(netdev); + } +} + +/** + * \brief Start Tx queues + * @param netdev network device + */ +static inline void txqs_start(struct net_device *netdev) +{ + if (netif_is_multiqueue(netdev)) { + int i; + + for (i = 0; i < netdev->num_tx_queues; i++) + netif_start_subqueue(netdev, i); + } else { + netif_start_queue(netdev); + } +} + +/** + * \brief Wake Tx queues + * @param netdev network device + */ +static inline void txqs_wake(struct net_device *netdev) +{ + if (netif_is_multiqueue(netdev)) { + int i; + + for (i = 0; i < netdev->num_tx_queues; i++) + netif_wake_subqueue(netdev, i); + } else { + netif_wake_queue(netdev); + } +} + +/** + * \brief Stop Tx queue + * @param netdev network device + */ +static void stop_txq(struct net_device *netdev) +{ + txqs_stop(netdev); +} + +/** + * \brief Start Tx queue + * @param netdev network device + */ +static void start_txq(struct net_device *netdev) +{ + struct lio *lio = GET_LIO(netdev); + + if (lio->linfo.link.s.status) { + txqs_start(netdev); + return; + } +} + +/** + * \brief Wake a queue + * @param netdev network device + * @param q which queue to wake + */ +static inline void wake_q(struct net_device *netdev, int q) +{ + if (netif_is_multiqueue(netdev)) + netif_wake_subqueue(netdev, q); + else + netif_wake_queue(netdev); +} + +/** + * \brief Stop a queue + * @param netdev network device + * @param q which queue to stop + */ +static inline void stop_q(struct net_device *netdev, int q) +{ + if (netif_is_multiqueue(netdev)) + netif_stop_subqueue(netdev, q); + else + netif_stop_queue(netdev); +} + +/** + * \brief Check Tx queue status, and take appropriate action + * @param lio per-network private data + * @returns 0 if full, number of queues woken up otherwise + */ +static inline int check_txq_status(struct lio *lio) +{ + int ret_val = 0; + + if (netif_is_multiqueue(lio->netdev)) { + int numqs = lio->netdev->num_tx_queues; + int q, iq = 0; + + /* check each sub-queue state */ + for (q = 0; q < numqs; q++) { + iq = lio->linfo.txpciq[q & (lio->linfo.num_txpciq - 1)]; + if (octnet_iq_is_full(lio->oct_dev, iq)) + continue; + wake_q(lio->netdev, q); + ret_val++; + } + } else { + if (octnet_iq_is_full(lio->oct_dev, lio->txq)) + return 0; + wake_q(lio->netdev, lio->txq); + ret_val = 1; + } + return ret_val; +} + +/** + * Remove the node at the head of the list. The list would be empty at + * the end of this call if there are no more nodes in the list. + */ +static inline struct list_head *list_delete_head(struct list_head *root) +{ + struct list_head *node; + + if ((root->prev == root) && (root->next == root)) + node = NULL; + else + node = root->next; + + if (node) + list_del(node); + + return node; +} + +/** + * \brief Delete gather list + * @param lio per-network private data + */ +static void delete_glist(struct lio *lio) +{ + struct octnic_gather *g; + + do { + g = (struct octnic_gather *) + list_delete_head(&lio->glist); + if (g) { + if (g->sg) + kfree((void *)((unsigned long)g->sg - + g->adjust)); + kfree(g); + } + } while (g); +} + +/** + * \brief Setup gather list + * @param lio per-network private data + */ +static int setup_glist(struct lio *lio) +{ + int i; + struct octnic_gather *g; + + INIT_LIST_HEAD(&lio->glist); + + for (i = 0; i < lio->tx_qsize; i++) { + g = kzalloc(sizeof(*g), GFP_KERNEL); + if (!g) + break; + + g->sg_size = + ((ROUNDUP4(OCTNIC_MAX_SG) >> 2) * OCT_SG_ENTRY_SIZE); + + g->sg = kmalloc(g->sg_size + 8, GFP_KERNEL); + if (!g->sg) { + kfree(g); + break; + } + + /* The gather component should be aligned on 64-bit boundary */ + if (((unsigned long)g->sg) & 7) { + g->adjust = 8 - (((unsigned long)g->sg) & 7); + g->sg = (struct octeon_sg_entry *) + ((unsigned long)g->sg + g->adjust); + } + list_add_tail(&g->list, &lio->glist); + } + + if (i == lio->tx_qsize) + return 0; + + delete_glist(lio); + return 1; +} + +/** + * \brief Print link information + * @param netdev network device + */ +static void print_link_info(struct net_device *netdev) +{ + struct lio *lio = GET_LIO(netdev); + + if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED) { + struct oct_link_info *linfo = &lio->linfo; + + if (linfo->link.s.status) { + netif_info(lio, link, lio->netdev, "%d Mbps %s Duplex UP\n", + linfo->link.s.speed, + (linfo->link.s.duplex) ? "Full" : "Half"); + } else { + netif_info(lio, link, lio->netdev, "Link Down\n"); + } + } +} + +/** + * \brief Update link status + * @param netdev network device + * @param ls link status structure + * + * Called on receipt of a link status response from the core application to + * update each interface's link status. + */ +static inline void update_link_status(struct net_device *netdev, + union oct_link_status *ls) +{ + struct lio *lio = GET_LIO(netdev); + + if ((lio->intf_open) && (lio->linfo.link.u64 != ls->u64)) { + lio->linfo.link.u64 = ls->u64; + + print_link_info(netdev); + + if (lio->linfo.link.s.status) { + netif_carrier_on(netdev); + /* start_txq(netdev); */ + txqs_wake(netdev); + } else { + netif_carrier_off(netdev); + stop_txq(netdev); + } + } +} + +/** + * \brief Droq packet processor sceduler + * @param oct octeon device + */ +static +void liquidio_schedule_droq_pkt_handlers(struct octeon_device *oct) +{ + struct octeon_device_priv *oct_priv = + (struct octeon_device_priv *)oct->priv; + u64 oq_no; + struct octeon_droq *droq; + + if (oct->int_status & OCT_DEV_INTR_PKT_DATA) { + for (oq_no = 0; oq_no < MAX_OCTEON_OUTPUT_QUEUES; oq_no++) { + if (!(oct->droq_intr & (1 << oq_no))) + continue; + + droq = oct->droq[oq_no]; + + if (droq->ops.poll_mode) { + droq->ops.napi_fn(droq); + oct_priv->napi_mask |= (1 << oq_no); + } else { + tasklet_schedule(&oct_priv->droq_tasklet); + } + } + } +} + +/** + * \brief Interrupt handler for octeon + * @param irq unused + * @param dev octeon device + */ +static +irqreturn_t liquidio_intr_handler(int irq __attribute__((unused)), void *dev) +{ + struct octeon_device *oct = (struct octeon_device *)dev; + irqreturn_t ret; + + /* Disable our interrupts for the duration of ISR */ + oct->fn_list.disable_interrupt(oct->chip); + + ret = oct->fn_list.process_interrupt_regs(oct); + + if (ret == IRQ_HANDLED) + liquidio_schedule_droq_pkt_handlers(oct); + + /* Re-enable our interrupts */ + if (!(atomic_read(&oct->status) == OCT_DEV_IN_RESET)) + oct->fn_list.enable_interrupt(oct->chip); + + return ret; +} + +/** + * \brief Setup interrupt for octeon device + * @param oct octeon device + * + * Enable interrupt in Octeon device as given in the PCI interrupt mask. + */ +static int octeon_setup_interrupt(struct octeon_device *oct) +{ + int irqret, err; + + err = pci_enable_msi(oct->pci_dev); + if (err) + dev_warn(&oct->pci_dev->dev, "Reverting to legacy interrupts. Error: %d\n", + err); + else + oct->flags |= LIO_FLAG_MSI_ENABLED; + + irqret = request_irq(oct->pci_dev->irq, liquidio_intr_handler, + IRQF_SHARED, "octeon", oct); + if (irqret) { + if (oct->flags & LIO_FLAG_MSI_ENABLED) + pci_disable_msi(oct->pci_dev); + dev_err(&oct->pci_dev->dev, "Request IRQ failed with code: %d\n", + irqret); + return 1; + } + + return 0; +} + +/** + * \brief PCI probe handler + * @param pdev PCI device structure + * @param ent unused + */ +static int liquidio_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct octeon_device *oct_dev = NULL; + struct handshake *hs; + + oct_dev = octeon_allocate_device(pdev->device, + sizeof(struct octeon_device_priv)); + if (!oct_dev) { + dev_err(&pdev->dev, "Unable to allocate device\n"); + return -ENOMEM; + } + + dev_info(&pdev->dev, "Initializing device %x:%x.\n", + (u32)pdev->vendor, (u32)pdev->device); + + /* Assign octeon_device for this device to the private data area. */ + pci_set_drvdata(pdev, oct_dev); + + /* set linux specific device pointer */ + oct_dev->pci_dev = (void *)pdev; + + hs = &handshake[oct_dev->octeon_id]; + init_completion(&hs->init); + init_completion(&hs->started); + hs->pci_dev = pdev; + + if (oct_dev->octeon_id == 0) + /* first LiquidIO NIC is detected */ + complete(&first_stage); + + if (octeon_device_init(oct_dev)) { + liquidio_remove(pdev); + return -ENOMEM; + } + + dev_dbg(&oct_dev->pci_dev->dev, "Device is ready\n"); + + return 0; +} + +/** + *\brief Destroy resources associated with octeon device + * @param pdev PCI device structure + * @param ent unused + */ +static void octeon_destroy_resources(struct octeon_device *oct) +{ + int i; + struct octeon_device_priv *oct_priv = + (struct octeon_device_priv *)oct->priv; + + struct handshake *hs; + + switch (atomic_read(&oct->status)) { + case OCT_DEV_RUNNING: + case OCT_DEV_CORE_OK: + + /* No more instructions will be forwarded. */ + atomic_set(&oct->status, OCT_DEV_IN_RESET); + + oct->app_mode = CVM_DRV_INVALID_APP; + dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n", + lio_get_state_string(&oct->status)); + + schedule_timeout_uninterruptible(HZ / 10); + + /* fallthrough */ + case OCT_DEV_HOST_OK: + + /* fallthrough */ + case OCT_DEV_CONSOLE_INIT_DONE: + /* Remove any consoles */ + octeon_remove_consoles(oct); + + /* fallthrough */ + case OCT_DEV_IO_QUEUES_DONE: + if (wait_for_pending_requests(oct)) + dev_err(&oct->pci_dev->dev, "There were pending requests\n"); + + if (lio_wait_for_instr_fetch(oct)) + dev_err(&oct->pci_dev->dev, "IQ had pending instructions\n"); + + /* Disable the input and output queues now. No more packets will + * arrive from Octeon, but we should wait for all packet + * processing to finish. + */ + oct->fn_list.disable_io_queues(oct); + + if (lio_wait_for_oq_pkts(oct)) + dev_err(&oct->pci_dev->dev, "OQ had pending packets\n"); + + /* Disable interrupts */ + oct->fn_list.disable_interrupt(oct->chip); + + /* Release the interrupt line */ + free_irq(oct->pci_dev->irq, oct); + + if (oct->flags & LIO_FLAG_MSI_ENABLED) + pci_disable_msi(oct->pci_dev); + + /* Soft reset the octeon device before exiting */ + oct->fn_list.soft_reset(oct); + + /* Disable the device, releasing the PCI INT */ + pci_disable_device(oct->pci_dev); + + /* fallthrough */ + case OCT_DEV_IN_RESET: + case OCT_DEV_DROQ_INIT_DONE: + /*atomic_set(&oct->status, OCT_DEV_DROQ_INIT_DONE);*/ + mdelay(100); + for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES; i++) { + if (!(oct->io_qmask.oq & (1UL << i))) + continue; + octeon_delete_droq(oct, i); + } + + /* Force any pending handshakes to complete */ + for (i = 0; i < MAX_OCTEON_DEVICES; i++) { + hs = &handshake[i]; + + if (hs->pci_dev) { + handshake[oct->octeon_id].init_ok = 0; + complete(&handshake[oct->octeon_id].init); + handshake[oct->octeon_id].started_ok = 0; + complete(&handshake[oct->octeon_id].started); + } + } + + /* fallthrough */ + case OCT_DEV_RESP_LIST_INIT_DONE: + octeon_delete_response_list(oct); + + /* fallthrough */ + case OCT_DEV_SC_BUFF_POOL_INIT_DONE: + octeon_free_sc_buffer_pool(oct); + + /* fallthrough */ + case OCT_DEV_INSTR_QUEUE_INIT_DONE: + for (i = 0; i < MAX_OCTEON_INSTR_QUEUES; i++) { + if (!(oct->io_qmask.iq & (1UL << i))) + continue; + octeon_delete_instr_queue(oct, i); + } + + /* fallthrough */ + case OCT_DEV_DISPATCH_INIT_DONE: + octeon_delete_dispatch_list(oct); + cancel_delayed_work_sync(&oct->nic_poll_work.work); + + /* fallthrough */ + case OCT_DEV_PCI_MAP_DONE: + octeon_unmap_pci_barx(oct, 0); + octeon_unmap_pci_barx(oct, 1); + + /* fallthrough */ + case OCT_DEV_BEGIN_STATE: + /* Nothing to be done here either */ + break; + } /* end switch(oct->status) */ + + tasklet_kill(&oct_priv->droq_tasklet); +} + +/** + * \brief Send Rx control command + * @param lio per-network private data + * @param start_stop whether to start or stop + */ +static void send_rx_ctrl_cmd(struct lio *lio, int start_stop) +{ + struct octnic_ctrl_pkt nctrl; + struct octnic_ctrl_params nparams; + + memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt)); + + nctrl.ncmd.s.cmd = OCTNET_CMD_RX_CTL; + nctrl.ncmd.s.param1 = lio->linfo.ifidx; + nctrl.ncmd.s.param2 = start_stop; + nctrl.netpndev = (u64)lio->netdev; + + nparams.resp_order = OCTEON_RESP_NORESPONSE; + + if (octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl, nparams) < 0) + netif_info(lio, rx_err, lio->netdev, "Failed to send RX Control message\n"); +} + +/** + * \brief Destroy NIC device interface + * @param oct octeon device + * @param ifidx which interface to destroy + * + * Cleanup associated with each interface for an Octeon device when NIC + * module is being unloaded or if initialization fails during load. + */ +static void liquidio_destroy_nic_device(struct octeon_device *oct, int ifidx) +{ + struct net_device *netdev = oct->props[ifidx].netdev; + struct lio *lio; + + if (!netdev) { + dev_err(&oct->pci_dev->dev, "%s No netdevice ptr for index %d\n", + __func__, ifidx); + return; + } + + lio = GET_LIO(netdev); + + dev_dbg(&oct->pci_dev->dev, "NIC device cleanup\n"); + + send_rx_ctrl_cmd(lio, 0); + + if (atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING) + txqs_stop(netdev); + + if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED) + unregister_netdev(netdev); + + delete_glist(lio); + + free_netdev(netdev); + + oct->props[ifidx].netdev = NULL; +} + +/** + * \brief Stop complete NIC functionality + * @param oct octeon device + */ +static int liquidio_stop_nic_module(struct octeon_device *oct) +{ + int i, j; + struct lio *lio; + + dev_dbg(&oct->pci_dev->dev, "Stopping network interfaces\n"); + if (!oct->ifcount) { + dev_err(&oct->pci_dev->dev, "Init for Octeon was not completed\n"); + return 1; + } + + for (i = 0; i < oct->ifcount; i++) { + lio = GET_LIO(oct->props[i].netdev); + for (j = 0; j < lio->linfo.num_rxpciq; j++) + octeon_unregister_droq_ops(oct, lio->linfo.rxpciq[j]); + } + + for (i = 0; i < oct->ifcount; i++) + liquidio_destroy_nic_device(oct, i); + + dev_dbg(&oct->pci_dev->dev, "Network interfaces stopped\n"); + return 0; +} + +/** + * \brief Cleans up resources at unload time + * @param pdev PCI device structure + */ +static void liquidio_remove(struct pci_dev *pdev) +{ + struct octeon_device *oct_dev = pci_get_drvdata(pdev); + + dev_dbg(&oct_dev->pci_dev->dev, "Stopping device\n"); + + if (oct_dev->app_mode && (oct_dev->app_mode == CVM_DRV_NIC_APP)) + liquidio_stop_nic_module(oct_dev); + + /* Reset the octeon device and cleanup all memory allocated for + * the octeon device by driver. + */ + octeon_destroy_resources(oct_dev); + + dev_info(&oct_dev->pci_dev->dev, "Device removed\n"); + + /* This octeon device has been removed. Update the global + * data structure to reflect this. Free the device structure. + */ + octeon_free_device_mem(oct_dev); +} + +/** + * \brief Identify the Octeon device and to map the BAR address space + * @param oct octeon device + */ +static int octeon_chip_specific_setup(struct octeon_device *oct) +{ + u32 dev_id, rev_id; + int ret = 1; + + pci_read_config_dword(oct->pci_dev, 0, &dev_id); + pci_read_config_dword(oct->pci_dev, 8, &rev_id); + oct->rev_id = rev_id & 0xff; + + switch (dev_id) { + case OCTEON_CN68XX_PCIID: + oct->chip_id = OCTEON_CN68XX; + ret = lio_setup_cn68xx_octeon_device(oct); + break; + + case OCTEON_CN66XX_PCIID: + oct->chip_id = OCTEON_CN66XX; + ret = lio_setup_cn66xx_octeon_device(oct); + break; + default: + dev_err(&oct->pci_dev->dev, "Unknown device found (dev_id: %x)\n", + dev_id); + } + + if (!ret) + dev_info(&oct->pci_dev->dev, "CN68XX PASS%d.%d %s\n", + OCTEON_MAJOR_REV(oct), + OCTEON_MINOR_REV(oct), + octeon_get_conf(oct)->card_name); + + return ret; +} + +/** + * \brief PCI initialization for each Octeon device. + * @param oct octeon device + */ +static int octeon_pci_os_setup(struct octeon_device *oct) +{ + /* setup PCI stuff first */ + if (pci_enable_device(oct->pci_dev)) { + dev_err(&oct->pci_dev->dev, "pci_enable_device failed\n"); + return 1; + } + + if (dma_set_mask_and_coherent(&oct->pci_dev->dev, DMA_BIT_MASK(64))) { + dev_err(&oct->pci_dev->dev, "Unexpected DMA device capability\n"); + return 1; + } + + /* Enable PCI DMA Master. */ + pci_set_master(oct->pci_dev); + + return 0; +} + +/** + * \brief Check Tx queue state for a given network buffer + * @param lio per-network private data + * @param skb network buffer + */ +static inline int check_txq_state(struct lio *lio, struct sk_buff *skb) +{ + int q = 0, iq = 0; + + if (netif_is_multiqueue(lio->netdev)) { + q = skb->queue_mapping; + iq = lio->linfo.txpciq[(q & (lio->linfo.num_txpciq - 1))]; + } else { + iq = lio->txq; + } + + if (octnet_iq_is_full(lio->oct_dev, iq)) + return 0; + wake_q(lio->netdev, q); + return 1; +} + +/** + * \brief Unmap and free network buffer + * @param buf buffer + */ +static void free_netbuf(void *buf) +{ + struct sk_buff *skb; + struct octnet_buf_free_info *finfo; + struct lio *lio; + + finfo = (struct octnet_buf_free_info *)buf; + skb = finfo->skb; + lio = finfo->lio; + + dma_unmap_single(&lio->oct_dev->pci_dev->dev, finfo->dptr, skb->len, + DMA_TO_DEVICE); + + check_txq_state(lio, skb); + + recv_buffer_free((struct sk_buff *)skb); +} + +/** + * \brief Unmap and free gather buffer + * @param buf buffer + */ +static void free_netsgbuf(void *buf) +{ + struct octnet_buf_free_info *finfo; + struct sk_buff *skb; + struct lio *lio; + struct octnic_gather *g; + int i, frags; + + finfo = (struct octnet_buf_free_info *)buf; + skb = finfo->skb; + lio = finfo->lio; + g = finfo->g; + frags = skb_shinfo(skb)->nr_frags; + + dma_unmap_single(&lio->oct_dev->pci_dev->dev, + g->sg[0].ptr[0], (skb->len - skb->data_len), + DMA_TO_DEVICE); + + i = 1; + while (frags--) { + struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i - 1]; + + pci_unmap_page((lio->oct_dev)->pci_dev, + g->sg[(i >> 2)].ptr[(i & 3)], + frag->size, DMA_TO_DEVICE); + i++; + } + + dma_unmap_single(&lio->oct_dev->pci_dev->dev, + finfo->dptr, g->sg_size, + DMA_TO_DEVICE); + + spin_lock(&lio->lock); + list_add_tail(&g->list, &lio->glist); + spin_unlock(&lio->lock); + + check_txq_state(lio, skb); /* mq support: sub-queue state check */ + + recv_buffer_free((struct sk_buff *)skb); +} + +/** + * \brief Unmap and free gather buffer with response + * @param buf buffer + */ +static void free_netsgbuf_with_resp(void *buf) +{ + struct octeon_soft_command *sc; + struct octnet_buf_free_info *finfo; + struct sk_buff *skb; + struct lio *lio; + struct octnic_gather *g; + int i, frags; + + sc = (struct octeon_soft_command *)buf; + skb = (struct sk_buff *)sc->callback_arg; + finfo = (struct octnet_buf_free_info *)&skb->cb; + + lio = finfo->lio; + g = finfo->g; + frags = skb_shinfo(skb)->nr_frags; + + dma_unmap_single(&lio->oct_dev->pci_dev->dev, + g->sg[0].ptr[0], (skb->len - skb->data_len), + DMA_TO_DEVICE); + + i = 1; + while (frags--) { + struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i - 1]; + + pci_unmap_page((lio->oct_dev)->pci_dev, + g->sg[(i >> 2)].ptr[(i & 3)], + frag->size, DMA_TO_DEVICE); + i++; + } + + dma_unmap_single(&lio->oct_dev->pci_dev->dev, + finfo->dptr, g->sg_size, + DMA_TO_DEVICE); + + spin_lock(&lio->lock); + list_add_tail(&g->list, &lio->glist); + spin_unlock(&lio->lock); + + /* Don't free the skb yet */ + + check_txq_state(lio, skb); +} + +/** + * \brief Adjust ptp frequency + * @param ptp PTP clock info + * @param ppb how much to adjust by, in parts-per-billion + */ +static int liquidio_ptp_adjfreq(struct ptp_clock_info *ptp, s32 ppb) +{ + struct lio *lio = container_of(ptp, struct lio, ptp_info); + struct octeon_device *oct = (struct octeon_device *)lio->oct_dev; + u64 comp, delta; + unsigned long flags; + bool neg_adj = false; + + if (ppb < 0) { + neg_adj = true; + ppb = -ppb; + } + + /* The hardware adds the clock compensation value to the + * PTP clock on every coprocessor clock cycle, so we + * compute the delta in terms of coprocessor clocks. + */ + delta = (u64)ppb << 32; + do_div(delta, oct->coproc_clock_rate); + + spin_lock_irqsave(&lio->ptp_lock, flags); + comp = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_COMP); + if (neg_adj) + comp -= delta; + else + comp += delta; + lio_pci_writeq(oct, comp, CN6XXX_MIO_PTP_CLOCK_COMP); + spin_unlock_irqrestore(&lio->ptp_lock, flags); + + return 0; +} + +/** + * \brief Adjust ptp time + * @param ptp PTP clock info + * @param delta how much to adjust by, in nanosecs + */ +static int liquidio_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta) +{ + unsigned long flags; + struct lio *lio = container_of(ptp, struct lio, ptp_info); + + spin_lock_irqsave(&lio->ptp_lock, flags); + lio->ptp_adjust += delta; + spin_unlock_irqrestore(&lio->ptp_lock, flags); + + return 0; +} + +/** + * \brief Get hardware clock time, including any adjustment + * @param ptp PTP clock info + * @param ts timespec + */ +static int liquidio_ptp_gettime(struct ptp_clock_info *ptp, + struct timespec64 *ts) +{ + u64 ns; + u32 remainder; + unsigned long flags; + struct lio *lio = container_of(ptp, struct lio, ptp_info); + struct octeon_device *oct = (struct octeon_device *)lio->oct_dev; + + spin_lock_irqsave(&lio->ptp_lock, flags); + ns = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_HI); + ns += lio->ptp_adjust; + spin_unlock_irqrestore(&lio->ptp_lock, flags); + + ts->tv_sec = div_u64_rem(ns, 1000000000ULL, &remainder); + ts->tv_nsec = remainder; + + return 0; +} + +/** + * \brief Set hardware clock time. Reset adjustment + * @param ptp PTP clock info + * @param ts timespec + */ +static int liquidio_ptp_settime(struct ptp_clock_info *ptp, + const struct timespec64 *ts) +{ + u64 ns; + unsigned long flags; + struct lio *lio = container_of(ptp, struct lio, ptp_info); + struct octeon_device *oct = (struct octeon_device *)lio->oct_dev; + + ns = timespec_to_ns(ts); + + spin_lock_irqsave(&lio->ptp_lock, flags); + lio_pci_writeq(oct, ns, CN6XXX_MIO_PTP_CLOCK_HI); + lio->ptp_adjust = 0; + spin_unlock_irqrestore(&lio->ptp_lock, flags); + + return 0; +} + +/** + * \brief Check if PTP is enabled + * @param ptp PTP clock info + * @param rq request + * @param on is it on + */ +static int liquidio_ptp_enable(struct ptp_clock_info *ptp, + struct ptp_clock_request *rq, int on) +{ + return -EOPNOTSUPP; +} + +/** + * \brief Open PTP clock source + * @param netdev network device + */ +static void oct_ptp_open(struct net_device *netdev) +{ + struct lio *lio = GET_LIO(netdev); + struct octeon_device *oct = (struct octeon_device *)lio->oct_dev; + + spin_lock_init(&lio->ptp_lock); + + snprintf(lio->ptp_info.name, 16, "%s", netdev->name); + lio->ptp_info.owner = THIS_MODULE; + lio->ptp_info.max_adj = 250000000; + lio->ptp_info.n_alarm = 0; + lio->ptp_info.n_ext_ts = 0; + lio->ptp_info.n_per_out = 0; + lio->ptp_info.pps = 0; + lio->ptp_info.adjfreq = liquidio_ptp_adjfreq; + lio->ptp_info.adjtime = liquidio_ptp_adjtime; + lio->ptp_info.gettime64 = liquidio_ptp_gettime; + lio->ptp_info.settime64 = liquidio_ptp_settime; + lio->ptp_info.enable = liquidio_ptp_enable; + + lio->ptp_adjust = 0; + + lio->ptp_clock = ptp_clock_register(&lio->ptp_info, + &oct->pci_dev->dev); + + if (IS_ERR(lio->ptp_clock)) + lio->ptp_clock = NULL; +} + +/** + * \brief Init PTP clock + * @param oct octeon device + */ +static void liquidio_ptp_init(struct octeon_device *oct) +{ + u64 clock_comp, cfg; + + clock_comp = (u64)NSEC_PER_SEC << 32; + do_div(clock_comp, oct->coproc_clock_rate); + lio_pci_writeq(oct, clock_comp, CN6XXX_MIO_PTP_CLOCK_COMP); + + /* Enable */ + cfg = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_CFG); + lio_pci_writeq(oct, cfg | 0x01, CN6XXX_MIO_PTP_CLOCK_CFG); +} + +/** + * \brief Load firmware to device + * @param oct octeon device + * + * Maps device to firmware filename, requests firmware, and downloads it + */ +static int load_firmware(struct octeon_device *oct) +{ + int ret = 0; + const struct firmware *fw; + char fw_name[LIO_MAX_FW_FILENAME_LEN]; + char *tmp_fw_type; + + if (strncmp(fw_type, LIO_FW_NAME_TYPE_NONE, + sizeof(LIO_FW_NAME_TYPE_NONE)) == 0) { + dev_info(&oct->pci_dev->dev, "Skipping firmware load\n"); + return ret; + } + + if (fw_type[0] == '\0') + tmp_fw_type = LIO_FW_NAME_TYPE_NIC; + else + tmp_fw_type = fw_type; + + sprintf(fw_name, "%s%s%s_%s%s", LIO_FW_DIR, LIO_FW_BASE_NAME, + octeon_get_conf(oct)->card_name, tmp_fw_type, + LIO_FW_NAME_SUFFIX); + + ret = request_firmware(&fw, fw_name, &oct->pci_dev->dev); + if (ret) { + dev_err(&oct->pci_dev->dev, "Request firmware failed. Could not find file %s.\n.", + fw_name); + return ret; + } + + ret = octeon_download_firmware(oct, fw->data, fw->size); + + release_firmware(fw); + + return ret; +} + +/** + * \brief Setup output queue + * @param oct octeon device + * @param q_no which queue + * @param num_descs how many descriptors + * @param desc_size size of each descriptor + * @param app_ctx application context + */ +static int octeon_setup_droq(struct octeon_device *oct, int q_no, int num_descs, + int desc_size, void *app_ctx) +{ + int ret_val = 0; + + dev_dbg(&oct->pci_dev->dev, "Creating Droq: %d\n", q_no); + /* droq creation and local register settings. */ + ret_val = octeon_create_droq(oct, q_no, num_descs, desc_size, app_ctx); + if (ret_val == -1) + return ret_val; + + if (ret_val == 1) { + dev_dbg(&oct->pci_dev->dev, "Using default droq %d\n", q_no); + return 0; + } + /* tasklet creation for the droq */ + + /* Enable the droq queues */ + octeon_set_droq_pkt_op(oct, q_no, 1); + + /* Send Credit for Octeon Output queues. Credits are always + * sent after the output queue is enabled. + */ + writel(oct->droq[q_no]->max_count, + oct->droq[q_no]->pkts_credit_reg); + + return ret_val; +} + +/** + * \brief Callback for getting interface configuration + * @param status status of request + * @param buf pointer to resp structure + */ +static void if_cfg_callback(struct octeon_device *oct, + u32 status, + void *buf) +{ + struct octeon_soft_command *sc = (struct octeon_soft_command *)buf; + struct liquidio_if_cfg_resp *resp; + struct liquidio_if_cfg_context *ctx; + + resp = (struct liquidio_if_cfg_resp *)sc->virtrptr; + ctx = (struct liquidio_if_cfg_context *)sc->ctxptr; + + oct = lio_get_device(ctx->octeon_id); + if (resp->status) + dev_err(&oct->pci_dev->dev, "nic if cfg instruction failed. Status: %llx\n", + CVM_CAST64(resp->status)); + ACCESS_ONCE(ctx->cond) = 1; + + /* This barrier is required to be sure that the response has been + * written fully before waking up the handler + */ + wmb(); + + wake_up_interruptible(&ctx->wc); +} + +/** + * \brief Select queue based on hash + * @param dev Net device + * @param skb sk_buff structure + * @returns selected queue number + */ +static u16 select_q(struct net_device *dev, struct sk_buff *skb, + void *accel_priv, select_queue_fallback_t fallback) +{ + int qindex; + struct lio *lio; + + lio = GET_LIO(dev); + /* select queue on chosen queue_mapping or core */ + qindex = skb_rx_queue_recorded(skb) ? + skb_get_rx_queue(skb) : smp_processor_id(); + return (u16)(qindex & (lio->linfo.num_txpciq - 1)); +} + +/** Routine to push packets arriving on Octeon interface upto network layer. + * @param oct_id - octeon device id. + * @param skbuff - skbuff struct to be passed to network layer. + * @param len - size of total data received. + * @param rh - Control header associated with the packet + * @param param - additional control data with the packet + */ +static void +liquidio_push_packet(u32 octeon_id, + void *skbuff, + u32 len, + union octeon_rh *rh, + void *param) +{ + struct napi_struct *napi = param; + struct octeon_device *oct = lio_get_device(octeon_id); + struct sk_buff *skb = (struct sk_buff *)skbuff; + struct skb_shared_hwtstamps *shhwtstamps; + u64 ns; + struct net_device *netdev = + (struct net_device *)oct->props[rh->r_dh.link].netdev; + struct octeon_droq *droq = container_of(param, struct octeon_droq, + napi); + if (netdev) { + int packet_was_received; + struct lio *lio = GET_LIO(netdev); + + /* Do not proceed if the interface is not in RUNNING state. */ + if (!ifstate_check(lio, LIO_IFSTATE_RUNNING)) { + recv_buffer_free(skb); + droq->stats.rx_dropped++; + return; + } + + skb->dev = netdev; + + if (rh->r_dh.has_hwtstamp) { + /* timestamp is included from the hardware at the + * beginning of the packet. + */ + if (ifstate_check(lio, + LIO_IFSTATE_RX_TIMESTAMP_ENABLED)) { + /* Nanoseconds are in the first 64-bits + * of the packet. + */ + memcpy(&ns, (skb->data), sizeof(ns)); + shhwtstamps = skb_hwtstamps(skb); + shhwtstamps->hwtstamp = + ns_to_ktime(ns + lio->ptp_adjust); + } + skb_pull(skb, sizeof(ns)); + } + + skb->protocol = eth_type_trans(skb, skb->dev); + + if ((netdev->features & NETIF_F_RXCSUM) && + (rh->r_dh.csum_verified == CNNIC_CSUM_VERIFIED)) + /* checksum has already been verified */ + skb->ip_summed = CHECKSUM_UNNECESSARY; + else + skb->ip_summed = CHECKSUM_NONE; + + packet_was_received = napi_gro_receive(napi, skb) != GRO_DROP; + + if (packet_was_received) { + droq->stats.rx_bytes_received += len; + droq->stats.rx_pkts_received++; + netdev->last_rx = jiffies; + } else { + droq->stats.rx_dropped++; + netif_info(lio, rx_err, lio->netdev, + "droq:%d error rx_dropped:%llu\n", + droq->q_no, droq->stats.rx_dropped); + } + + } else { + recv_buffer_free(skb); + } +} + +/** + * \brief wrapper for calling napi_schedule + * @param param parameters to pass to napi_schedule + * + * Used when scheduling on different CPUs + */ +static void napi_schedule_wrapper(void *param) +{ + struct napi_struct *napi = param; + + napi_schedule(napi); +} + +/** + * \brief callback when receive interrupt occurs and we are in NAPI mode + * @param arg pointer to octeon output queue + */ +static void liquidio_napi_drv_callback(void *arg) +{ + struct octeon_droq *droq = arg; + int this_cpu = smp_processor_id(); + + if (droq->cpu_id == this_cpu) { + napi_schedule(&droq->napi); + } else { + struct call_single_data *csd = &droq->csd; + + csd->func = napi_schedule_wrapper; + csd->info = &droq->napi; + csd->flags = 0; + + smp_call_function_single_async(droq->cpu_id, csd); + } +} + +/** + * \brief Main NAPI poll function + * @param droq octeon output queue + * @param budget maximum number of items to process + */ +static int liquidio_napi_do_rx(struct octeon_droq *droq, int budget) +{ + int work_done; + struct lio *lio = GET_LIO(droq->napi.dev); + struct octeon_device *oct = lio->oct_dev; + + work_done = octeon_process_droq_poll_cmd(oct, droq->q_no, + POLL_EVENT_PROCESS_PKTS, + budget); + if (work_done < 0) { + netif_info(lio, rx_err, lio->netdev, + "Receive work_done < 0, rxq:%d\n", droq->q_no); + goto octnet_napi_finish; + } + + if (work_done > budget) + dev_err(&oct->pci_dev->dev, ">>>> %s work_done: %d budget: %d\n", + __func__, work_done, budget); + + return work_done; + +octnet_napi_finish: + napi_complete(&droq->napi); + octeon_process_droq_poll_cmd(oct, droq->q_no, POLL_EVENT_ENABLE_INTR, + 0); + return 0; +} + +/** + * \brief Entry point for NAPI polling + * @param napi NAPI structure + * @param budget maximum number of items to process + */ +static int liquidio_napi_poll(struct napi_struct *napi, int budget) +{ + struct octeon_droq *droq; + int work_done; + + droq = container_of(napi, struct octeon_droq, napi); + + work_done = liquidio_napi_do_rx(droq, budget); + + if (work_done < budget) { + napi_complete(napi); + octeon_process_droq_poll_cmd(droq->oct_dev, droq->q_no, + POLL_EVENT_ENABLE_INTR, 0); + return 0; + } + + return work_done; +} + +/** + * \brief Setup input and output queues + * @param octeon_dev octeon device + * @param net_device Net device + * + * Note: Queues are with respect to the octeon device. Thus + * an input queue is for egress packets, and output queues + * are for ingress packets. + */ +static inline int setup_io_queues(struct octeon_device *octeon_dev, + struct net_device *net_device) +{ + static int first_time = 1; + static struct octeon_droq_ops droq_ops; + static int cpu_id; + static int cpu_id_modulus; + struct octeon_droq *droq; + struct napi_struct *napi; + int q, q_no, retval = 0; + struct lio *lio; + int num_tx_descs; + + lio = GET_LIO(net_device); + if (first_time) { + first_time = 0; + memset(&droq_ops, 0, sizeof(struct octeon_droq_ops)); + + droq_ops.fptr = liquidio_push_packet; + + droq_ops.poll_mode = 1; + droq_ops.napi_fn = liquidio_napi_drv_callback; + cpu_id = 0; + cpu_id_modulus = num_present_cpus(); + } + + /* set up DROQs. */ + for (q = 0; q < lio->linfo.num_rxpciq; q++) { + q_no = lio->linfo.rxpciq[q]; + + retval = octeon_setup_droq(octeon_dev, q_no, + CFG_GET_NUM_RX_DESCS_NIC_IF + (octeon_get_conf(octeon_dev), + lio->ifidx), + CFG_GET_NUM_RX_BUF_SIZE_NIC_IF + (octeon_get_conf(octeon_dev), + lio->ifidx), NULL); + if (retval) { + dev_err(&octeon_dev->pci_dev->dev, + " %s : Runtime DROQ(RxQ) creation failed.\n", + __func__); + return 1; + } + + droq = octeon_dev->droq[q_no]; + napi = &droq->napi; + netif_napi_add(net_device, napi, liquidio_napi_poll, 64); + + /* designate a CPU for this droq */ + droq->cpu_id = cpu_id; + cpu_id++; + if (cpu_id >= cpu_id_modulus) + cpu_id = 0; + + octeon_register_droq_ops(octeon_dev, q_no, &droq_ops); + } + + /* set up IQs. */ + for (q = 0; q < lio->linfo.num_txpciq; q++) { + num_tx_descs = CFG_GET_NUM_TX_DESCS_NIC_IF(octeon_get_conf + (octeon_dev), + lio->ifidx); + retval = octeon_setup_iq(octeon_dev, lio->linfo.txpciq[q], + num_tx_descs, + netdev_get_tx_queue(net_device, q)); + if (retval) { + dev_err(&octeon_dev->pci_dev->dev, + " %s : Runtime IQ(TxQ) creation failed.\n", + __func__); + return 1; + } + } + + return 0; +} + +/** + * \brief Poll routine for checking transmit queue status + * @param work work_struct data structure + */ +static void octnet_poll_check_txq_status(struct work_struct *work) +{ + struct cavium_wk *wk = (struct cavium_wk *)work; + struct lio *lio = (struct lio *)wk->ctxptr; + + if (!ifstate_check(lio, LIO_IFSTATE_RUNNING)) + return; + + check_txq_status(lio); + queue_delayed_work(lio->txq_status_wq.wq, + &lio->txq_status_wq.wk.work, msecs_to_jiffies(1)); +} + +/** + * \brief Sets up the txq poll check + * @param netdev network device + */ +static inline void setup_tx_poll_fn(struct net_device *netdev) +{ + struct lio *lio = GET_LIO(netdev); + struct octeon_device *oct = lio->oct_dev; + + lio->txq_status_wq.wq = create_workqueue("txq-status"); + if (!lio->txq_status_wq.wq) { + dev_err(&oct->pci_dev->dev, "unable to create cavium txq status wq\n"); + return; + } + INIT_DELAYED_WORK(&lio->txq_status_wq.wk.work, + octnet_poll_check_txq_status); + lio->txq_status_wq.wk.ctxptr = lio; + queue_delayed_work(lio->txq_status_wq.wq, + &lio->txq_status_wq.wk.work, msecs_to_jiffies(1)); +} + +/** + * \brief Net device open for LiquidIO + * @param netdev network device + */ +static int liquidio_open(struct net_device *netdev) +{ + struct lio *lio = GET_LIO(netdev); + struct octeon_device *oct = lio->oct_dev; + struct napi_struct *napi, *n; + + list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list) + napi_enable(napi); + + oct_ptp_open(netdev); + + ifstate_set(lio, LIO_IFSTATE_RUNNING); + setup_tx_poll_fn(netdev); + start_txq(netdev); + + netif_info(lio, ifup, lio->netdev, "Interface Open, ready for traffic\n"); + try_module_get(THIS_MODULE); + + /* tell Octeon to start forwarding packets to host */ + send_rx_ctrl_cmd(lio, 1); + + /* Ready for link status updates */ + lio->intf_open = 1; + + dev_info(&oct->pci_dev->dev, "%s interface is opened\n", + netdev->name); + + return 0; +} + +/** + * \brief Net device stop for LiquidIO + * @param netdev network device + */ +static int liquidio_stop(struct net_device *netdev) +{ + struct napi_struct *napi, *n; + struct lio *lio = GET_LIO(netdev); + struct octeon_device *oct = lio->oct_dev; + + netif_info(lio, ifdown, lio->netdev, "Stopping interface!\n"); + /* Inform that netif carrier is down */ + lio->intf_open = 0; + lio->linfo.link.s.status = 0; + + netif_carrier_off(netdev); + + /* tell Octeon to stop forwarding packets to host */ + send_rx_ctrl_cmd(lio, 0); + + cancel_delayed_work_sync(&lio->txq_status_wq.wk.work); + flush_workqueue(lio->txq_status_wq.wq); + destroy_workqueue(lio->txq_status_wq.wq); + + if (lio->ptp_clock) { + ptp_clock_unregister(lio->ptp_clock); + lio->ptp_clock = NULL; + } + + ifstate_reset(lio, LIO_IFSTATE_RUNNING); + + /* This is a hack that allows DHCP to continue working. */ + set_bit(__LINK_STATE_START, &lio->netdev->state); + + list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list) + napi_disable(napi); + + txqs_stop(netdev); + + dev_info(&oct->pci_dev->dev, "%s interface is stopped\n", netdev->name); + module_put(THIS_MODULE); + + return 0; +} + +void liquidio_link_ctrl_cmd_completion(void *nctrl_ptr) +{ + struct octnic_ctrl_pkt *nctrl = (struct octnic_ctrl_pkt *)nctrl_ptr; + struct net_device *netdev = (struct net_device *)nctrl->netpndev; + struct lio *lio = GET_LIO(netdev); + struct octeon_device *oct = lio->oct_dev; + + switch (nctrl->ncmd.s.cmd) { + case OCTNET_CMD_CHANGE_DEVFLAGS: + case OCTNET_CMD_SET_MULTI_LIST: + break; + + case OCTNET_CMD_CHANGE_MACADDR: + /* If command is successful, change the MACADDR. */ + netif_info(lio, probe, lio->netdev, " MACAddr changed to 0x%llx\n", + CVM_CAST64(nctrl->udd[0])); + dev_info(&oct->pci_dev->dev, "%s MACAddr changed to 0x%llx\n", + netdev->name, CVM_CAST64(nctrl->udd[0])); + memcpy(netdev->dev_addr, ((u8 *)&nctrl->udd[0]) + 2, ETH_ALEN); + break; + + case OCTNET_CMD_CHANGE_MTU: + /* If command is successful, change the MTU. */ + netif_info(lio, probe, lio->netdev, " MTU Changed from %d to %d\n", + netdev->mtu, nctrl->ncmd.s.param2); + dev_info(&oct->pci_dev->dev, "%s MTU Changed from %d to %d\n", + netdev->name, netdev->mtu, + nctrl->ncmd.s.param2); + netdev->mtu = nctrl->ncmd.s.param2; + break; + + case OCTNET_CMD_GPIO_ACCESS: + netif_info(lio, probe, lio->netdev, "LED Flashing visual identification\n"); + + break; + + case OCTNET_CMD_LRO_ENABLE: + dev_info(&oct->pci_dev->dev, "%s LRO Enabled\n", netdev->name); + break; + + case OCTNET_CMD_LRO_DISABLE: + dev_info(&oct->pci_dev->dev, "%s LRO Disabled\n", + netdev->name); + break; + + case OCTNET_CMD_VERBOSE_ENABLE: + dev_info(&oct->pci_dev->dev, "%s LRO Enabled\n", netdev->name); + break; + + case OCTNET_CMD_VERBOSE_DISABLE: + dev_info(&oct->pci_dev->dev, "%s LRO Disabled\n", + netdev->name); + break; + + case OCTNET_CMD_SET_SETTINGS: + dev_info(&oct->pci_dev->dev, "%s settings changed\n", + netdev->name); + + break; + + default: + dev_err(&oct->pci_dev->dev, "%s Unknown cmd %d\n", __func__, + nctrl->ncmd.s.cmd); + } +} + +/** + * \brief Converts a mask based on net device flags + * @param netdev network device + * + * This routine generates a octnet_ifflags mask from the net device flags + * received from the OS. + */ +static inline enum octnet_ifflags get_new_flags(struct net_device *netdev) +{ + enum octnet_ifflags f = OCTNET_IFFLAG_UNICAST; + + if (netdev->flags & IFF_PROMISC) + f |= OCTNET_IFFLAG_PROMISC; + + if (netdev->flags & IFF_ALLMULTI) + f |= OCTNET_IFFLAG_ALLMULTI; + + if (netdev->flags & IFF_MULTICAST) { + f |= OCTNET_IFFLAG_MULTICAST; + + /* Accept all multicast addresses if there are more than we + * can handle + */ + if (netdev_mc_count(netdev) > MAX_OCTEON_MULTICAST_ADDR) + f |= OCTNET_IFFLAG_ALLMULTI; + } + + if (netdev->flags & IFF_BROADCAST) + f |= OCTNET_IFFLAG_BROADCAST; + + return f; +} + +/** + * \brief Net device set_multicast_list + * @param netdev network device + */ +static void liquidio_set_mcast_list(struct net_device *netdev) +{ + struct lio *lio = GET_LIO(netdev); + struct octeon_device *oct = lio->oct_dev; + struct octnic_ctrl_pkt nctrl; + struct octnic_ctrl_params nparams; + struct netdev_hw_addr *ha; + u64 *mc; + int ret, i; + int mc_count = min(netdev_mc_count(netdev), MAX_OCTEON_MULTICAST_ADDR); + + memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt)); + + /* Create a ctrl pkt command to be sent to core app. */ + nctrl.ncmd.u64 = 0; + nctrl.ncmd.s.cmd = OCTNET_CMD_SET_MULTI_LIST; + nctrl.ncmd.s.param1 = lio->linfo.ifidx; + nctrl.ncmd.s.param2 = get_new_flags(netdev); + nctrl.ncmd.s.param3 = mc_count; + nctrl.ncmd.s.more = mc_count; + nctrl.netpndev = (u64)netdev; + nctrl.cb_fn = liquidio_link_ctrl_cmd_completion; + + /* copy all the addresses into the udd */ + i = 0; + mc = &nctrl.udd[0]; + netdev_for_each_mc_addr(ha, netdev) { + *mc = 0; + memcpy(((u8 *)mc) + 2, ha->addr, ETH_ALEN); + /* no need to swap bytes */ + + if (++mc > &nctrl.udd[mc_count]) + break; + } + + /* Apparently, any activity in this call from the kernel has to + * be atomic. So we won't wait for response. + */ + nctrl.wait_time = 0; + + nparams.resp_order = OCTEON_RESP_NORESPONSE; + + ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl, nparams); + if (ret < 0) { + dev_err(&oct->pci_dev->dev, "DEVFLAGS change failed in core (ret: 0x%x)\n", + ret); + } +} + +/** + * \brief Net device set_mac_address + * @param netdev network device + */ +static int liquidio_set_mac(struct net_device *netdev, void *p) +{ + int ret = 0; + struct lio *lio = GET_LIO(netdev); + struct octeon_device *oct = lio->oct_dev; + struct sockaddr *addr = (struct sockaddr *)p; + struct octnic_ctrl_pkt nctrl; + struct octnic_ctrl_params nparams; + + if ((!is_valid_ether_addr(addr->sa_data)) || + (ifstate_check(lio, LIO_IFSTATE_RUNNING))) + return -EADDRNOTAVAIL; + + memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt)); + + nctrl.ncmd.u64 = 0; + nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MACADDR; + nctrl.ncmd.s.param1 = lio->linfo.ifidx; + nctrl.ncmd.s.param2 = 0; + nctrl.ncmd.s.more = 1; + nctrl.netpndev = (u64)netdev; + nctrl.cb_fn = liquidio_link_ctrl_cmd_completion; + nctrl.wait_time = 100; + + nctrl.udd[0] = 0; + /* The MAC Address is presented in network byte order. */ + memcpy((u8 *)&nctrl.udd[0] + 2, addr->sa_data, ETH_ALEN); + + nparams.resp_order = OCTEON_RESP_ORDERED; + + ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl, nparams); + if (ret < 0) { + dev_err(&oct->pci_dev->dev, "MAC Address change failed\n"); + return -ENOMEM; + } + memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); + memcpy(((u8 *)&lio->linfo.hw_addr) + 2, addr->sa_data, ETH_ALEN); + + return 0; +} + +/** + * \brief Net device get_stats + * @param netdev network device + */ +static struct net_device_stats *liquidio_get_stats(struct net_device *netdev) +{ + struct lio *lio = GET_LIO(netdev); + struct net_device_stats *stats = &netdev->stats; + struct octeon_device *oct; + u64 pkts = 0, drop = 0, bytes = 0; + struct oct_droq_stats *oq_stats; + struct oct_iq_stats *iq_stats; + int i, iq_no, oq_no; + + oct = lio->oct_dev; + + for (i = 0; i < lio->linfo.num_txpciq; i++) { + iq_no = lio->linfo.txpciq[i]; + iq_stats = &oct->instr_queue[iq_no]->stats; + pkts += iq_stats->tx_done; + drop += iq_stats->tx_dropped; + bytes += iq_stats->tx_tot_bytes; + } + + stats->tx_packets = pkts; + stats->tx_bytes = bytes; + stats->tx_dropped = drop; + + pkts = 0; + drop = 0; + bytes = 0; + + for (i = 0; i < lio->linfo.num_rxpciq; i++) { + oq_no = lio->linfo.rxpciq[i]; + oq_stats = &oct->droq[oq_no]->stats; + pkts += oq_stats->rx_pkts_received; + drop += (oq_stats->rx_dropped + + oq_stats->dropped_nodispatch + + oq_stats->dropped_toomany + + oq_stats->dropped_nomem); + bytes += oq_stats->rx_bytes_received; + } + + stats->rx_bytes = bytes; + stats->rx_packets = pkts; + stats->rx_dropped = drop; + + return stats; +} + +/** + * \brief Net device change_mtu + * @param netdev network device + */ +static int liquidio_change_mtu(struct net_device *netdev, int new_mtu) +{ + struct lio *lio = GET_LIO(netdev); + struct octeon_device *oct = lio->oct_dev; + struct octnic_ctrl_pkt nctrl; + struct octnic_ctrl_params nparams; + int max_frm_size = new_mtu + OCTNET_FRM_HEADER_SIZE; + int ret = 0; + + /* Limit the MTU to make sure the ethernet packets are between 64 bytes + * and 65535 bytes + */ + if ((max_frm_size < OCTNET_MIN_FRM_SIZE) || + (max_frm_size > OCTNET_MAX_FRM_SIZE)) { + dev_err(&oct->pci_dev->dev, "Invalid MTU: %d\n", new_mtu); + dev_err(&oct->pci_dev->dev, "Valid range %d and %d\n", + (OCTNET_MIN_FRM_SIZE - OCTNET_FRM_HEADER_SIZE), + (OCTNET_MAX_FRM_SIZE - OCTNET_FRM_HEADER_SIZE)); + return -EINVAL; + } + + memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt)); + + nctrl.ncmd.u64 = 0; + nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MTU; + nctrl.ncmd.s.param1 = lio->linfo.ifidx; + nctrl.ncmd.s.param2 = new_mtu; + nctrl.wait_time = 100; + nctrl.netpndev = (u64)netdev; + nctrl.cb_fn = liquidio_link_ctrl_cmd_completion; + + nparams.resp_order = OCTEON_RESP_ORDERED; + + ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl, nparams); + if (ret < 0) { + dev_err(&oct->pci_dev->dev, "Failed to set MTU\n"); + return -1; + } + + lio->mtu = new_mtu; + + return 0; +} + +/** + * \brief Handler for SIOCSHWTSTAMP ioctl + * @param netdev network device + * @param ifr interface request + * @param cmd command + */ +static int hwtstamp_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) +{ + struct hwtstamp_config conf; + struct lio *lio = GET_LIO(netdev); + + if (copy_from_user(&conf, ifr->ifr_data, sizeof(conf))) + return -EFAULT; + + if (conf.flags) + return -EINVAL; + + switch (conf.tx_type) { + case HWTSTAMP_TX_ON: + case HWTSTAMP_TX_OFF: + break; + default: + return -ERANGE; + } + + switch (conf.rx_filter) { + case HWTSTAMP_FILTER_NONE: + break; + case HWTSTAMP_FILTER_ALL: + case HWTSTAMP_FILTER_SOME: + case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: + case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: + case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: + case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: + case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: + case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: + case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: + case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: + case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: + case HWTSTAMP_FILTER_PTP_V2_EVENT: + case HWTSTAMP_FILTER_PTP_V2_SYNC: + case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: + conf.rx_filter = HWTSTAMP_FILTER_ALL; + break; + default: + return -ERANGE; + } + + if (conf.rx_filter == HWTSTAMP_FILTER_ALL) + ifstate_set(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED); + + else + ifstate_reset(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED); + + return copy_to_user(ifr->ifr_data, &conf, sizeof(conf)) ? -EFAULT : 0; +} + +/** + * \brief ioctl handler + * @param netdev network device + * @param ifr interface request + * @param cmd command + */ +static int liquidio_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) +{ + switch (cmd) { + case SIOCSHWTSTAMP: + return hwtstamp_ioctl(netdev, ifr, cmd); + default: + return -EOPNOTSUPP; + } +} + +/** + * \brief handle a Tx timestamp response + * @param status response status + * @param buf pointer to skb + */ +static void handle_timestamp(struct octeon_device *oct, + u32 status, + void *buf) +{ + struct octnet_buf_free_info *finfo; + struct octeon_soft_command *sc; + struct oct_timestamp_resp *resp; + struct lio *lio; + struct sk_buff *skb = (struct sk_buff *)buf; + + finfo = (struct octnet_buf_free_info *)skb->cb; + lio = finfo->lio; + sc = finfo->sc; + oct = lio->oct_dev; + resp = (struct oct_timestamp_resp *)sc->virtrptr; + + if (status != OCTEON_REQUEST_DONE) { + dev_err(&oct->pci_dev->dev, "Tx timestamp instruction failed. Status: %llx\n", + CVM_CAST64(status)); + resp->timestamp = 0; + } + + octeon_swap_8B_data(&resp->timestamp, 1); + + if (unlikely((skb_shinfo(skb)->tx_flags | SKBTX_IN_PROGRESS) != 0)) { + struct skb_shared_hwtstamps ts; + u64 ns = resp->timestamp; + + netif_info(lio, tx_done, lio->netdev, + "Got resulting SKBTX_HW_TSTAMP skb=%p ns=%016llu\n", + skb, (unsigned long long)ns); + ts.hwtstamp = ns_to_ktime(ns + lio->ptp_adjust); + skb_tstamp_tx(skb, &ts); + } + + octeon_free_soft_command(oct, sc); + recv_buffer_free(skb); +} + +/* \brief Send a data packet that will be timestamped + * @param oct octeon device + * @param ndata pointer to network data + * @param finfo pointer to private network data + */ +static inline int send_nic_timestamp_pkt(struct octeon_device *oct, + struct octnic_data_pkt *ndata, + struct octnet_buf_free_info *finfo, + int xmit_more) +{ + int retval; + struct octeon_soft_command *sc; + struct octeon_instr_ih *ih; + struct octeon_instr_rdp *rdp; + struct lio *lio; + int ring_doorbell; + + lio = finfo->lio; + + sc = octeon_alloc_soft_command_resp(oct, &ndata->cmd, + sizeof(struct oct_timestamp_resp)); + finfo->sc = sc; + + if (!sc) { + dev_err(&oct->pci_dev->dev, "No memory for timestamped data packet\n"); + return IQ_SEND_FAILED; + } + + if (ndata->reqtype == REQTYPE_NORESP_NET) + ndata->reqtype = REQTYPE_RESP_NET; + else if (ndata->reqtype == REQTYPE_NORESP_NET_SG) + ndata->reqtype = REQTYPE_RESP_NET_SG; + + sc->callback = handle_timestamp; + sc->callback_arg = finfo->skb; + sc->iq_no = ndata->q_no; + + ih = (struct octeon_instr_ih *)&sc->cmd.ih; + rdp = (struct octeon_instr_rdp *)&sc->cmd.rdp; + + ring_doorbell = !xmit_more; + retval = octeon_send_command(oct, sc->iq_no, ring_doorbell, &sc->cmd, + sc, ih->dlengsz, ndata->reqtype); + + if (retval) { + dev_err(&oct->pci_dev->dev, "timestamp data packet failed status: %x\n", + retval); + octeon_free_soft_command(oct, sc); + } else { + netif_info(lio, tx_queued, lio->netdev, "Queued timestamp packet\n"); + } + + return retval; +} + +static inline int is_ipv4(struct sk_buff *skb) +{ + return (skb->protocol == htons(ETH_P_IP)) && + (ip_hdr(skb)->version == 4); +} + +static inline int is_vlan(struct sk_buff *skb) +{ + return skb->protocol == htons(ETH_P_8021Q); +} + +static inline int is_ip_fragmented(struct sk_buff *skb) +{ + /* The Don't fragment and Reserved flag fields are ignored. + * IP is fragmented if + * - the More fragments bit is set (indicating this IP is a fragment + * with more to follow; the current offset could be 0 ). + * - ths offset field is non-zero. + */ + return (ip_hdr(skb)->frag_off & htons(IP_MF | IP_OFFSET)) ? 1 : 0; +} + +static inline int is_ipv6(struct sk_buff *skb) +{ + return (skb->protocol == htons(ETH_P_IPV6)) && + (ipv6_hdr(skb)->version == 6); +} + +static inline int is_with_extn_hdr(struct sk_buff *skb) +{ + return (ipv6_hdr(skb)->nexthdr != IPPROTO_TCP) && + (ipv6_hdr(skb)->nexthdr != IPPROTO_UDP); +} + +static inline int is_tcpudp(struct sk_buff *skb) +{ + return (ip_hdr(skb)->protocol == IPPROTO_TCP) || + (ip_hdr(skb)->protocol == IPPROTO_UDP); +} + +static inline u32 get_ipv4_5tuple_tag(struct sk_buff *skb) +{ + u32 tag; + struct iphdr *iphdr = ip_hdr(skb); + + tag = crc32(0, &iphdr->protocol, 1); + tag = crc32(tag, (u8 *)&iphdr->saddr, 8); + tag = crc32(tag, skb_transport_header(skb), 4); + return tag; +} + +static inline u32 get_ipv6_5tuple_tag(struct sk_buff *skb) +{ + u32 tag; + struct ipv6hdr *ipv6hdr = ipv6_hdr(skb); + + tag = crc32(0, &ipv6hdr->nexthdr, 1); + tag = crc32(tag, (u8 *)&ipv6hdr->saddr, 32); + tag = crc32(tag, skb_transport_header(skb), 4); + return tag; +} + +/** \brief Transmit networks packets to the Octeon interface + * @param skbuff skbuff struct to be passed to network layer. + * @param netdev pointer to network device + * @returns whether the packet was transmitted to the device okay or not + * (NETDEV_TX_OK or NETDEV_TX_BUSY) + */ +static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev) +{ + struct lio *lio; + struct octnet_buf_free_info *finfo; + union octnic_cmd_setup cmdsetup; + struct octnic_data_pkt ndata; + struct octeon_device *oct; + struct oct_iq_stats *stats; + int cpu = 0, status = 0; + int q_idx = 0, iq_no = 0; + int xmit_more; + u32 tag = 0; + + lio = GET_LIO(netdev); + oct = lio->oct_dev; + + if (netif_is_multiqueue(netdev)) { + cpu = skb->queue_mapping; + q_idx = (cpu & (lio->linfo.num_txpciq - 1)); + iq_no = lio->linfo.txpciq[q_idx]; + } else { + iq_no = lio->txq; + } + + stats = &oct->instr_queue[iq_no]->stats; + + /* Check for all conditions in which the current packet cannot be + * transmitted. + */ + if (!(atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING) || + (!lio->linfo.link.s.status) || + (skb->len <= 0)) { + netif_info(lio, tx_err, lio->netdev, + "Transmit failed link_status : %d\n", + lio->linfo.link.s.status); + goto lio_xmit_failed; + } + + /* Use space in skb->cb to store info used to unmap and + * free the buffers. + */ + finfo = (struct octnet_buf_free_info *)skb->cb; + finfo->lio = lio; + finfo->skb = skb; + finfo->sc = NULL; + + /* Prepare the attributes for the data to be passed to OSI. */ + memset(&ndata, 0, sizeof(struct octnic_data_pkt)); + + ndata.buf = (void *)finfo; + + ndata.q_no = iq_no; + + if (netif_is_multiqueue(netdev)) { + if (octnet_iq_is_full(oct, ndata.q_no)) { + /* defer sending if queue is full */ + netif_info(lio, tx_err, lio->netdev, "Transmit failed iq:%d full\n", + ndata.q_no); + stats->tx_iq_busy++; + return NETDEV_TX_BUSY; + } + } else { + if (octnet_iq_is_full(oct, lio->txq)) { + /* defer sending if queue is full */ + stats->tx_iq_busy++; + netif_info(lio, tx_err, lio->netdev, "Transmit failed iq:%d full\n", + ndata.q_no); + return NETDEV_TX_BUSY; + } + } + /* pr_info(" XMIT - valid Qs: %d, 1st Q no: %d, cpu: %d, q_no:%d\n", + * lio->linfo.num_txpciq, lio->txq, cpu, ndata.q_no ); + */ + + ndata.datasize = skb->len; + + cmdsetup.u64 = 0; + cmdsetup.s.ifidx = lio->linfo.ifidx; + + if (skb->ip_summed == CHECKSUM_PARTIAL) { + if (is_ipv4(skb) && !is_ip_fragmented(skb) && is_tcpudp(skb)) { + tag = get_ipv4_5tuple_tag(skb); + + cmdsetup.s.cksum_offset = sizeof(struct ethhdr) + 1; + + if (ip_hdr(skb)->ihl > 5) + cmdsetup.s.ipv4opts_ipv6exthdr = + OCT_PKT_PARAM_IPV4OPTS; + + } else if (is_ipv6(skb)) { + tag = get_ipv6_5tuple_tag(skb); + + cmdsetup.s.cksum_offset = sizeof(struct ethhdr) + 1; + + if (is_with_extn_hdr(skb)) + cmdsetup.s.ipv4opts_ipv6exthdr = + OCT_PKT_PARAM_IPV6EXTHDR; + + } else if (is_vlan(skb)) { + if (vlan_eth_hdr(skb)->h_vlan_encapsulated_proto + == htons(ETH_P_IP) && + !is_ip_fragmented(skb) && is_tcpudp(skb)) { + tag = get_ipv4_5tuple_tag(skb); + + cmdsetup.s.cksum_offset = + sizeof(struct vlan_ethhdr) + 1; + + if (ip_hdr(skb)->ihl > 5) + cmdsetup.s.ipv4opts_ipv6exthdr = + OCT_PKT_PARAM_IPV4OPTS; + + } else if (vlan_eth_hdr(skb)->h_vlan_encapsulated_proto + == htons(ETH_P_IPV6)) { + tag = get_ipv6_5tuple_tag(skb); + + cmdsetup.s.cksum_offset = + sizeof(struct vlan_ethhdr) + 1; + + if (is_with_extn_hdr(skb)) + cmdsetup.s.ipv4opts_ipv6exthdr = + OCT_PKT_PARAM_IPV6EXTHDR; + } + } + } + if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { + skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; + cmdsetup.s.timestamp = 1; + } + + if (skb_shinfo(skb)->nr_frags == 0) { + cmdsetup.s.u.datasize = skb->len; + octnet_prepare_pci_cmd(&ndata.cmd, &cmdsetup, tag); + /* Offload checksum calculation for TCP/UDP packets */ + ndata.cmd.dptr = dma_map_single(&oct->pci_dev->dev, + skb->data, + skb->len, + DMA_TO_DEVICE); + if (dma_mapping_error(&oct->pci_dev->dev, ndata.cmd.dptr)) { + dev_err(&oct->pci_dev->dev, "%s DMA mapping error 1\n", + __func__); + return NETDEV_TX_BUSY; + } + + finfo->dptr = ndata.cmd.dptr; + + ndata.reqtype = REQTYPE_NORESP_NET; + + } else { + int i, frags; + struct skb_frag_struct *frag; + struct octnic_gather *g; + + spin_lock(&lio->lock); + g = (struct octnic_gather *)list_delete_head(&lio->glist); + spin_unlock(&lio->lock); + + if (!g) { + netif_info(lio, tx_err, lio->netdev, + "Transmit scatter gather: glist null!\n"); + goto lio_xmit_failed; + } + + cmdsetup.s.gather = 1; + cmdsetup.s.u.gatherptrs = (skb_shinfo(skb)->nr_frags + 1); + octnet_prepare_pci_cmd(&ndata.cmd, &cmdsetup, tag); + + memset(g->sg, 0, g->sg_size); + + g->sg[0].ptr[0] = dma_map_single(&oct->pci_dev->dev, + skb->data, + (skb->len - skb->data_len), + DMA_TO_DEVICE); + if (dma_mapping_error(&oct->pci_dev->dev, g->sg[0].ptr[0])) { + dev_err(&oct->pci_dev->dev, "%s DMA mapping error 2\n", + __func__); + return NETDEV_TX_BUSY; + } + add_sg_size(&g->sg[0], (skb->len - skb->data_len), 0); + + frags = skb_shinfo(skb)->nr_frags; + i = 1; + while (frags--) { + frag = &skb_shinfo(skb)->frags[i - 1]; + + g->sg[(i >> 2)].ptr[(i & 3)] = + dma_map_page(&oct->pci_dev->dev, + frag->page.p, + frag->page_offset, + frag->size, + DMA_TO_DEVICE); + + add_sg_size(&g->sg[(i >> 2)], frag->size, (i & 3)); + i++; + } + + ndata.cmd.dptr = dma_map_single(&oct->pci_dev->dev, + g->sg, g->sg_size, + DMA_TO_DEVICE); + if (dma_mapping_error(&oct->pci_dev->dev, ndata.cmd.dptr)) { + dev_err(&oct->pci_dev->dev, "%s DMA mapping error 3\n", + __func__); + dma_unmap_single(&oct->pci_dev->dev, g->sg[0].ptr[0], + skb->len - skb->data_len, + DMA_TO_DEVICE); + return NETDEV_TX_BUSY; + } + + finfo->dptr = ndata.cmd.dptr; + finfo->g = g; + + ndata.reqtype = REQTYPE_NORESP_NET_SG; + } + + if (skb_shinfo(skb)->gso_size) { + struct octeon_instr_irh *irh = + (struct octeon_instr_irh *)&ndata.cmd.irh; + union tx_info *tx_info = (union tx_info *)&ndata.cmd.ossp[0]; + + irh->len = 1; /* to indicate that ossp[0] contains tx_info */ + tx_info->s.gso_size = skb_shinfo(skb)->gso_size; + tx_info->s.gso_segs = skb_shinfo(skb)->gso_segs; + } + + xmit_more = skb->xmit_more; + + if (unlikely(cmdsetup.s.timestamp)) + status = send_nic_timestamp_pkt(oct, &ndata, finfo, xmit_more); + else + status = octnet_send_nic_data_pkt(oct, &ndata, xmit_more); + if (status == IQ_SEND_FAILED) + goto lio_xmit_failed; + + netif_info(lio, tx_queued, lio->netdev, "Transmit queued successfully\n"); + + if (status == IQ_SEND_STOP) + stop_q(lio->netdev, q_idx); + + netdev->trans_start = jiffies; + + stats->tx_done++; + stats->tx_tot_bytes += skb->len; + + return NETDEV_TX_OK; + +lio_xmit_failed: + stats->tx_dropped++; + netif_info(lio, tx_err, lio->netdev, "IQ%d Transmit dropped:%llu\n", + iq_no, stats->tx_dropped); + dma_unmap_single(&oct->pci_dev->dev, ndata.cmd.dptr, + ndata.datasize, DMA_TO_DEVICE); + recv_buffer_free(skb); + return NETDEV_TX_OK; +} + +/** \brief Network device Tx timeout + * @param netdev pointer to network device + */ +static void liquidio_tx_timeout(struct net_device *netdev) +{ + struct lio *lio; + + lio = GET_LIO(netdev); + + netif_info(lio, tx_err, lio->netdev, + "Transmit timeout tx_dropped:%ld, waking up queues now!!\n", + netdev->stats.tx_dropped); + netdev->trans_start = jiffies; + txqs_wake(netdev); +} + +int liquidio_set_feature(struct net_device *netdev, int cmd) +{ + struct lio *lio = GET_LIO(netdev); + struct octeon_device *oct = lio->oct_dev; + struct octnic_ctrl_pkt nctrl; + struct octnic_ctrl_params nparams; + int ret = 0; + + memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt)); + + nctrl.ncmd.u64 = 0; + nctrl.ncmd.s.cmd = cmd; + nctrl.ncmd.s.param1 = lio->linfo.ifidx; + nctrl.ncmd.s.param2 = OCTNIC_LROIPV4 | OCTNIC_LROIPV6; + nctrl.wait_time = 100; + nctrl.netpndev = (u64)netdev; + nctrl.cb_fn = liquidio_link_ctrl_cmd_completion; + + nparams.resp_order = OCTEON_RESP_NORESPONSE; + + ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl, nparams); + if (ret < 0) { + dev_err(&oct->pci_dev->dev, "Feature change failed in core (ret: 0x%x)\n", + ret); + } + return ret; +} + +/** \brief Net device fix features + * @param netdev pointer to network device + * @param request features requested + * @returns updated features list + */ +static netdev_features_t liquidio_fix_features(struct net_device *netdev, + netdev_features_t request) +{ + struct lio *lio = netdev_priv(netdev); + + if ((request & NETIF_F_RXCSUM) && + !(lio->dev_capability & NETIF_F_RXCSUM)) + request &= ~NETIF_F_RXCSUM; + + if ((request & NETIF_F_HW_CSUM) && + !(lio->dev_capability & NETIF_F_HW_CSUM)) + request &= ~NETIF_F_HW_CSUM; + + if ((request & NETIF_F_TSO) && !(lio->dev_capability & NETIF_F_TSO)) + request &= ~NETIF_F_TSO; + + if ((request & NETIF_F_TSO6) && !(lio->dev_capability & NETIF_F_TSO6)) + request &= ~NETIF_F_TSO6; + + if ((request & NETIF_F_LRO) && !(lio->dev_capability & NETIF_F_LRO)) + request &= ~NETIF_F_LRO; + + /*Disable LRO if RXCSUM is off */ + if (!(request & NETIF_F_RXCSUM) && (netdev->features & NETIF_F_LRO) && + (lio->dev_capability & NETIF_F_LRO)) + request &= ~NETIF_F_LRO; + + return request; +} + +/** \brief Net device set features + * @param netdev pointer to network device + * @param features features to enable/disable + */ +static int liquidio_set_features(struct net_device *netdev, + netdev_features_t features) +{ + struct lio *lio = netdev_priv(netdev); + + if (!((netdev->features ^ features) & NETIF_F_LRO)) + return 0; + + if ((features & NETIF_F_LRO) && (lio->dev_capability & NETIF_F_LRO)) + liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE); + else if (!(features & NETIF_F_LRO) && + (lio->dev_capability & NETIF_F_LRO)) + liquidio_set_feature(netdev, OCTNET_CMD_LRO_DISABLE); + + return 0; +} + +static struct net_device_ops lionetdevops = { + .ndo_open = liquidio_open, + .ndo_stop = liquidio_stop, + .ndo_start_xmit = liquidio_xmit, + .ndo_get_stats = liquidio_get_stats, + .ndo_set_mac_address = liquidio_set_mac, + .ndo_set_rx_mode = liquidio_set_mcast_list, + .ndo_tx_timeout = liquidio_tx_timeout, + .ndo_change_mtu = liquidio_change_mtu, + .ndo_do_ioctl = liquidio_ioctl, + .ndo_fix_features = liquidio_fix_features, + .ndo_set_features = liquidio_set_features, +}; + +/** \brief Entry point for the liquidio module + */ +static int __init liquidio_init(void) +{ + int i; + struct handshake *hs; + + init_completion(&first_stage); + + octeon_init_device_list(conf_type); + + if (liquidio_init_pci()) + return -EINVAL; + + wait_for_completion_timeout(&first_stage, msecs_to_jiffies(1000)); + + for (i = 0; i < MAX_OCTEON_DEVICES; i++) { + hs = &handshake[i]; + if (hs->pci_dev) { + wait_for_completion(&hs->init); + if (!hs->init_ok) { + /* init handshake failed */ + dev_err(&hs->pci_dev->dev, + "Failed to init device\n"); + liquidio_deinit_pci(); + return -EIO; + } + } + } + + for (i = 0; i < MAX_OCTEON_DEVICES; i++) { + hs = &handshake[i]; + if (hs->pci_dev) { + wait_for_completion_timeout(&hs->started, + msecs_to_jiffies(30000)); + if (!hs->started_ok) { + /* starter handshake failed */ + dev_err(&hs->pci_dev->dev, + "Firmware failed to start\n"); + liquidio_deinit_pci(); + return -EIO; + } + } + } + + return 0; +} + +static int lio_nic_info(struct octeon_recv_info *recv_info, void *buf) +{ + struct octeon_device *oct = (struct octeon_device *)buf; + struct octeon_recv_pkt *recv_pkt = recv_info->recv_pkt; + int ifidx = 0; + union oct_link_status *ls; + int i; + + if ((recv_pkt->buffer_size[0] != sizeof(*ls)) || + (recv_pkt->rh.r_nic_info.ifidx > oct->ifcount)) { + dev_err(&oct->pci_dev->dev, "Malformed NIC_INFO, len=%d, ifidx=%d\n", + recv_pkt->buffer_size[0], + recv_pkt->rh.r_nic_info.ifidx); + goto nic_info_err; + } + + ifidx = recv_pkt->rh.r_nic_info.ifidx; + ls = (union oct_link_status *)get_rbd(recv_pkt->buffer_ptr[0]); + + octeon_swap_8B_data((u64 *)ls, (sizeof(union oct_link_status)) >> 3); + + update_link_status(oct->props[ifidx].netdev, ls); + +nic_info_err: + for (i = 0; i < recv_pkt->buffer_count; i++) + recv_buffer_free(recv_pkt->buffer_ptr[i]); + octeon_free_recv_info(recv_info); + return 0; +} + +/** + * \brief Setup network interfaces + * @param octeon_dev octeon device + * + * Called during init time for each device. It assumes the NIC + * is already up and running. The link information for each + * interface is passed in link_info. + */ +static int setup_nic_devices(struct octeon_device *octeon_dev) +{ + struct lio *lio = NULL; + struct net_device *netdev; + u8 mac[6], i, j; + struct octeon_soft_command *sc; + struct liquidio_if_cfg_context *ctx; + struct liquidio_if_cfg_resp *resp; + struct octdev_props *props; + int retval, num_iqueues, num_oqueues, q_no; + u64 q_mask; + int num_cpus = num_online_cpus(); + union oct_nic_if_cfg if_cfg; + unsigned int base_queue; + unsigned int gmx_port_id; + u32 resp_size, ctx_size; + + /* This is to handle link status changes */ + octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC, + OPCODE_NIC_INFO, + lio_nic_info, octeon_dev); + + /* REQTYPE_RESP_NET and REQTYPE_SOFT_COMMAND do not have free functions. + * They are handled directly. + */ + octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET, + free_netbuf); + + octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET_SG, + free_netsgbuf); + + octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_RESP_NET_SG, + free_netsgbuf_with_resp); + + for (i = 0; i < octeon_dev->ifcount; i++) { + resp_size = sizeof(struct liquidio_if_cfg_resp); + ctx_size = sizeof(struct liquidio_if_cfg_context); + sc = (struct octeon_soft_command *) + octeon_alloc_soft_command(octeon_dev, 0, + resp_size, ctx_size); + resp = (struct liquidio_if_cfg_resp *)sc->virtrptr; + ctx = (struct liquidio_if_cfg_context *)sc->ctxptr; + + num_iqueues = + CFG_GET_NUM_TXQS_NIC_IF(octeon_get_conf(octeon_dev), i); + num_oqueues = + CFG_GET_NUM_RXQS_NIC_IF(octeon_get_conf(octeon_dev), i); + base_queue = + CFG_GET_BASE_QUE_NIC_IF(octeon_get_conf(octeon_dev), i); + gmx_port_id = + CFG_GET_GMXID_NIC_IF(octeon_get_conf(octeon_dev), i); + if (num_iqueues > num_cpus) + num_iqueues = num_cpus; + if (num_oqueues > num_cpus) + num_oqueues = num_cpus; + dev_dbg(&octeon_dev->pci_dev->dev, + "requesting config for interface %d, iqs %d, oqs %d\n", + i, num_iqueues, num_oqueues); + ACCESS_ONCE(ctx->cond) = 0; + ctx->octeon_id = lio_get_device_id(octeon_dev); + init_waitqueue_head(&ctx->wc); + + if_cfg.u64 = 0; + if_cfg.s.num_iqueues = num_iqueues; + if_cfg.s.num_oqueues = num_oqueues; + if_cfg.s.base_queue = base_queue; + if_cfg.s.gmx_port_id = gmx_port_id; + octeon_prepare_soft_command(octeon_dev, sc, OPCODE_NIC, + OPCODE_NIC_IF_CFG, i, + if_cfg.u64, 0); + + sc->callback = if_cfg_callback; + sc->callback_arg = sc; + sc->wait_time = 1000; + + retval = octeon_send_soft_command(octeon_dev, sc); + if (retval) { + dev_err(&octeon_dev->pci_dev->dev, + "iq/oq config failed status: %x\n", + retval); + /* Soft instr is freed by driver in case of failure. */ + goto setup_nic_dev_fail; + } + + /* Sleep on a wait queue till the cond flag indicates that the + * response arrived or timed-out. + */ + sleep_cond(&ctx->wc, &ctx->cond); + retval = resp->status; + if (retval) { + dev_err(&octeon_dev->pci_dev->dev, "iq/oq config failed\n"); + goto setup_nic_dev_fail; + } + + octeon_swap_8B_data((u64 *)(&resp->cfg_info), + (sizeof(struct liquidio_if_cfg_info)) >> 3); + + num_iqueues = hweight64(resp->cfg_info.iqmask); + num_oqueues = hweight64(resp->cfg_info.oqmask); + + if (!(num_iqueues) || !(num_oqueues)) { + dev_err(&octeon_dev->pci_dev->dev, + "Got bad iqueues (%016llx) or oqueues (%016llx) from firmware.\n", + resp->cfg_info.iqmask, + resp->cfg_info.oqmask); + goto setup_nic_dev_fail; + } + dev_dbg(&octeon_dev->pci_dev->dev, + "interface %d, iqmask %016llx, oqmask %016llx, numiqueues %d, numoqueues %d\n", + i, resp->cfg_info.iqmask, resp->cfg_info.oqmask, + num_iqueues, num_oqueues); + netdev = alloc_etherdev_mq(LIO_SIZE, num_iqueues); + + if (!netdev) { + dev_err(&octeon_dev->pci_dev->dev, "Device allocation failed\n"); + goto setup_nic_dev_fail; + } + + props = &octeon_dev->props[i]; + props->netdev = netdev; + + if (num_iqueues > 1) + lionetdevops.ndo_select_queue = select_q; + + /* Associate the routines that will handle different + * netdev tasks. + */ + netdev->netdev_ops = &lionetdevops; + + lio = GET_LIO(netdev); + + memset(lio, 0, sizeof(struct lio)); + + lio->linfo.ifidx = resp->cfg_info.ifidx; + lio->ifidx = resp->cfg_info.ifidx; + + lio->linfo.num_rxpciq = num_oqueues; + lio->linfo.num_txpciq = num_iqueues; + q_mask = resp->cfg_info.oqmask; + /* q_mask is 0-based and already verified mask is nonzero */ + for (j = 0; j < num_oqueues; j++) { + q_no = __ffs64(q_mask); + q_mask &= (~(1UL << q_no)); + lio->linfo.rxpciq[j] = q_no; + } + q_mask = resp->cfg_info.iqmask; + for (j = 0; j < num_iqueues; j++) { + q_no = __ffs64(q_mask); + q_mask &= (~(1UL << q_no)); + lio->linfo.txpciq[j] = q_no; + } + lio->linfo.hw_addr = resp->cfg_info.linfo.hw_addr; + lio->linfo.gmxport = resp->cfg_info.linfo.gmxport; + lio->linfo.link.u64 = resp->cfg_info.linfo.link.u64; + + lio->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); + + lio->dev_capability = NETIF_F_HIGHDMA + | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM + | NETIF_F_SG | NETIF_F_RXCSUM + | NETIF_F_TSO | NETIF_F_TSO6 + | NETIF_F_LRO; + netif_set_gso_max_size(netdev, OCTNIC_GSO_MAX_SIZE); + + netdev->features = lio->dev_capability; + netdev->vlan_features = lio->dev_capability; + + netdev->hw_features = lio->dev_capability; + + /* Point to the properties for octeon device to which this + * interface belongs. + */ + lio->oct_dev = octeon_dev; + lio->octprops = props; + lio->netdev = netdev; + spin_lock_init(&lio->lock); + + dev_dbg(&octeon_dev->pci_dev->dev, + "if%d gmx: %d hw_addr: 0x%llx\n", i, + lio->linfo.gmxport, CVM_CAST64(lio->linfo.hw_addr)); + + /* 64-bit swap required on LE machines */ + octeon_swap_8B_data(&lio->linfo.hw_addr, 1); + for (j = 0; j < 6; j++) + mac[j] = *((u8 *)(((u8 *)&lio->linfo.hw_addr) + 2 + j)); + + /* Copy MAC Address to OS network device structure */ + + ether_addr_copy(netdev->dev_addr, mac); + + if (setup_io_queues(octeon_dev, netdev)) { + dev_err(&octeon_dev->pci_dev->dev, "I/O queues creation failed\n"); + goto setup_nic_dev_fail; + } + + ifstate_set(lio, LIO_IFSTATE_DROQ_OPS); + + /* By default all interfaces on a single Octeon uses the same + * tx and rx queues + */ + lio->txq = lio->linfo.txpciq[0]; + lio->rxq = lio->linfo.rxpciq[0]; + + lio->tx_qsize = octeon_get_tx_qsize(octeon_dev, lio->txq); + lio->rx_qsize = octeon_get_rx_qsize(octeon_dev, lio->rxq); + + if (setup_glist(lio)) { + dev_err(&octeon_dev->pci_dev->dev, + "Gather list allocation failed\n"); + goto setup_nic_dev_fail; + } + + /* Register ethtool support */ + liquidio_set_ethtool_ops(netdev); + + liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE); + + if ((debug != -1) && (debug & NETIF_MSG_HW)) + liquidio_set_feature(netdev, OCTNET_CMD_VERBOSE_ENABLE); + + /* Register the network device with the OS */ + if (register_netdev(netdev)) { + dev_err(&octeon_dev->pci_dev->dev, "Device registration failed\n"); + goto setup_nic_dev_fail; + } + + dev_dbg(&octeon_dev->pci_dev->dev, + "Setup NIC ifidx:%d mac:%02x%02x%02x%02x%02x%02x\n", + i, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]); + netif_carrier_off(netdev); + + if (lio->linfo.link.s.status) { + netif_carrier_on(netdev); + start_txq(netdev); + } else { + netif_carrier_off(netdev); + } + + ifstate_set(lio, LIO_IFSTATE_REGISTERED); + + dev_dbg(&octeon_dev->pci_dev->dev, + "NIC ifidx:%d Setup successful\n", i); + + octeon_free_soft_command(octeon_dev, sc); + } + + return 0; + +setup_nic_dev_fail: + + octeon_free_soft_command(octeon_dev, sc); + + while (i--) { + dev_err(&octeon_dev->pci_dev->dev, + "NIC ifidx:%d Setup failed\n", i); + liquidio_destroy_nic_device(octeon_dev, i); + } + return -ENODEV; +} + +/** + * \brief initialize the NIC + * @param oct octeon device + * + * This initialization routine is called once the Octeon device application is + * up and running + */ +static int liquidio_init_nic_module(struct octeon_device *oct) +{ + struct oct_intrmod_cfg *intrmod_cfg; + int retval = 0; + int num_nic_ports = CFG_GET_NUM_NIC_PORTS(octeon_get_conf(oct)); + + dev_dbg(&oct->pci_dev->dev, "Initializing network interfaces\n"); + + /* only default iq and oq were initialized + * initialize the rest as well + */ + /* run port_config command for each port */ + oct->ifcount = num_nic_ports; + + memset(oct->props, 0, + sizeof(struct octdev_props) * num_nic_ports); + + retval = setup_nic_devices(oct); + if (retval) { + dev_err(&oct->pci_dev->dev, "Setup NIC devices failed\n"); + goto octnet_init_failure; + } + + liquidio_ptp_init(oct); + + /* Initialize interrupt moderation params */ + intrmod_cfg = &((struct octeon_device *)oct)->intrmod; + intrmod_cfg->intrmod_enable = 1; + intrmod_cfg->intrmod_check_intrvl = LIO_INTRMOD_CHECK_INTERVAL; + intrmod_cfg->intrmod_maxpkt_ratethr = LIO_INTRMOD_MAXPKT_RATETHR; + intrmod_cfg->intrmod_minpkt_ratethr = LIO_INTRMOD_MINPKT_RATETHR; + intrmod_cfg->intrmod_maxcnt_trigger = LIO_INTRMOD_MAXCNT_TRIGGER; + intrmod_cfg->intrmod_maxtmr_trigger = LIO_INTRMOD_MAXTMR_TRIGGER; + intrmod_cfg->intrmod_mintmr_trigger = LIO_INTRMOD_MINTMR_TRIGGER; + intrmod_cfg->intrmod_mincnt_trigger = LIO_INTRMOD_MINCNT_TRIGGER; + + dev_dbg(&oct->pci_dev->dev, "Network interfaces ready\n"); + + return retval; + +octnet_init_failure: + + oct->ifcount = 0; + + return retval; +} + +/** + * \brief starter callback that invokes the remaining initialization work after + * the NIC is up and running. + * @param octptr work struct work_struct + */ +static void nic_starter(struct work_struct *work) +{ + struct octeon_device *oct; + struct cavium_wk *wk = (struct cavium_wk *)work; + + oct = (struct octeon_device *)wk->ctxptr; + + if (atomic_read(&oct->status) == OCT_DEV_RUNNING) + return; + + /* If the status of the device is CORE_OK, the core + * application has reported its application type. Call + * any registered handlers now and move to the RUNNING + * state. + */ + if (atomic_read(&oct->status) != OCT_DEV_CORE_OK) { + schedule_delayed_work(&oct->nic_poll_work.work, + LIQUIDIO_STARTER_POLL_INTERVAL_MS); + return; + } + + atomic_set(&oct->status, OCT_DEV_RUNNING); + + if (oct->app_mode && oct->app_mode == CVM_DRV_NIC_APP) { + dev_dbg(&oct->pci_dev->dev, "Starting NIC module\n"); + + if (liquidio_init_nic_module(oct)) + dev_err(&oct->pci_dev->dev, "NIC initialization failed\n"); + else + handshake[oct->octeon_id].started_ok = 1; + } else { + dev_err(&oct->pci_dev->dev, + "Unexpected application running on NIC (%d). Check firmware.\n", + oct->app_mode); + } + + complete(&handshake[oct->octeon_id].started); +} + +/** + * \brief Device initialization for each Octeon device that is probed + * @param octeon_dev octeon device + */ +static int octeon_device_init(struct octeon_device *octeon_dev) +{ + int j, ret; + struct octeon_device_priv *oct_priv = + (struct octeon_device_priv *)octeon_dev->priv; + atomic_set(&octeon_dev->status, OCT_DEV_BEGIN_STATE); + + /* Enable access to the octeon device and make its DMA capability + * known to the OS. + */ + if (octeon_pci_os_setup(octeon_dev)) + return 1; + + /* Identify the Octeon type and map the BAR address space. */ + if (octeon_chip_specific_setup(octeon_dev)) { + dev_err(&octeon_dev->pci_dev->dev, "Chip specific setup failed\n"); + return 1; + } + + atomic_set(&octeon_dev->status, OCT_DEV_PCI_MAP_DONE); + + octeon_dev->app_mode = CVM_DRV_INVALID_APP; + + /* Do a soft reset of the Octeon device. */ + if (octeon_dev->fn_list.soft_reset(octeon_dev)) + return 1; + + /* Initialize the dispatch mechanism used to push packets arriving on + * Octeon Output queues. + */ + if (octeon_init_dispatch_list(octeon_dev)) + return 1; + + octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC, + OPCODE_NIC_CORE_DRV_ACTIVE, + octeon_core_drv_init, + octeon_dev); + + INIT_DELAYED_WORK(&octeon_dev->nic_poll_work.work, nic_starter); + octeon_dev->nic_poll_work.ctxptr = (void *)octeon_dev; + schedule_delayed_work(&octeon_dev->nic_poll_work.work, + LIQUIDIO_STARTER_POLL_INTERVAL_MS); + + atomic_set(&octeon_dev->status, OCT_DEV_DISPATCH_INIT_DONE); + + octeon_set_io_queues_off(octeon_dev); + + /* Setup the data structures that manage this Octeon's Input queues. */ + if (octeon_setup_instr_queues(octeon_dev)) { + dev_err(&octeon_dev->pci_dev->dev, + "instruction queue initialization failed\n"); + /* On error, release any previously allocated queues */ + for (j = 0; j < octeon_dev->num_iqs; j++) + octeon_delete_instr_queue(octeon_dev, j); + return 1; + } + atomic_set(&octeon_dev->status, OCT_DEV_INSTR_QUEUE_INIT_DONE); + + /* Initialize soft command buffer pool + */ + if (octeon_setup_sc_buffer_pool(octeon_dev)) { + dev_err(&octeon_dev->pci_dev->dev, "sc buffer pool allocation failed\n"); + return 1; + } + atomic_set(&octeon_dev->status, OCT_DEV_SC_BUFF_POOL_INIT_DONE); + + /* Initialize lists to manage the requests of different types that + * arrive from user & kernel applications for this octeon device. + */ + if (octeon_setup_response_list(octeon_dev)) { + dev_err(&octeon_dev->pci_dev->dev, "Response list allocation failed\n"); + return 1; + } + atomic_set(&octeon_dev->status, OCT_DEV_RESP_LIST_INIT_DONE); + + if (octeon_setup_output_queues(octeon_dev)) { + dev_err(&octeon_dev->pci_dev->dev, "Output queue initialization failed\n"); + /* Release any previously allocated queues */ + for (j = 0; j < octeon_dev->num_oqs; j++) + octeon_delete_droq(octeon_dev, j); + } + + atomic_set(&octeon_dev->status, OCT_DEV_DROQ_INIT_DONE); + + /* The input and output queue registers were setup earlier (the queues + * were not enabled). Any additional registers that need to be + * programmed should be done now. + */ + ret = octeon_dev->fn_list.setup_device_regs(octeon_dev); + if (ret) { + dev_err(&octeon_dev->pci_dev->dev, + "Failed to configure device registers\n"); + return ret; + } + + /* Initialize the tasklet that handles output queue packet processing.*/ + dev_dbg(&octeon_dev->pci_dev->dev, "Initializing droq tasklet\n"); + tasklet_init(&oct_priv->droq_tasklet, octeon_droq_bh, + (unsigned long)octeon_dev); + + /* Setup the interrupt handler and record the INT SUM register address + */ + octeon_setup_interrupt(octeon_dev); + + /* Enable Octeon device interrupts */ + octeon_dev->fn_list.enable_interrupt(octeon_dev->chip); + + /* Enable the input and output queues for this Octeon device */ + octeon_dev->fn_list.enable_io_queues(octeon_dev); + + atomic_set(&octeon_dev->status, OCT_DEV_IO_QUEUES_DONE); + + dev_dbg(&octeon_dev->pci_dev->dev, "Waiting for DDR initialization...\n"); + + if (ddr_timeout == 0) { + dev_info(&octeon_dev->pci_dev->dev, + "WAITING. Set ddr_timeout to non-zero value to proceed with initialization.\n"); + } + + schedule_timeout_uninterruptible(HZ * LIO_RESET_SECS); + + /* Wait for the octeon to initialize DDR after the soft-reset. */ + ret = octeon_wait_for_ddr_init(octeon_dev, &ddr_timeout); + if (ret) { + dev_err(&octeon_dev->pci_dev->dev, + "DDR not initialized. Please confirm that board is configured to boot from Flash, ret: %d\n", + ret); + return 1; + } + + if (octeon_wait_for_bootloader(octeon_dev, 1000) != 0) { + dev_err(&octeon_dev->pci_dev->dev, "Board not responding\n"); + return 1; + } + + dev_dbg(&octeon_dev->pci_dev->dev, "Initializing consoles\n"); + ret = octeon_init_consoles(octeon_dev); + if (ret) { + dev_err(&octeon_dev->pci_dev->dev, "Could not access board consoles\n"); + return 1; + } + ret = octeon_add_console(octeon_dev, 0); + if (ret) { + dev_err(&octeon_dev->pci_dev->dev, "Could not access board console\n"); + return 1; + } + + atomic_set(&octeon_dev->status, OCT_DEV_CONSOLE_INIT_DONE); + + dev_dbg(&octeon_dev->pci_dev->dev, "Loading firmware\n"); + ret = load_firmware(octeon_dev); + if (ret) { + dev_err(&octeon_dev->pci_dev->dev, "Could not load firmware to board\n"); + return 1; + } + + handshake[octeon_dev->octeon_id].init_ok = 1; + complete(&handshake[octeon_dev->octeon_id].init); + + atomic_set(&octeon_dev->status, OCT_DEV_HOST_OK); + + /* Send Credit for Octeon Output queues. Credits are always sent after + * the output queue is enabled. + */ + for (j = 0; j < octeon_dev->num_oqs; j++) + writel(octeon_dev->droq[j]->max_count, + octeon_dev->droq[j]->pkts_credit_reg); + + /* Packets can start arriving on the output queues from this point. */ + + return 0; +} + +/** + * \brief Exits the module + */ +static void __exit liquidio_exit(void) +{ + liquidio_deinit_pci(); + + pr_info("LiquidIO network module is now unloaded\n"); +} + +module_init(liquidio_init); +module_exit(liquidio_exit); diff --git a/kernel/drivers/net/ethernet/cavium/liquidio/liquidio_common.h b/kernel/drivers/net/ethernet/cavium/liquidio/liquidio_common.h new file mode 100644 index 000000000..0ac347ccc --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/liquidio/liquidio_common.h @@ -0,0 +1,673 @@ +/********************************************************************** +* Author: Cavium, Inc. +* +* Contact: support@cavium.com +* Please include "LiquidIO" in the subject. +* +* Copyright (c) 2003-2015 Cavium, Inc. +* +* This file is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License, Version 2, as +* published by the Free Software Foundation. +* +* This file is distributed in the hope that it will be useful, but +* AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty +* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or +* NONINFRINGEMENT. See the GNU General Public License for more +* details. +* +* This file may also be available under a different license from Cavium. +* Contact Cavium, Inc. for more information +**********************************************************************/ + +/*! \file liquidio_common.h + * \brief Common: Structures and macros used in PCI-NIC package by core and + * host driver. + */ + +#ifndef __LIQUIDIO_COMMON_H__ +#define __LIQUIDIO_COMMON_H__ + +#include "octeon_config.h" + +#define LIQUIDIO_VERSION "1.1.9" +#define LIQUIDIO_MAJOR_VERSION 1 +#define LIQUIDIO_MINOR_VERSION 1 +#define LIQUIDIO_MICRO_VERSION 9 + +#define CONTROL_IQ 0 +/** Tag types used by Octeon cores in its work. */ +enum octeon_tag_type { + ORDERED_TAG = 0, + ATOMIC_TAG = 1, + NULL_TAG = 2, + NULL_NULL_TAG = 3 +}; + +/* pre-defined host->NIC tag values */ +#define LIO_CONTROL (0x11111110) +#define LIO_DATA(i) (0x11111111 + (i)) + +/* Opcodes used by host driver/apps to perform operations on the core. + * These are used to identify the major subsystem that the operation + * is for. + */ +#define OPCODE_CORE 0 /* used for generic core operations */ +#define OPCODE_NIC 1 /* used for NIC operations */ +#define OPCODE_LAST OPCODE_NIC + +/* Subcodes are used by host driver/apps to identify the sub-operation + * for the core. They only need to by unique for a given subsystem. + */ +#define OPCODE_SUBCODE(op, sub) (((op & 0x0f) << 8) | ((sub) & 0x7f)) + +/** OPCODE_CORE subcodes. For future use. */ + +/** OPCODE_NIC subcodes */ + +/* This subcode is sent by core PCI driver to indicate cores are ready. */ +#define OPCODE_NIC_CORE_DRV_ACTIVE 0x01 +#define OPCODE_NIC_NW_DATA 0x02 /* network packet data */ +#define OPCODE_NIC_CMD 0x03 +#define OPCODE_NIC_INFO 0x04 +#define OPCODE_NIC_PORT_STATS 0x05 +#define OPCODE_NIC_MDIO45 0x06 +#define OPCODE_NIC_TIMESTAMP 0x07 +#define OPCODE_NIC_INTRMOD_CFG 0x08 +#define OPCODE_NIC_IF_CFG 0x09 + +#define CORE_DRV_TEST_SCATTER_OP 0xFFF5 + +#define OPCODE_SLOW_PATH(rh) \ + (OPCODE_SUBCODE(rh->r.opcode, rh->r.subcode) != \ + OPCODE_SUBCODE(OPCODE_NIC, OPCODE_NIC_NW_DATA)) + +/* Application codes advertised by the core driver initialization packet. */ +#define CVM_DRV_APP_START 0x0 +#define CVM_DRV_NO_APP 0 +#define CVM_DRV_APP_COUNT 0x2 +#define CVM_DRV_BASE_APP (CVM_DRV_APP_START + 0x0) +#define CVM_DRV_NIC_APP (CVM_DRV_APP_START + 0x1) +#define CVM_DRV_INVALID_APP (CVM_DRV_APP_START + 0x2) +#define CVM_DRV_APP_END (CVM_DRV_INVALID_APP - 1) + +/* Macro to increment index. + * Index is incremented by count; if the sum exceeds + * max, index is wrapped-around to the start. + */ +#define INCR_INDEX(index, count, max) \ +do { \ + if (((index) + (count)) >= (max)) \ + index = ((index) + (count)) - (max); \ + else \ + index += (count); \ +} while (0) + +#define INCR_INDEX_BY1(index, max) \ +do { \ + if ((++(index)) == (max)) \ + index = 0; \ +} while (0) + +#define DECR_INDEX(index, count, max) \ +do { \ + if ((count) > (index)) \ + index = ((max) - ((count - index))); \ + else \ + index -= count; \ +} while (0) + +#define OCT_BOARD_NAME 32 +#define OCT_SERIAL_LEN 64 + +/* Structure used by core driver to send indication that the Octeon + * application is ready. + */ +struct octeon_core_setup { + u64 corefreq; + + char boardname[OCT_BOARD_NAME]; + + char board_serial_number[OCT_SERIAL_LEN]; + + u64 board_rev_major; + + u64 board_rev_minor; + +}; + +/*--------------------------- SCATTER GATHER ENTRY -----------------------*/ + +/* The Scatter-Gather List Entry. The scatter or gather component used with + * a Octeon input instruction has this format. + */ +struct octeon_sg_entry { + /** The first 64 bit gives the size of data in each dptr.*/ + union { + u16 size[4]; + u64 size64; + } u; + + /** The 4 dptr pointers for this entry. */ + u64 ptr[4]; + +}; + +#define OCT_SG_ENTRY_SIZE (sizeof(struct octeon_sg_entry)) + +/* \brief Add size to gather list + * @param sg_entry scatter/gather entry + * @param size size to add + * @param pos position to add it. + */ +static inline void add_sg_size(struct octeon_sg_entry *sg_entry, + u16 size, + u32 pos) +{ +#ifdef __BIG_ENDIAN_BITFIELD + sg_entry->u.size[pos] = size; +#else + sg_entry->u.size[3 - pos] = size; +#endif +} + +/*------------------------- End Scatter/Gather ---------------------------*/ + +#define OCTNET_FRM_PTP_HEADER_SIZE 8 +#define OCTNET_FRM_HEADER_SIZE 30 /* PTP timestamp + VLAN + Ethernet */ + +#define OCTNET_MIN_FRM_SIZE (64 + OCTNET_FRM_PTP_HEADER_SIZE) +#define OCTNET_MAX_FRM_SIZE (16000 + OCTNET_FRM_HEADER_SIZE) + +#define OCTNET_DEFAULT_FRM_SIZE (1500 + OCTNET_FRM_HEADER_SIZE) + +/** NIC Commands are sent using this Octeon Input Queue */ +#define OCTNET_CMD_Q 0 + +/* NIC Command types */ +#define OCTNET_CMD_CHANGE_MTU 0x1 +#define OCTNET_CMD_CHANGE_MACADDR 0x2 +#define OCTNET_CMD_CHANGE_DEVFLAGS 0x3 +#define OCTNET_CMD_RX_CTL 0x4 + +#define OCTNET_CMD_SET_MULTI_LIST 0x5 +#define OCTNET_CMD_CLEAR_STATS 0x6 + +/* command for setting the speed, duplex & autoneg */ +#define OCTNET_CMD_SET_SETTINGS 0x7 +#define OCTNET_CMD_SET_FLOW_CTL 0x8 + +#define OCTNET_CMD_MDIO_READ_WRITE 0x9 +#define OCTNET_CMD_GPIO_ACCESS 0xA +#define OCTNET_CMD_LRO_ENABLE 0xB +#define OCTNET_CMD_LRO_DISABLE 0xC +#define OCTNET_CMD_SET_RSS 0xD +#define OCTNET_CMD_WRITE_SA 0xE +#define OCTNET_CMD_DELETE_SA 0xF +#define OCTNET_CMD_UPDATE_SA 0x12 + +#define OCTNET_CMD_TNL_RX_CSUM_CTL 0x10 +#define OCTNET_CMD_TNL_TX_CSUM_CTL 0x11 +#define OCTNET_CMD_IPSECV2_AH_ESP_CTL 0x13 +#define OCTNET_CMD_VERBOSE_ENABLE 0x14 +#define OCTNET_CMD_VERBOSE_DISABLE 0x15 + +/* RX(packets coming from wire) Checksum verification flags */ +/* TCP/UDP csum */ +#define CNNIC_L4SUM_VERIFIED 0x1 +#define CNNIC_IPSUM_VERIFIED 0x2 +#define CNNIC_TUN_CSUM_VERIFIED 0x4 +#define CNNIC_CSUM_VERIFIED (CNNIC_IPSUM_VERIFIED | CNNIC_L4SUM_VERIFIED) + +/*LROIPV4 and LROIPV6 Flags*/ +#define OCTNIC_LROIPV4 0x1 +#define OCTNIC_LROIPV6 0x2 + +/* Interface flags communicated between host driver and core app. */ +enum octnet_ifflags { + OCTNET_IFFLAG_PROMISC = 0x01, + OCTNET_IFFLAG_ALLMULTI = 0x02, + OCTNET_IFFLAG_MULTICAST = 0x04, + OCTNET_IFFLAG_BROADCAST = 0x08, + OCTNET_IFFLAG_UNICAST = 0x10 +}; + +/* wqe + * --------------- 0 + * | wqe word0-3 | + * --------------- 32 + * | PCI IH | + * --------------- 40 + * | RPTR | + * --------------- 48 + * | PCI IRH | + * --------------- 56 + * | OCT_NET_CMD | + * --------------- 64 + * | Addtl 8-BData | + * | | + * --------------- + */ + +union octnet_cmd { + u64 u64; + + struct { +#ifdef __BIG_ENDIAN_BITFIELD + u64 cmd:5; + + u64 more:6; /* How many udd words follow the command */ + + u64 param1:29; + + u64 param2:16; + + u64 param3:8; + +#else + + u64 param3:8; + + u64 param2:16; + + u64 param1:29; + + u64 more:6; + + u64 cmd:5; + +#endif + } s; + +}; + +#define OCTNET_CMD_SIZE (sizeof(union octnet_cmd)) + +/** Instruction Header */ +struct octeon_instr_ih { +#ifdef __BIG_ENDIAN_BITFIELD + /** Raw mode indicator 1 = RAW */ + u64 raw:1; + + /** Gather indicator 1=gather*/ + u64 gather:1; + + /** Data length OR no. of entries in gather list */ + u64 dlengsz:14; + + /** Front Data size */ + u64 fsz:6; + + /** Packet Order / Work Unit selection (1 of 8)*/ + u64 qos:3; + + /** Core group selection (1 of 16) */ + u64 grp:4; + + /** Short Raw Packet Indicator 1=short raw pkt */ + u64 rs:1; + + /** Tag type */ + u64 tagtype:2; + + /** Tag Value */ + u64 tag:32; +#else + /** Tag Value */ + u64 tag:32; + + /** Tag type */ + u64 tagtype:2; + + /** Short Raw Packet Indicator 1=short raw pkt */ + u64 rs:1; + + /** Core group selection (1 of 16) */ + u64 grp:4; + + /** Packet Order / Work Unit selection (1 of 8)*/ + u64 qos:3; + + /** Front Data size */ + u64 fsz:6; + + /** Data length OR no. of entries in gather list */ + u64 dlengsz:14; + + /** Gather indicator 1=gather*/ + u64 gather:1; + + /** Raw mode indicator 1 = RAW */ + u64 raw:1; +#endif +}; + +/** Input Request Header */ +struct octeon_instr_irh { +#ifdef __BIG_ENDIAN_BITFIELD + u64 opcode:4; + u64 rflag:1; + u64 subcode:7; + u64 len:3; + u64 rid:13; + u64 reserved:4; + u64 ossp:32; /* opcode/subcode specific parameters */ +#else + u64 ossp:32; /* opcode/subcode specific parameters */ + u64 reserved:4; + u64 rid:13; + u64 len:3; + u64 subcode:7; + u64 rflag:1; + u64 opcode:4; +#endif +}; + +/** Return Data Parameters */ +struct octeon_instr_rdp { +#ifdef __BIG_ENDIAN_BITFIELD + u64 reserved:49; + u64 pcie_port:3; + u64 rlen:12; +#else + u64 rlen:12; + u64 pcie_port:3; + u64 reserved:49; +#endif +}; + +/** Receive Header */ +union octeon_rh { +#ifdef __BIG_ENDIAN_BITFIELD + u64 u64; + struct { + u64 opcode:4; + u64 subcode:8; + u64 len:3; /** additional 64-bit words */ + u64 rid:13; /** request id in response to pkt sent by host */ + u64 reserved:4; + u64 ossp:32; /** opcode/subcode specific parameters */ + } r; + struct { + u64 opcode:4; + u64 subcode:8; + u64 len:3; /** additional 64-bit words */ + u64 rid:13; /** request id in response to pkt sent by host */ + u64 extra:24; + u64 link:8; + u64 csum_verified:3; /** checksum verified. */ + u64 has_hwtstamp:1; /** Has hardware timestamp. 1 = yes. */ + } r_dh; + struct { + u64 opcode:4; + u64 subcode:8; + u64 len:3; /** additional 64-bit words */ + u64 rid:13; /** request id in response to pkt sent by host */ + u64 num_gmx_ports:8; + u64 max_nic_ports:8; + u64 app_cap_flags:4; + u64 app_mode:16; + } r_core_drv_init; + struct { + u64 opcode:4; + u64 subcode:8; + u64 len:3; /** additional 64-bit words */ + u64 rid:13; + u64 reserved:4; + u64 extra:25; + u64 ifidx:7; + } r_nic_info; +#else + u64 u64; + struct { + u64 ossp:32; /** opcode/subcode specific parameters */ + u64 reserved:4; + u64 rid:13; /** req id in response to pkt sent by host */ + u64 len:3; /** additional 64-bit words */ + u64 subcode:8; + u64 opcode:4; + } r; + struct { + u64 has_hwtstamp:1; /** 1 = has hwtstamp */ + u64 csum_verified:3; /** checksum verified. */ + u64 link:8; + u64 extra:24; + u64 rid:13; /** req id in response to pkt sent by host */ + u64 len:3; /** additional 64-bit words */ + u64 subcode:8; + u64 opcode:4; + } r_dh; + struct { + u64 app_mode:16; + u64 app_cap_flags:4; + u64 max_nic_ports:8; + u64 num_gmx_ports:8; + u64 rid:13; + u64 len:3; /** additional 64-bit words */ + u64 subcode:8; + u64 opcode:4; + } r_core_drv_init; + struct { + u64 ifidx:7; + u64 extra:25; + u64 reserved:4; + u64 rid:13; + u64 len:3; /** additional 64-bit words */ + u64 subcode:8; + u64 opcode:4; + } r_nic_info; +#endif +}; + +#define OCT_RH_SIZE (sizeof(union octeon_rh)) + +#define OCT_PKT_PARAM_IPV4OPTS 1 +#define OCT_PKT_PARAM_IPV6EXTHDR 2 + +union octnic_packet_params { + u32 u32; + struct { +#ifdef __BIG_ENDIAN_BITFIELD + u32 reserved:6; + u32 tnl_csum:1; + u32 ip_csum:1; + u32 ipv4opts_ipv6exthdr:2; + u32 ipsec_ops:4; + u32 tsflag:1; + u32 csoffset:9; + u32 ifidx:8; +#else + u32 ifidx:8; + u32 csoffset:9; + u32 tsflag:1; + u32 ipsec_ops:4; + u32 ipv4opts_ipv6exthdr:2; + u32 ip_csum:1; + u32 tnl_csum:1; + u32 reserved:6; +#endif + } s; +}; + +/** Status of a RGMII Link on Octeon as seen by core driver. */ +union oct_link_status { + u64 u64; + + struct { +#ifdef __BIG_ENDIAN_BITFIELD + u64 duplex:8; + u64 status:8; + u64 mtu:16; + u64 speed:16; + u64 autoneg:1; + u64 interface:4; + u64 pause:1; + u64 reserved:10; +#else + u64 reserved:10; + u64 pause:1; + u64 interface:4; + u64 autoneg:1; + u64 speed:16; + u64 mtu:16; + u64 status:8; + u64 duplex:8; +#endif + } s; +}; + +/** Information for a OCTEON ethernet interface shared between core & host. */ +struct oct_link_info { + union oct_link_status link; + u64 hw_addr; + +#ifdef __BIG_ENDIAN_BITFIELD + u16 gmxport; + u8 rsvd[3]; + u8 num_txpciq; + u8 num_rxpciq; + u8 ifidx; +#else + u8 ifidx; + u8 num_rxpciq; + u8 num_txpciq; + u8 rsvd[3]; + u16 gmxport; +#endif + + u8 txpciq[MAX_IOQS_PER_NICIF]; + u8 rxpciq[MAX_IOQS_PER_NICIF]; +}; + +#define OCT_LINK_INFO_SIZE (sizeof(struct oct_link_info)) + +struct liquidio_if_cfg_info { + u64 ifidx; + u64 iqmask; /** mask for IQs enabled for the port */ + u64 oqmask; /** mask for OQs enabled for the port */ + struct oct_link_info linfo; /** initial link information */ +}; + +/** Stats for each NIC port in RX direction. */ +struct nic_rx_stats { + /* link-level stats */ + u64 total_rcvd; + u64 bytes_rcvd; + u64 total_bcst; + u64 total_mcst; + u64 runts; + u64 ctl_rcvd; + u64 fifo_err; /* Accounts for over/under-run of buffers */ + u64 dmac_drop; + u64 fcs_err; + u64 jabber_err; + u64 l2_err; + u64 frame_err; + + /* firmware stats */ + u64 fw_total_rcvd; + u64 fw_total_fwd; + u64 fw_err_pko; + u64 fw_err_link; + u64 fw_err_drop; + u64 fw_lro_pkts; /* Number of packets that are LROed */ + u64 fw_lro_octs; /* Number of octets that are LROed */ + u64 fw_total_lro; /* Number of LRO packets formed */ + u64 fw_lro_aborts; /* Number of times lRO of packet aborted */ + /* intrmod: packet forward rate */ + u64 fwd_rate; +}; + +/** Stats for each NIC port in RX direction. */ +struct nic_tx_stats { + /* link-level stats */ + u64 total_pkts_sent; + u64 total_bytes_sent; + u64 mcast_pkts_sent; + u64 bcast_pkts_sent; + u64 ctl_sent; + u64 one_collision_sent; /* Packets sent after one collision*/ + u64 multi_collision_sent; /* Packets sent after multiple collision*/ + u64 max_collision_fail; /* Packets not sent due to max collisions */ + u64 max_deferral_fail; /* Packets not sent due to max deferrals */ + u64 fifo_err; /* Accounts for over/under-run of buffers */ + u64 runts; + u64 total_collisions; /* Total number of collisions detected */ + + /* firmware stats */ + u64 fw_total_sent; + u64 fw_total_fwd; + u64 fw_err_pko; + u64 fw_err_link; + u64 fw_err_drop; +}; + +struct oct_link_stats { + struct nic_rx_stats fromwire; + struct nic_tx_stats fromhost; + +}; + +#define LIO68XX_LED_CTRL_ADDR 0x3501 +#define LIO68XX_LED_CTRL_CFGON 0x1f +#define LIO68XX_LED_CTRL_CFGOFF 0x100 +#define LIO68XX_LED_BEACON_ADDR 0x3508 +#define LIO68XX_LED_BEACON_CFGON 0x47fd +#define LIO68XX_LED_BEACON_CFGOFF 0x11fc +#define VITESSE_PHY_GPIO_DRIVEON 0x1 +#define VITESSE_PHY_GPIO_CFG 0x8 +#define VITESSE_PHY_GPIO_DRIVEOFF 0x4 +#define VITESSE_PHY_GPIO_HIGH 0x2 +#define VITESSE_PHY_GPIO_LOW 0x3 + +struct oct_mdio_cmd { + u64 op; + u64 mdio_addr; + u64 value1; + u64 value2; + u64 value3; +}; + +#define OCT_LINK_STATS_SIZE (sizeof(struct oct_link_stats)) + +#define LIO_INTRMOD_CHECK_INTERVAL 1 +#define LIO_INTRMOD_MAXPKT_RATETHR 196608 /* max pkt rate threshold */ +#define LIO_INTRMOD_MINPKT_RATETHR 9216 /* min pkt rate threshold */ +#define LIO_INTRMOD_MAXCNT_TRIGGER 384 /* max pkts to trigger interrupt */ +#define LIO_INTRMOD_MINCNT_TRIGGER 1 /* min pkts to trigger interrupt */ +#define LIO_INTRMOD_MAXTMR_TRIGGER 128 /* max time to trigger interrupt */ +#define LIO_INTRMOD_MINTMR_TRIGGER 32 /* min time to trigger interrupt */ + +struct oct_intrmod_cfg { + u64 intrmod_enable; + u64 intrmod_check_intrvl; + u64 intrmod_maxpkt_ratethr; + u64 intrmod_minpkt_ratethr; + u64 intrmod_maxcnt_trigger; + u64 intrmod_maxtmr_trigger; + u64 intrmod_mincnt_trigger; + u64 intrmod_mintmr_trigger; +}; + +#define BASE_QUEUE_NOT_REQUESTED 65535 + +union oct_nic_if_cfg { + u64 u64; + struct { +#ifdef __BIG_ENDIAN_BITFIELD + u64 base_queue:16; + u64 num_iqueues:16; + u64 num_oqueues:16; + u64 gmx_port_id:8; + u64 reserved:8; +#else + u64 reserved:8; + u64 gmx_port_id:8; + u64 num_oqueues:16; + u64 num_iqueues:16; + u64 base_queue:16; +#endif + } s; +}; + +#endif diff --git a/kernel/drivers/net/ethernet/cavium/liquidio/liquidio_image.h b/kernel/drivers/net/ethernet/cavium/liquidio/liquidio_image.h new file mode 100644 index 000000000..93819bd86 --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/liquidio/liquidio_image.h @@ -0,0 +1,57 @@ +/********************************************************************** +* Author: Cavium, Inc. +* +* Contact: support@cavium.com +* Please include "LiquidIO" in the subject. +* +* Copyright (c) 2003-2015 Cavium, Inc. +* +* This file is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License, Version 2, as +* published by the Free Software Foundation. +* +* This file is distributed in the hope that it will be useful, but +* AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty +* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or +* NONINFRINGEMENT. See the GNU General Public License for more +* details. +* +* This file may also be available under a different license from Cavium. +* Contact Cavium, Inc. for more information +**********************************************************************/ +#ifndef _LIQUIDIO_IMAGE_H_ +#define _LIQUIDIO_IMAGE_H_ + +#define LIO_MAX_FW_TYPE_LEN (8) +#define LIO_MAX_FW_FILENAME_LEN (256) +#define LIO_FW_DIR "liquidio/" +#define LIO_FW_BASE_NAME "lio_" +#define LIO_FW_NAME_SUFFIX ".bin" +#define LIO_FW_NAME_TYPE_NIC "nic" +#define LIO_FW_NAME_TYPE_NONE "none" +#define LIO_MAX_FIRMWARE_VERSION_LEN 16 + +#define LIO_MAX_BOOTCMD_LEN 1024 +#define LIO_MAX_IMAGES 16 +#define LIO_NIC_MAGIC 0x434E4943 /* "CNIC" */ +struct octeon_firmware_desc { + __be64 addr; + __be32 len; + __be32 crc32; /* crc32 of image */ +}; + +/* Following the header is a list of 64-bit aligned binary images, + * as described by the desc field. + * Numeric fields are in network byte order. + */ +struct octeon_firmware_file_header { + __be32 magic; + char version[LIO_MAX_FIRMWARE_VERSION_LEN]; + char bootcmd[LIO_MAX_BOOTCMD_LEN]; + __be32 num_images; + struct octeon_firmware_desc desc[LIO_MAX_IMAGES]; + __be32 pad; + __be32 crc32; /* header checksum */ +}; + +#endif /* _LIQUIDIO_IMAGE_H_ */ diff --git a/kernel/drivers/net/ethernet/cavium/liquidio/octeon_config.h b/kernel/drivers/net/ethernet/cavium/liquidio/octeon_config.h new file mode 100644 index 000000000..62a8dd5cd --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/liquidio/octeon_config.h @@ -0,0 +1,424 @@ +/********************************************************************** +* Author: Cavium, Inc. +* +* Contact: support@cavium.com +* Please include "LiquidIO" in the subject. +* +* Copyright (c) 2003-2015 Cavium, Inc. +* +* This file is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License, Version 2, as +* published by the Free Software Foundation. +* +* This file is distributed in the hope that it will be useful, but +* AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty +* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or +* NONINFRINGEMENT. See the GNU General Public License for more +* details. +* +* This file may also be available under a different license from Cavium. +* Contact Cavium, Inc. for more information +**********************************************************************/ + +/*! \file octeon_config.h + * \brief Host Driver: Configuration data structures for the host driver. + */ + +#ifndef __OCTEON_CONFIG_H__ +#define __OCTEON_CONFIG_H__ + +/*--------------------------CONFIG VALUES------------------------*/ + +/* The following macros affect the way the driver data structures + * are generated for Octeon devices. + * They can be modified. + */ + +/* Maximum octeon devices defined as MAX_OCTEON_NICIF to support + * multiple(<= MAX_OCTEON_NICIF) Miniports + */ +#define MAX_OCTEON_NICIF 32 +#define MAX_OCTEON_DEVICES MAX_OCTEON_NICIF +#define MAX_OCTEON_LINKS MAX_OCTEON_NICIF +#define MAX_OCTEON_MULTICAST_ADDR 32 + +/* CN6xxx IQ configuration macros */ +#define CN6XXX_MAX_INPUT_QUEUES 32 +#define CN6XXX_MAX_IQ_DESCRIPTORS 2048 +#define CN6XXX_DB_MIN 1 +#define CN6XXX_DB_MAX 8 +#define CN6XXX_DB_TIMEOUT 1 + +/* CN6xxx OQ configuration macros */ +#define CN6XXX_MAX_OUTPUT_QUEUES 32 +#define CN6XXX_MAX_OQ_DESCRIPTORS 2048 +#define CN6XXX_OQ_BUF_SIZE 1536 +#define CN6XXX_OQ_PKTSPER_INTR ((CN6XXX_MAX_OQ_DESCRIPTORS < 512) ? \ + (CN6XXX_MAX_OQ_DESCRIPTORS / 4) : 128) +#define CN6XXX_OQ_REFIL_THRESHOLD ((CN6XXX_MAX_OQ_DESCRIPTORS < 512) ? \ + (CN6XXX_MAX_OQ_DESCRIPTORS / 4) : 128) + +#define CN6XXX_OQ_INTR_PKT 64 +#define CN6XXX_OQ_INTR_TIME 100 +#define DEFAULT_NUM_NIC_PORTS_66XX 2 +#define DEFAULT_NUM_NIC_PORTS_68XX 4 +#define DEFAULT_NUM_NIC_PORTS_68XX_210NV 2 + +/* common OCTEON configuration macros */ +#define CN6XXX_CFG_IO_QUEUES 32 +#define OCTEON_32BYTE_INSTR 32 +#define OCTEON_64BYTE_INSTR 64 +#define OCTEON_MAX_BASE_IOQ 4 +#define OCTEON_OQ_BUFPTR_MODE 0 +#define OCTEON_OQ_INFOPTR_MODE 1 + +#define OCTEON_DMA_INTR_PKT 64 +#define OCTEON_DMA_INTR_TIME 1000 + +#define MAX_TXQS_PER_INTF 8 +#define MAX_RXQS_PER_INTF 8 +#define DEF_TXQS_PER_INTF 4 +#define DEF_RXQS_PER_INTF 4 + +#define INVALID_IOQ_NO 0xff + +#define DEFAULT_POW_GRP 0 + +/* Macros to get octeon config params */ +#define CFG_GET_IQ_CFG(cfg) ((cfg)->iq) +#define CFG_GET_IQ_MAX_Q(cfg) ((cfg)->iq.max_iqs) +#define CFG_GET_IQ_PENDING_LIST_SIZE(cfg) ((cfg)->iq.pending_list_size) +#define CFG_GET_IQ_INSTR_TYPE(cfg) ((cfg)->iq.instr_type) +#define CFG_GET_IQ_DB_MIN(cfg) ((cfg)->iq.db_min) +#define CFG_GET_IQ_DB_TIMEOUT(cfg) ((cfg)->iq.db_timeout) + +#define CFG_GET_OQ_MAX_Q(cfg) ((cfg)->oq.max_oqs) +#define CFG_GET_OQ_INFO_PTR(cfg) ((cfg)->oq.info_ptr) +#define CFG_GET_OQ_PKTS_PER_INTR(cfg) ((cfg)->oq.pkts_per_intr) +#define CFG_GET_OQ_REFILL_THRESHOLD(cfg) ((cfg)->oq.refill_threshold) +#define CFG_GET_OQ_INTR_PKT(cfg) ((cfg)->oq.oq_intr_pkt) +#define CFG_GET_OQ_INTR_TIME(cfg) ((cfg)->oq.oq_intr_time) +#define CFG_SET_OQ_INTR_PKT(cfg, val) (cfg)->oq.oq_intr_pkt = val +#define CFG_SET_OQ_INTR_TIME(cfg, val) (cfg)->oq.oq_intr_time = val + +#define CFG_GET_DMA_INTR_PKT(cfg) ((cfg)->dma.dma_intr_pkt) +#define CFG_GET_DMA_INTR_TIME(cfg) ((cfg)->dma.dma_intr_time) +#define CFG_GET_NUM_NIC_PORTS(cfg) ((cfg)->num_nic_ports) +#define CFG_GET_NUM_DEF_TX_DESCS(cfg) ((cfg)->num_def_tx_descs) +#define CFG_GET_NUM_DEF_RX_DESCS(cfg) ((cfg)->num_def_rx_descs) +#define CFG_GET_DEF_RX_BUF_SIZE(cfg) ((cfg)->def_rx_buf_size) + +#define CFG_GET_MAX_TXQS_NIC_IF(cfg, idx) \ + ((cfg)->nic_if_cfg[idx].max_txqs) +#define CFG_GET_NUM_TXQS_NIC_IF(cfg, idx) \ + ((cfg)->nic_if_cfg[idx].num_txqs) +#define CFG_GET_MAX_RXQS_NIC_IF(cfg, idx) \ + ((cfg)->nic_if_cfg[idx].max_rxqs) +#define CFG_GET_NUM_RXQS_NIC_IF(cfg, idx) \ + ((cfg)->nic_if_cfg[idx].num_rxqs) +#define CFG_GET_NUM_RX_DESCS_NIC_IF(cfg, idx) \ + ((cfg)->nic_if_cfg[idx].num_rx_descs) +#define CFG_GET_NUM_TX_DESCS_NIC_IF(cfg, idx) \ + ((cfg)->nic_if_cfg[idx].num_tx_descs) +#define CFG_GET_NUM_RX_BUF_SIZE_NIC_IF(cfg, idx) \ + ((cfg)->nic_if_cfg[idx].rx_buf_size) +#define CFG_GET_BASE_QUE_NIC_IF(cfg, idx) \ + ((cfg)->nic_if_cfg[idx].base_queue) +#define CFG_GET_GMXID_NIC_IF(cfg, idx) \ + ((cfg)->nic_if_cfg[idx].gmx_port_id) + +#define CFG_GET_CTRL_Q_GRP(cfg) ((cfg)->misc.ctrlq_grp) +#define CFG_GET_HOST_LINK_QUERY_INTERVAL(cfg) \ + ((cfg)->misc.host_link_query_interval) +#define CFG_GET_OCT_LINK_QUERY_INTERVAL(cfg) \ + ((cfg)->misc.oct_link_query_interval) +#define CFG_GET_IS_SLI_BP_ON(cfg) ((cfg)->misc.enable_sli_oq_bp) + +/* Max IOQs per OCTEON Link */ +#define MAX_IOQS_PER_NICIF 32 + +enum lio_card_type { + LIO_210SV = 0, /* Two port, 66xx */ + LIO_210NV, /* Two port, 68xx */ + LIO_410NV /* Four port, 68xx */ +}; + +#define LIO_210SV_NAME "210sv" +#define LIO_210NV_NAME "210nv" +#define LIO_410NV_NAME "410nv" + +/** Structure to define the configuration attributes for each Input queue. + * Applicable to all Octeon processors + **/ +struct octeon_iq_config { +#ifdef __BIG_ENDIAN_BITFIELD + u64 reserved:32; + + /** Minimum ticks to wait before checking for pending instructions. */ + u64 db_timeout:16; + + /** Minimum number of commands pending to be posted to Octeon + * before driver hits the Input queue doorbell. + */ + u64 db_min:8; + + /** Command size - 32 or 64 bytes */ + u64 instr_type:32; + + /** Pending list size (usually set to the sum of the size of all Input + * queues) + */ + u64 pending_list_size:32; + + /* Max number of IQs available */ + u64 max_iqs:8; +#else + /* Max number of IQs available */ + u64 max_iqs:8; + + /** Pending list size (usually set to the sum of the size of all Input + * queues) + */ + u64 pending_list_size:32; + + /** Command size - 32 or 64 bytes */ + u64 instr_type:32; + + /** Minimum number of commands pending to be posted to Octeon + * before driver hits the Input queue doorbell. + */ + u64 db_min:8; + + /** Minimum ticks to wait before checking for pending instructions. */ + u64 db_timeout:16; + + u64 reserved:32; +#endif +}; + +/** Structure to define the configuration attributes for each Output queue. + * Applicable to all Octeon processors + **/ +struct octeon_oq_config { +#ifdef __BIG_ENDIAN_BITFIELD + u64 reserved:16; + + u64 pkts_per_intr:16; + + /** Interrupt Coalescing (Time Interval). Octeon will interrupt the + * host if atleast one packet was sent in the time interval specified + * by this field. The driver uses time interval interrupt coalescing + * by default. The time is specified in microseconds. + */ + u64 oq_intr_time:16; + + /** Interrupt Coalescing (Packet Count). Octeon will interrupt the host + * only if it sent as many packets as specified by this field. + * The driver + * usually does not use packet count interrupt coalescing. + */ + u64 oq_intr_pkt:16; + + /** The number of buffers that were consumed during packet processing by + * the driver on this Output queue before the driver attempts to + * replenish + * the descriptor ring with new buffers. + */ + u64 refill_threshold:16; + + /** If set, the Output queue uses info-pointer mode. (Default: 1 ) */ + u64 info_ptr:32; + + /* Max number of OQs available */ + u64 max_oqs:8; + +#else + /* Max number of OQs available */ + u64 max_oqs:8; + + /** If set, the Output queue uses info-pointer mode. (Default: 1 ) */ + u64 info_ptr:32; + + /** The number of buffers that were consumed during packet processing by + * the driver on this Output queue before the driver attempts to + * replenish + * the descriptor ring with new buffers. + */ + u64 refill_threshold:16; + + /** Interrupt Coalescing (Packet Count). Octeon will interrupt the host + * only if it sent as many packets as specified by this field. + * The driver + * usually does not use packet count interrupt coalescing. + */ + u64 oq_intr_pkt:16; + + /** Interrupt Coalescing (Time Interval). Octeon will interrupt the + * host if atleast one packet was sent in the time interval specified + * by this field. The driver uses time interval interrupt coalescing + * by default. The time is specified in microseconds. + */ + u64 oq_intr_time:16; + + u64 pkts_per_intr:16; + + u64 reserved:16; +#endif + +}; + +/** This structure conatins the NIC link configuration attributes, + * common for all the OCTEON Modles. + */ +struct octeon_nic_if_config { +#ifdef __BIG_ENDIAN_BITFIELD + u64 reserved:56; + + u64 base_queue:16; + + u64 gmx_port_id:8; + + /* SKB size, We need not change buf size even for Jumbo frames. + * Octeon can send jumbo frames in 4 consecutive descriptors, + */ + u64 rx_buf_size:16; + + /* Num of desc for tx rings */ + u64 num_tx_descs:16; + + /* Num of desc for rx rings */ + u64 num_rx_descs:16; + + /* Actual configured value. Range could be: 1...max_rxqs */ + u64 num_rxqs:16; + + /* Max Rxqs: Half for each of the two ports :max_oq/2 */ + u64 max_rxqs:16; + + /* Actual configured value. Range could be: 1...max_txqs */ + u64 num_txqs:16; + + /* Max Txqs: Half for each of the two ports :max_iq/2 */ + u64 max_txqs:16; +#else + /* Max Txqs: Half for each of the two ports :max_iq/2 */ + u64 max_txqs:16; + + /* Actual configured value. Range could be: 1...max_txqs */ + u64 num_txqs:16; + + /* Max Rxqs: Half for each of the two ports :max_oq/2 */ + u64 max_rxqs:16; + + /* Actual configured value. Range could be: 1...max_rxqs */ + u64 num_rxqs:16; + + /* Num of desc for rx rings */ + u64 num_rx_descs:16; + + /* Num of desc for tx rings */ + u64 num_tx_descs:16; + + /* SKB size, We need not change buf size even for Jumbo frames. + * Octeon can send jumbo frames in 4 consecutive descriptors, + */ + u64 rx_buf_size:16; + + u64 gmx_port_id:8; + + u64 base_queue:16; + + u64 reserved:56; +#endif + +}; + +/** Structure to define the configuration attributes for meta data. + * Applicable to all Octeon processors. + */ + +struct octeon_misc_config { +#ifdef __BIG_ENDIAN_BITFIELD + /** Host link status polling period */ + u64 host_link_query_interval:32; + /** Oct link status polling period */ + u64 oct_link_query_interval:32; + + u64 enable_sli_oq_bp:1; + /** Control IQ Group */ + u64 ctrlq_grp:4; +#else + /** Control IQ Group */ + u64 ctrlq_grp:4; + /** BP for SLI OQ */ + u64 enable_sli_oq_bp:1; + /** Host link status polling period */ + u64 oct_link_query_interval:32; + /** Oct link status polling period */ + u64 host_link_query_interval:32; +#endif +}; + +/** Structure to define the configuration for all OCTEON processors. */ +struct octeon_config { + u16 card_type; + char *card_name; + + /** Input Queue attributes. */ + struct octeon_iq_config iq; + + /** Output Queue attributes. */ + struct octeon_oq_config oq; + + /** NIC Port Configuration */ + struct octeon_nic_if_config nic_if_cfg[MAX_OCTEON_NICIF]; + + /** Miscellaneous attributes */ + struct octeon_misc_config misc; + + int num_nic_ports; + + int num_def_tx_descs; + + /* Num of desc for rx rings */ + int num_def_rx_descs; + + int def_rx_buf_size; + +}; + +/* The following config values are fixed and should not be modified. */ + +/* Maximum address space to be mapped for Octeon's BAR1 index-based access. */ +#define MAX_BAR1_MAP_INDEX 2 +#define OCTEON_BAR1_ENTRY_SIZE (4 * 1024 * 1024) + +/* BAR1 Index 0 to (MAX_BAR1_MAP_INDEX - 1) for normal mapped memory access. + * Bar1 register at MAX_BAR1_MAP_INDEX used by driver for dynamic access. + */ +#define MAX_BAR1_IOREMAP_SIZE ((MAX_BAR1_MAP_INDEX + 1) * \ + OCTEON_BAR1_ENTRY_SIZE) + +/* Response lists - 1 ordered, 1 unordered-blocking, 1 unordered-nonblocking + * NoResponse Lists are now maintained with each IQ. (Dec' 2007). + */ +#define MAX_RESPONSE_LISTS 4 + +/* Opcode hash bits. The opcode is hashed on the lower 6-bits to lookup the + * dispatch table. + */ +#define OPCODE_MASK_BITS 6 + +/* Mask for the 6-bit lookup hash */ +#define OCTEON_OPCODE_MASK 0x3f + +/* Size of the dispatch table. The 6-bit hash can index into 2^6 entries */ +#define DISPATCH_LIST_SIZE BIT(OPCODE_MASK_BITS) + +/* Maximum number of Octeon Instruction (command) queues */ +#define MAX_OCTEON_INSTR_QUEUES CN6XXX_MAX_INPUT_QUEUES + +/* Maximum number of Octeon Instruction (command) queues */ +#define MAX_OCTEON_OUTPUT_QUEUES CN6XXX_MAX_OUTPUT_QUEUES + +#endif /* __OCTEON_CONFIG_H__ */ diff --git a/kernel/drivers/net/ethernet/cavium/liquidio/octeon_console.c b/kernel/drivers/net/ethernet/cavium/liquidio/octeon_console.c new file mode 100644 index 000000000..466147e40 --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/liquidio/octeon_console.c @@ -0,0 +1,723 @@ +/********************************************************************** +* Author: Cavium, Inc. +* +* Contact: support@cavium.com +* Please include "LiquidIO" in the subject. +* +* Copyright (c) 2003-2015 Cavium, Inc. +* +* This file is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License, Version 2, as +* published by the Free Software Foundation. +* +* This file is distributed in the hope that it will be useful, but +* AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty +* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or +* NONINFRINGEMENT. See the GNU General Public License for more +* details. +* +* This file may also be available under a different license from Cavium. +* Contact Cavium, Inc. for more information +**********************************************************************/ + +/** + * @file octeon_console.c + */ +#include <linux/version.h> +#include <linux/types.h> +#include <linux/list.h> +#include <linux/interrupt.h> +#include <linux/pci.h> +#include <linux/kthread.h> +#include <linux/netdevice.h> +#include "octeon_config.h" +#include "liquidio_common.h" +#include "octeon_droq.h" +#include "octeon_iq.h" +#include "response_manager.h" +#include "octeon_device.h" +#include "octeon_nic.h" +#include "octeon_main.h" +#include "octeon_network.h" +#include "cn66xx_regs.h" +#include "cn66xx_device.h" +#include "cn68xx_regs.h" +#include "cn68xx_device.h" +#include "liquidio_image.h" +#include "octeon_mem_ops.h" + +static void octeon_remote_lock(void); +static void octeon_remote_unlock(void); +static u64 cvmx_bootmem_phy_named_block_find(struct octeon_device *oct, + const char *name, + u32 flags); + +#define MIN(a, b) min((a), (b)) +#define CAST_ULL(v) ((u64)(v)) + +#define BOOTLOADER_PCI_READ_BUFFER_DATA_ADDR 0x0006c008 +#define BOOTLOADER_PCI_READ_BUFFER_LEN_ADDR 0x0006c004 +#define BOOTLOADER_PCI_READ_BUFFER_OWNER_ADDR 0x0006c000 +#define BOOTLOADER_PCI_READ_DESC_ADDR 0x0006c100 +#define BOOTLOADER_PCI_WRITE_BUFFER_STR_LEN 248 + +#define OCTEON_PCI_IO_BUF_OWNER_OCTEON 0x00000001 +#define OCTEON_PCI_IO_BUF_OWNER_HOST 0x00000002 + +/** Can change without breaking ABI */ +#define CVMX_BOOTMEM_NUM_NAMED_BLOCKS 64 + +/** minimum alignment of bootmem alloced blocks */ +#define CVMX_BOOTMEM_ALIGNMENT_SIZE (16ull) + +/** CVMX bootmem descriptor major version */ +#define CVMX_BOOTMEM_DESC_MAJ_VER 3 +/* CVMX bootmem descriptor minor version */ +#define CVMX_BOOTMEM_DESC_MIN_VER 0 + +/* Current versions */ +#define OCTEON_PCI_CONSOLE_MAJOR_VERSION 1 +#define OCTEON_PCI_CONSOLE_MINOR_VERSION 0 +#define OCTEON_PCI_CONSOLE_BLOCK_NAME "__pci_console" +#define OCTEON_CONSOLE_POLL_INTERVAL_MS 100 /* 10 times per second */ + +/* First three members of cvmx_bootmem_desc are left in original +** positions for backwards compatibility. +** Assumes big endian target +*/ +struct cvmx_bootmem_desc { + /** spinlock to control access to list */ + u32 lock; + + /** flags for indicating various conditions */ + u32 flags; + + u64 head_addr; + + /** incremented changed when incompatible changes made */ + u32 major_version; + + /** incremented changed when compatible changes made, + * reset to zero when major incremented + */ + u32 minor_version; + + u64 app_data_addr; + u64 app_data_size; + + /** number of elements in named blocks array */ + u32 nb_num_blocks; + + /** length of name array in bootmem blocks */ + u32 named_block_name_len; + + /** address of named memory block descriptors */ + u64 named_block_array_addr; +}; + +/* Structure that defines a single console. + * + * Note: when read_index == write_index, the buffer is empty. + * The actual usable size of each console is console_buf_size -1; + */ +struct octeon_pci_console { + u64 input_base_addr; + u32 input_read_index; + u32 input_write_index; + u64 output_base_addr; + u32 output_read_index; + u32 output_write_index; + u32 lock; + u32 buf_size; +}; + +/* This is the main container structure that contains all the information + * about all PCI consoles. The address of this structure is passed to various + * routines that operation on PCI consoles. + */ +struct octeon_pci_console_desc { + u32 major_version; + u32 minor_version; + u32 lock; + u32 flags; + u32 num_consoles; + u32 pad; + /* must be 64 bit aligned here... */ + /* Array of addresses of octeon_pci_console structures */ + u64 console_addr_array[0]; + /* Implicit storage for console_addr_array */ +}; + +/** + * This macro returns the size of a member of a structure. + * Logically it is the same as "sizeof(s::field)" in C++, but + * C lacks the "::" operator. + */ +#define SIZEOF_FIELD(s, field) sizeof(((s *)NULL)->field) + +/** + * This macro returns a member of the cvmx_bootmem_desc + * structure. These members can't be directly addressed as + * they might be in memory not directly reachable. In the case + * where bootmem is compiled with LINUX_HOST, the structure + * itself might be located on a remote Octeon. The argument + * "field" is the member name of the cvmx_bootmem_desc to read. + * Regardless of the type of the field, the return type is always + * a u64. + */ +#define CVMX_BOOTMEM_DESC_GET_FIELD(oct, field) \ + __cvmx_bootmem_desc_get(oct, oct->bootmem_desc_addr, \ + offsetof(struct cvmx_bootmem_desc, field), \ + SIZEOF_FIELD(struct cvmx_bootmem_desc, field)) + +#define __cvmx_bootmem_lock(flags) +#define __cvmx_bootmem_unlock(flags) + +/** + * This macro returns a member of the + * cvmx_bootmem_named_block_desc structure. These members can't + * be directly addressed as they might be in memory not directly + * reachable. In the case where bootmem is compiled with + * LINUX_HOST, the structure itself might be located on a remote + * Octeon. The argument "field" is the member name of the + * cvmx_bootmem_named_block_desc to read. Regardless of the type + * of the field, the return type is always a u64. The "addr" + * parameter is the physical address of the structure. + */ +#define CVMX_BOOTMEM_NAMED_GET_FIELD(oct, addr, field) \ + __cvmx_bootmem_desc_get(oct, addr, \ + offsetof(struct cvmx_bootmem_named_block_desc, field), \ + SIZEOF_FIELD(struct cvmx_bootmem_named_block_desc, field)) + +/** + * This function is the implementation of the get macros defined + * for individual structure members. The argument are generated + * by the macros inorder to read only the needed memory. + * + * @param oct Pointer to current octeon device + * @param base 64bit physical address of the complete structure + * @param offset Offset from the beginning of the structure to the member being + * accessed. + * @param size Size of the structure member. + * + * @return Value of the structure member promoted into a u64. + */ +static inline u64 __cvmx_bootmem_desc_get(struct octeon_device *oct, + u64 base, + u32 offset, + u32 size) +{ + base = (1ull << 63) | (base + offset); + switch (size) { + case 4: + return octeon_read_device_mem32(oct, base); + case 8: + return octeon_read_device_mem64(oct, base); + default: + return 0; + } +} + +/** + * This function retrieves the string name of a named block. It is + * more complicated than a simple memcpy() since the named block + * descriptor may not be directly accessible. + * + * @param addr Physical address of the named block descriptor + * @param str String to receive the named block string name + * @param len Length of the string buffer, which must match the length + * stored in the bootmem descriptor. + */ +static void CVMX_BOOTMEM_NAMED_GET_NAME(struct octeon_device *oct, + u64 addr, + char *str, + u32 len) +{ + addr += offsetof(struct cvmx_bootmem_named_block_desc, name); + octeon_pci_read_core_mem(oct, addr, str, len); + str[len] = 0; +} + +/* See header file for descriptions of functions */ + +/** + * Check the version information on the bootmem descriptor + * + * @param exact_match + * Exact major version to check against. A zero means + * check that the version supports named blocks. + * + * @return Zero if the version is correct. Negative if the version is + * incorrect. Failures also cause a message to be displayed. + */ +static int __cvmx_bootmem_check_version(struct octeon_device *oct, + u32 exact_match) +{ + u32 major_version; + u32 minor_version; + + if (!oct->bootmem_desc_addr) + oct->bootmem_desc_addr = + octeon_read_device_mem64(oct, + BOOTLOADER_PCI_READ_DESC_ADDR); + major_version = + (u32)CVMX_BOOTMEM_DESC_GET_FIELD(oct, major_version); + minor_version = + (u32)CVMX_BOOTMEM_DESC_GET_FIELD(oct, minor_version); + dev_dbg(&oct->pci_dev->dev, "%s: major_version=%d\n", __func__, + major_version); + if ((major_version > 3) || + (exact_match && major_version != exact_match)) { + dev_err(&oct->pci_dev->dev, "bootmem ver mismatch %d.%d addr:0x%llx\n", + major_version, minor_version, + CAST_ULL(oct->bootmem_desc_addr)); + return -1; + } else { + return 0; + } +} + +static const struct cvmx_bootmem_named_block_desc +*__cvmx_bootmem_find_named_block_flags(struct octeon_device *oct, + const char *name, u32 flags) +{ + struct cvmx_bootmem_named_block_desc *desc = + &oct->bootmem_named_block_desc; + u64 named_addr = cvmx_bootmem_phy_named_block_find(oct, name, flags); + + if (named_addr) { + desc->base_addr = CVMX_BOOTMEM_NAMED_GET_FIELD(oct, named_addr, + base_addr); + desc->size = + CVMX_BOOTMEM_NAMED_GET_FIELD(oct, named_addr, size); + strncpy(desc->name, name, sizeof(desc->name)); + desc->name[sizeof(desc->name) - 1] = 0; + return &oct->bootmem_named_block_desc; + } else { + return NULL; + } +} + +static u64 cvmx_bootmem_phy_named_block_find(struct octeon_device *oct, + const char *name, + u32 flags) +{ + u64 result = 0; + + __cvmx_bootmem_lock(flags); + if (!__cvmx_bootmem_check_version(oct, 3)) { + u32 i; + u64 named_block_array_addr = + CVMX_BOOTMEM_DESC_GET_FIELD(oct, + named_block_array_addr); + u32 num_blocks = (u32) + CVMX_BOOTMEM_DESC_GET_FIELD(oct, nb_num_blocks); + u32 name_length = (u32) + CVMX_BOOTMEM_DESC_GET_FIELD(oct, named_block_name_len); + u64 named_addr = named_block_array_addr; + + for (i = 0; i < num_blocks; i++) { + u64 named_size = + CVMX_BOOTMEM_NAMED_GET_FIELD(oct, named_addr, + size); + if (name && named_size) { + char *name_tmp = + kmalloc(name_length + 1, GFP_KERNEL); + CVMX_BOOTMEM_NAMED_GET_NAME(oct, named_addr, + name_tmp, + name_length); + if (!strncmp(name, name_tmp, name_length)) { + result = named_addr; + kfree(name_tmp); + break; + } + kfree(name_tmp); + } else if (!name && !named_size) { + result = named_addr; + break; + } + + named_addr += + sizeof(struct cvmx_bootmem_named_block_desc); + } + } + __cvmx_bootmem_unlock(flags); + return result; +} + +/** + * Find a named block on the remote Octeon + * + * @param name Name of block to find + * @param base_addr Address the block is at (OUTPUT) + * @param size The size of the block (OUTPUT) + * + * @return Zero on success, One on failure. + */ +static int octeon_named_block_find(struct octeon_device *oct, const char *name, + u64 *base_addr, u64 *size) +{ + const struct cvmx_bootmem_named_block_desc *named_block; + + octeon_remote_lock(); + named_block = __cvmx_bootmem_find_named_block_flags(oct, name, 0); + octeon_remote_unlock(); + if (named_block) { + *base_addr = named_block->base_addr; + *size = named_block->size; + return 0; + } + return 1; +} + +static void octeon_remote_lock(void) +{ + /* fill this in if any sharing is needed */ +} + +static void octeon_remote_unlock(void) +{ + /* fill this in if any sharing is needed */ +} + +int octeon_console_send_cmd(struct octeon_device *oct, char *cmd_str, + u32 wait_hundredths) +{ + u32 len = strlen(cmd_str); + + dev_dbg(&oct->pci_dev->dev, "sending \"%s\" to bootloader\n", cmd_str); + + if (len > BOOTLOADER_PCI_WRITE_BUFFER_STR_LEN - 1) { + dev_err(&oct->pci_dev->dev, "Command string too long, max length is: %d\n", + BOOTLOADER_PCI_WRITE_BUFFER_STR_LEN - 1); + return -1; + } + + if (octeon_wait_for_bootloader(oct, wait_hundredths) != 0) { + dev_err(&oct->pci_dev->dev, "Bootloader not ready for command.\n"); + return -1; + } + + /* Write command to bootloader */ + octeon_remote_lock(); + octeon_pci_write_core_mem(oct, BOOTLOADER_PCI_READ_BUFFER_DATA_ADDR, + (u8 *)cmd_str, len); + octeon_write_device_mem32(oct, BOOTLOADER_PCI_READ_BUFFER_LEN_ADDR, + len); + octeon_write_device_mem32(oct, BOOTLOADER_PCI_READ_BUFFER_OWNER_ADDR, + OCTEON_PCI_IO_BUF_OWNER_OCTEON); + + /* Bootloader should accept command very quickly + * if it really was ready + */ + if (octeon_wait_for_bootloader(oct, 200) != 0) { + octeon_remote_unlock(); + dev_err(&oct->pci_dev->dev, "Bootloader did not accept command.\n"); + return -1; + } + octeon_remote_unlock(); + return 0; +} + +int octeon_wait_for_bootloader(struct octeon_device *oct, + u32 wait_time_hundredths) +{ + dev_dbg(&oct->pci_dev->dev, "waiting %d0 ms for bootloader\n", + wait_time_hundredths); + + if (octeon_mem_access_ok(oct)) + return -1; + + while (wait_time_hundredths > 0 && + octeon_read_device_mem32(oct, + BOOTLOADER_PCI_READ_BUFFER_OWNER_ADDR) + != OCTEON_PCI_IO_BUF_OWNER_HOST) { + if (--wait_time_hundredths <= 0) + return -1; + schedule_timeout_uninterruptible(HZ / 100); + } + return 0; +} + +static void octeon_console_handle_result(struct octeon_device *oct, + size_t console_num, + char *buffer, s32 bytes_read) +{ + struct octeon_console *console; + + console = &oct->console[console_num]; + + console->waiting = 0; +} + +static char console_buffer[OCTEON_CONSOLE_MAX_READ_BYTES]; + +static void output_console_line(struct octeon_device *oct, + struct octeon_console *console, + size_t console_num, + char *console_buffer, + s32 bytes_read) +{ + char *line; + s32 i; + + line = console_buffer; + for (i = 0; i < bytes_read; i++) { + /* Output a line at a time, prefixed */ + if (console_buffer[i] == '\n') { + console_buffer[i] = '\0'; + if (console->leftover[0]) { + dev_info(&oct->pci_dev->dev, "%lu: %s%s\n", + console_num, console->leftover, + line); + console->leftover[0] = '\0'; + } else { + dev_info(&oct->pci_dev->dev, "%lu: %s\n", + console_num, line); + } + line = &console_buffer[i + 1]; + } + } + + /* Save off any leftovers */ + if (line != &console_buffer[bytes_read]) { + console_buffer[bytes_read] = '\0'; + strcpy(console->leftover, line); + } +} + +static void check_console(struct work_struct *work) +{ + s32 bytes_read, tries, total_read; + struct octeon_console *console; + struct cavium_wk *wk = (struct cavium_wk *)work; + struct octeon_device *oct = (struct octeon_device *)wk->ctxptr; + size_t console_num = wk->ctxul; + u32 delay; + + console = &oct->console[console_num]; + tries = 0; + total_read = 0; + + do { + /* Take console output regardless of whether it will + * be logged + */ + bytes_read = + octeon_console_read(oct, console_num, console_buffer, + sizeof(console_buffer) - 1, 0); + if (bytes_read > 0) { + total_read += bytes_read; + if (console->waiting) { + octeon_console_handle_result(oct, console_num, + console_buffer, + bytes_read); + } + if (octeon_console_debug_enabled(console_num)) { + output_console_line(oct, console, console_num, + console_buffer, bytes_read); + } + } else if (bytes_read < 0) { + dev_err(&oct->pci_dev->dev, "Error reading console %lu, ret=%d\n", + console_num, bytes_read); + } + + tries++; + } while ((bytes_read > 0) && (tries < 16)); + + /* If nothing is read after polling the console, + * output any leftovers if any + */ + if (octeon_console_debug_enabled(console_num) && + (total_read == 0) && (console->leftover[0])) { + dev_info(&oct->pci_dev->dev, "%lu: %s\n", + console_num, console->leftover); + console->leftover[0] = '\0'; + } + + delay = OCTEON_CONSOLE_POLL_INTERVAL_MS; + + schedule_delayed_work(&wk->work, msecs_to_jiffies(delay)); +} + +int octeon_init_consoles(struct octeon_device *oct) +{ + int ret = 0; + u64 addr, size; + + ret = octeon_mem_access_ok(oct); + if (ret) { + dev_err(&oct->pci_dev->dev, "Memory access not okay'\n"); + return ret; + } + + ret = octeon_named_block_find(oct, OCTEON_PCI_CONSOLE_BLOCK_NAME, &addr, + &size); + if (ret) { + dev_err(&oct->pci_dev->dev, "Could not find console '%s'\n", + OCTEON_PCI_CONSOLE_BLOCK_NAME); + return ret; + } + + /* num_consoles > 0, is an indication that the consoles + * are accessible + */ + oct->num_consoles = octeon_read_device_mem32(oct, + addr + offsetof(struct octeon_pci_console_desc, + num_consoles)); + oct->console_desc_addr = addr; + + dev_dbg(&oct->pci_dev->dev, "Initialized consoles. %d available\n", + oct->num_consoles); + + return ret; +} + +int octeon_add_console(struct octeon_device *oct, u32 console_num) +{ + int ret = 0; + u32 delay; + u64 coreaddr; + struct delayed_work *work; + struct octeon_console *console; + + if (console_num >= oct->num_consoles) { + dev_err(&oct->pci_dev->dev, + "trying to read from console number %d when only 0 to %d exist\n", + console_num, oct->num_consoles); + } else { + console = &oct->console[console_num]; + + console->waiting = 0; + + coreaddr = oct->console_desc_addr + console_num * 8 + + offsetof(struct octeon_pci_console_desc, + console_addr_array); + console->addr = octeon_read_device_mem64(oct, coreaddr); + coreaddr = console->addr + offsetof(struct octeon_pci_console, + buf_size); + console->buffer_size = octeon_read_device_mem32(oct, coreaddr); + coreaddr = console->addr + offsetof(struct octeon_pci_console, + input_base_addr); + console->input_base_addr = + octeon_read_device_mem64(oct, coreaddr); + coreaddr = console->addr + offsetof(struct octeon_pci_console, + output_base_addr); + console->output_base_addr = + octeon_read_device_mem64(oct, coreaddr); + console->leftover[0] = '\0'; + + work = &oct->console_poll_work[console_num].work; + + INIT_DELAYED_WORK(work, check_console); + oct->console_poll_work[console_num].ctxptr = (void *)oct; + oct->console_poll_work[console_num].ctxul = console_num; + delay = OCTEON_CONSOLE_POLL_INTERVAL_MS; + schedule_delayed_work(work, msecs_to_jiffies(delay)); + + if (octeon_console_debug_enabled(console_num)) { + ret = octeon_console_send_cmd(oct, + "setenv pci_console_active 1", + 2000); + } + + console->active = 1; + } + + return ret; +} + +/** + * Removes all consoles + * + * @param oct octeon device + */ +void octeon_remove_consoles(struct octeon_device *oct) +{ + u32 i; + struct octeon_console *console; + + for (i = 0; i < oct->num_consoles; i++) { + console = &oct->console[i]; + + if (!console->active) + continue; + + cancel_delayed_work_sync(&oct->console_poll_work[i]. + work); + console->addr = 0; + console->buffer_size = 0; + console->input_base_addr = 0; + console->output_base_addr = 0; + } + + oct->num_consoles = 0; +} + +static inline int octeon_console_free_bytes(u32 buffer_size, + u32 wr_idx, + u32 rd_idx) +{ + if (rd_idx >= buffer_size || wr_idx >= buffer_size) + return -1; + + return ((buffer_size - 1) - (wr_idx - rd_idx)) % buffer_size; +} + +static inline int octeon_console_avail_bytes(u32 buffer_size, + u32 wr_idx, + u32 rd_idx) +{ + if (rd_idx >= buffer_size || wr_idx >= buffer_size) + return -1; + + return buffer_size - 1 - + octeon_console_free_bytes(buffer_size, wr_idx, rd_idx); +} + +int octeon_console_read(struct octeon_device *oct, u32 console_num, + char *buffer, u32 buf_size, u32 flags) +{ + int bytes_to_read; + u32 rd_idx, wr_idx; + struct octeon_console *console; + + if (console_num >= oct->num_consoles) { + dev_err(&oct->pci_dev->dev, "Attempted to read from disabled console %d\n", + console_num); + return 0; + } + + console = &oct->console[console_num]; + + /* Check to see if any data is available. + * Maybe optimize this with 64-bit read. + */ + rd_idx = octeon_read_device_mem32(oct, console->addr + + offsetof(struct octeon_pci_console, output_read_index)); + wr_idx = octeon_read_device_mem32(oct, console->addr + + offsetof(struct octeon_pci_console, output_write_index)); + + bytes_to_read = octeon_console_avail_bytes(console->buffer_size, + wr_idx, rd_idx); + if (bytes_to_read <= 0) + return bytes_to_read; + + bytes_to_read = MIN(bytes_to_read, (s32)buf_size); + + /* Check to see if what we want to read is not contiguous, and limit + * ourselves to the contiguous block + */ + if (rd_idx + bytes_to_read >= console->buffer_size) + bytes_to_read = console->buffer_size - rd_idx; + + octeon_pci_read_core_mem(oct, console->output_base_addr + rd_idx, + buffer, bytes_to_read); + octeon_write_device_mem32(oct, console->addr + + offsetof(struct octeon_pci_console, + output_read_index), + (rd_idx + bytes_to_read) % + console->buffer_size); + + return bytes_to_read; +} diff --git a/kernel/drivers/net/ethernet/cavium/liquidio/octeon_device.c b/kernel/drivers/net/ethernet/cavium/liquidio/octeon_device.c new file mode 100644 index 000000000..f67641a2f --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/liquidio/octeon_device.c @@ -0,0 +1,1304 @@ +/********************************************************************** +* Author: Cavium, Inc. +* +* Contact: support@cavium.com +* Please include "LiquidIO" in the subject. +* +* Copyright (c) 2003-2015 Cavium, Inc. +* +* This file is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License, Version 2, as +* published by the Free Software Foundation. +* +* This file is distributed in the hope that it will be useful, but +* AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty +* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or +* NONINFRINGEMENT. See the GNU General Public License for more +* details. +* +* This file may also be available under a different license from Cavium. +* Contact Cavium, Inc. for more information +**********************************************************************/ +#include <linux/version.h> +#include <linux/types.h> +#include <linux/list.h> +#include <linux/interrupt.h> +#include <linux/pci.h> +#include <linux/crc32.h> +#include <linux/kthread.h> +#include <linux/netdevice.h> +#include <linux/vmalloc.h> +#include "octeon_config.h" +#include "liquidio_common.h" +#include "octeon_droq.h" +#include "octeon_iq.h" +#include "response_manager.h" +#include "octeon_device.h" +#include "octeon_nic.h" +#include "octeon_main.h" +#include "octeon_network.h" +#include "cn66xx_regs.h" +#include "cn66xx_device.h" +#include "cn68xx_regs.h" +#include "cn68xx_device.h" +#include "liquidio_image.h" +#include "octeon_mem_ops.h" + +/** Default configuration + * for CN66XX OCTEON Models. + */ +static struct octeon_config default_cn66xx_conf = { + .card_type = LIO_210SV, + .card_name = LIO_210SV_NAME, + + /** IQ attributes */ + .iq = { + .max_iqs = CN6XXX_CFG_IO_QUEUES, + .pending_list_size = + (CN6XXX_MAX_IQ_DESCRIPTORS * CN6XXX_CFG_IO_QUEUES), + .instr_type = OCTEON_64BYTE_INSTR, + .db_min = CN6XXX_DB_MIN, + .db_timeout = CN6XXX_DB_TIMEOUT, + } + , + + /** OQ attributes */ + .oq = { + .max_oqs = CN6XXX_CFG_IO_QUEUES, + .info_ptr = OCTEON_OQ_INFOPTR_MODE, + .refill_threshold = CN6XXX_OQ_REFIL_THRESHOLD, + .oq_intr_pkt = CN6XXX_OQ_INTR_PKT, + .oq_intr_time = CN6XXX_OQ_INTR_TIME, + .pkts_per_intr = CN6XXX_OQ_PKTSPER_INTR, + } + , + + .num_nic_ports = DEFAULT_NUM_NIC_PORTS_66XX, + .num_def_rx_descs = CN6XXX_MAX_OQ_DESCRIPTORS, + .num_def_tx_descs = CN6XXX_MAX_IQ_DESCRIPTORS, + .def_rx_buf_size = CN6XXX_OQ_BUF_SIZE, + + /* For ethernet interface 0: Port cfg Attributes */ + .nic_if_cfg[0] = { + /* Max Txqs: Half for each of the two ports :max_iq/2 */ + .max_txqs = MAX_TXQS_PER_INTF, + + /* Actual configured value. Range could be: 1...max_txqs */ + .num_txqs = DEF_TXQS_PER_INTF, + + /* Max Rxqs: Half for each of the two ports :max_oq/2 */ + .max_rxqs = MAX_RXQS_PER_INTF, + + /* Actual configured value. Range could be: 1...max_rxqs */ + .num_rxqs = DEF_RXQS_PER_INTF, + + /* Num of desc for rx rings */ + .num_rx_descs = CN6XXX_MAX_OQ_DESCRIPTORS, + + /* Num of desc for tx rings */ + .num_tx_descs = CN6XXX_MAX_IQ_DESCRIPTORS, + + /* SKB size, We need not change buf size even for Jumbo frames. + * Octeon can send jumbo frames in 4 consecutive descriptors, + */ + .rx_buf_size = CN6XXX_OQ_BUF_SIZE, + + .base_queue = BASE_QUEUE_NOT_REQUESTED, + + .gmx_port_id = 0, + }, + + .nic_if_cfg[1] = { + /* Max Txqs: Half for each of the two ports :max_iq/2 */ + .max_txqs = MAX_TXQS_PER_INTF, + + /* Actual configured value. Range could be: 1...max_txqs */ + .num_txqs = DEF_TXQS_PER_INTF, + + /* Max Rxqs: Half for each of the two ports :max_oq/2 */ + .max_rxqs = MAX_RXQS_PER_INTF, + + /* Actual configured value. Range could be: 1...max_rxqs */ + .num_rxqs = DEF_RXQS_PER_INTF, + + /* Num of desc for rx rings */ + .num_rx_descs = CN6XXX_MAX_OQ_DESCRIPTORS, + + /* Num of desc for tx rings */ + .num_tx_descs = CN6XXX_MAX_IQ_DESCRIPTORS, + + /* SKB size, We need not change buf size even for Jumbo frames. + * Octeon can send jumbo frames in 4 consecutive descriptors, + */ + .rx_buf_size = CN6XXX_OQ_BUF_SIZE, + + .base_queue = BASE_QUEUE_NOT_REQUESTED, + + .gmx_port_id = 1, + }, + + /** Miscellaneous attributes */ + .misc = { + /* Host driver link query interval */ + .oct_link_query_interval = 100, + + /* Octeon link query interval */ + .host_link_query_interval = 500, + + .enable_sli_oq_bp = 0, + + /* Control queue group */ + .ctrlq_grp = 1, + } + , +}; + +/** Default configuration + * for CN68XX OCTEON Model. + */ + +static struct octeon_config default_cn68xx_conf = { + .card_type = LIO_410NV, + .card_name = LIO_410NV_NAME, + + /** IQ attributes */ + .iq = { + .max_iqs = CN6XXX_CFG_IO_QUEUES, + .pending_list_size = + (CN6XXX_MAX_IQ_DESCRIPTORS * CN6XXX_CFG_IO_QUEUES), + .instr_type = OCTEON_64BYTE_INSTR, + .db_min = CN6XXX_DB_MIN, + .db_timeout = CN6XXX_DB_TIMEOUT, + } + , + + /** OQ attributes */ + .oq = { + .max_oqs = CN6XXX_CFG_IO_QUEUES, + .info_ptr = OCTEON_OQ_INFOPTR_MODE, + .refill_threshold = CN6XXX_OQ_REFIL_THRESHOLD, + .oq_intr_pkt = CN6XXX_OQ_INTR_PKT, + .oq_intr_time = CN6XXX_OQ_INTR_TIME, + .pkts_per_intr = CN6XXX_OQ_PKTSPER_INTR, + } + , + + .num_nic_ports = DEFAULT_NUM_NIC_PORTS_68XX, + .num_def_rx_descs = CN6XXX_MAX_OQ_DESCRIPTORS, + .num_def_tx_descs = CN6XXX_MAX_IQ_DESCRIPTORS, + .def_rx_buf_size = CN6XXX_OQ_BUF_SIZE, + + .nic_if_cfg[0] = { + /* Max Txqs: Half for each of the two ports :max_iq/2 */ + .max_txqs = MAX_TXQS_PER_INTF, + + /* Actual configured value. Range could be: 1...max_txqs */ + .num_txqs = DEF_TXQS_PER_INTF, + + /* Max Rxqs: Half for each of the two ports :max_oq/2 */ + .max_rxqs = MAX_RXQS_PER_INTF, + + /* Actual configured value. Range could be: 1...max_rxqs */ + .num_rxqs = DEF_RXQS_PER_INTF, + + /* Num of desc for rx rings */ + .num_rx_descs = CN6XXX_MAX_OQ_DESCRIPTORS, + + /* Num of desc for tx rings */ + .num_tx_descs = CN6XXX_MAX_IQ_DESCRIPTORS, + + /* SKB size, We need not change buf size even for Jumbo frames. + * Octeon can send jumbo frames in 4 consecutive descriptors, + */ + .rx_buf_size = CN6XXX_OQ_BUF_SIZE, + + .base_queue = BASE_QUEUE_NOT_REQUESTED, + + .gmx_port_id = 0, + }, + + .nic_if_cfg[1] = { + /* Max Txqs: Half for each of the two ports :max_iq/2 */ + .max_txqs = MAX_TXQS_PER_INTF, + + /* Actual configured value. Range could be: 1...max_txqs */ + .num_txqs = DEF_TXQS_PER_INTF, + + /* Max Rxqs: Half for each of the two ports :max_oq/2 */ + .max_rxqs = MAX_RXQS_PER_INTF, + + /* Actual configured value. Range could be: 1...max_rxqs */ + .num_rxqs = DEF_RXQS_PER_INTF, + + /* Num of desc for rx rings */ + .num_rx_descs = CN6XXX_MAX_OQ_DESCRIPTORS, + + /* Num of desc for tx rings */ + .num_tx_descs = CN6XXX_MAX_IQ_DESCRIPTORS, + + /* SKB size, We need not change buf size even for Jumbo frames. + * Octeon can send jumbo frames in 4 consecutive descriptors, + */ + .rx_buf_size = CN6XXX_OQ_BUF_SIZE, + + .base_queue = BASE_QUEUE_NOT_REQUESTED, + + .gmx_port_id = 1, + }, + + .nic_if_cfg[2] = { + /* Max Txqs: Half for each of the two ports :max_iq/2 */ + .max_txqs = MAX_TXQS_PER_INTF, + + /* Actual configured value. Range could be: 1...max_txqs */ + .num_txqs = DEF_TXQS_PER_INTF, + + /* Max Rxqs: Half for each of the two ports :max_oq/2 */ + .max_rxqs = MAX_RXQS_PER_INTF, + + /* Actual configured value. Range could be: 1...max_rxqs */ + .num_rxqs = DEF_RXQS_PER_INTF, + + /* Num of desc for rx rings */ + .num_rx_descs = CN6XXX_MAX_OQ_DESCRIPTORS, + + /* Num of desc for tx rings */ + .num_tx_descs = CN6XXX_MAX_IQ_DESCRIPTORS, + + /* SKB size, We need not change buf size even for Jumbo frames. + * Octeon can send jumbo frames in 4 consecutive descriptors, + */ + .rx_buf_size = CN6XXX_OQ_BUF_SIZE, + + .base_queue = BASE_QUEUE_NOT_REQUESTED, + + .gmx_port_id = 2, + }, + + .nic_if_cfg[3] = { + /* Max Txqs: Half for each of the two ports :max_iq/2 */ + .max_txqs = MAX_TXQS_PER_INTF, + + /* Actual configured value. Range could be: 1...max_txqs */ + .num_txqs = DEF_TXQS_PER_INTF, + + /* Max Rxqs: Half for each of the two ports :max_oq/2 */ + .max_rxqs = MAX_RXQS_PER_INTF, + + /* Actual configured value. Range could be: 1...max_rxqs */ + .num_rxqs = DEF_RXQS_PER_INTF, + + /* Num of desc for rx rings */ + .num_rx_descs = CN6XXX_MAX_OQ_DESCRIPTORS, + + /* Num of desc for tx rings */ + .num_tx_descs = CN6XXX_MAX_IQ_DESCRIPTORS, + + /* SKB size, We need not change buf size even for Jumbo frames. + * Octeon can send jumbo frames in 4 consecutive descriptors, + */ + .rx_buf_size = CN6XXX_OQ_BUF_SIZE, + + .base_queue = BASE_QUEUE_NOT_REQUESTED, + + .gmx_port_id = 3, + }, + + /** Miscellaneous attributes */ + .misc = { + /* Host driver link query interval */ + .oct_link_query_interval = 100, + + /* Octeon link query interval */ + .host_link_query_interval = 500, + + .enable_sli_oq_bp = 0, + + /* Control queue group */ + .ctrlq_grp = 1, + } + , +}; + +/** Default configuration + * for CN68XX OCTEON Model. + */ +static struct octeon_config default_cn68xx_210nv_conf = { + .card_type = LIO_210NV, + .card_name = LIO_210NV_NAME, + + /** IQ attributes */ + + .iq = { + .max_iqs = CN6XXX_CFG_IO_QUEUES, + .pending_list_size = + (CN6XXX_MAX_IQ_DESCRIPTORS * CN6XXX_CFG_IO_QUEUES), + .instr_type = OCTEON_64BYTE_INSTR, + .db_min = CN6XXX_DB_MIN, + .db_timeout = CN6XXX_DB_TIMEOUT, + } + , + + /** OQ attributes */ + .oq = { + .max_oqs = CN6XXX_CFG_IO_QUEUES, + .info_ptr = OCTEON_OQ_INFOPTR_MODE, + .refill_threshold = CN6XXX_OQ_REFIL_THRESHOLD, + .oq_intr_pkt = CN6XXX_OQ_INTR_PKT, + .oq_intr_time = CN6XXX_OQ_INTR_TIME, + .pkts_per_intr = CN6XXX_OQ_PKTSPER_INTR, + } + , + + .num_nic_ports = DEFAULT_NUM_NIC_PORTS_68XX_210NV, + .num_def_rx_descs = CN6XXX_MAX_OQ_DESCRIPTORS, + .num_def_tx_descs = CN6XXX_MAX_IQ_DESCRIPTORS, + .def_rx_buf_size = CN6XXX_OQ_BUF_SIZE, + + .nic_if_cfg[0] = { + /* Max Txqs: Half for each of the two ports :max_iq/2 */ + .max_txqs = MAX_TXQS_PER_INTF, + + /* Actual configured value. Range could be: 1...max_txqs */ + .num_txqs = DEF_TXQS_PER_INTF, + + /* Max Rxqs: Half for each of the two ports :max_oq/2 */ + .max_rxqs = MAX_RXQS_PER_INTF, + + /* Actual configured value. Range could be: 1...max_rxqs */ + .num_rxqs = DEF_RXQS_PER_INTF, + + /* Num of desc for rx rings */ + .num_rx_descs = CN6XXX_MAX_OQ_DESCRIPTORS, + + /* Num of desc for tx rings */ + .num_tx_descs = CN6XXX_MAX_IQ_DESCRIPTORS, + + /* SKB size, We need not change buf size even for Jumbo frames. + * Octeon can send jumbo frames in 4 consecutive descriptors, + */ + .rx_buf_size = CN6XXX_OQ_BUF_SIZE, + + .base_queue = BASE_QUEUE_NOT_REQUESTED, + + .gmx_port_id = 0, + }, + + .nic_if_cfg[1] = { + /* Max Txqs: Half for each of the two ports :max_iq/2 */ + .max_txqs = MAX_TXQS_PER_INTF, + + /* Actual configured value. Range could be: 1...max_txqs */ + .num_txqs = DEF_TXQS_PER_INTF, + + /* Max Rxqs: Half for each of the two ports :max_oq/2 */ + .max_rxqs = MAX_RXQS_PER_INTF, + + /* Actual configured value. Range could be: 1...max_rxqs */ + .num_rxqs = DEF_RXQS_PER_INTF, + + /* Num of desc for rx rings */ + .num_rx_descs = CN6XXX_MAX_OQ_DESCRIPTORS, + + /* Num of desc for tx rings */ + .num_tx_descs = CN6XXX_MAX_IQ_DESCRIPTORS, + + /* SKB size, We need not change buf size even for Jumbo frames. + * Octeon can send jumbo frames in 4 consecutive descriptors, + */ + .rx_buf_size = CN6XXX_OQ_BUF_SIZE, + + .base_queue = BASE_QUEUE_NOT_REQUESTED, + + .gmx_port_id = 1, + }, + + /** Miscellaneous attributes */ + .misc = { + /* Host driver link query interval */ + .oct_link_query_interval = 100, + + /* Octeon link query interval */ + .host_link_query_interval = 500, + + .enable_sli_oq_bp = 0, + + /* Control queue group */ + .ctrlq_grp = 1, + } + , +}; + +enum { + OCTEON_CONFIG_TYPE_DEFAULT = 0, + NUM_OCTEON_CONFS, +}; + +static struct octeon_config_ptr { + u32 conf_type; +} oct_conf_info[MAX_OCTEON_DEVICES] = { + { + OCTEON_CONFIG_TYPE_DEFAULT, + }, { + OCTEON_CONFIG_TYPE_DEFAULT, + }, { + OCTEON_CONFIG_TYPE_DEFAULT, + }, { + OCTEON_CONFIG_TYPE_DEFAULT, + }, +}; + +static char oct_dev_state_str[OCT_DEV_STATES + 1][32] = { + "BEGIN", "PCI-MAP-DONE", "DISPATCH-INIT-DONE", + "IQ-INIT-DONE", "SCBUFF-POOL-INIT-DONE", "RESPLIST-INIT-DONE", + "DROQ-INIT-DONE", "IO-QUEUES-INIT-DONE", "CONSOLE-INIT-DONE", + "HOST-READY", "CORE-READY", "RUNNING", "IN-RESET", + "INVALID" +}; + +static char oct_dev_app_str[CVM_DRV_APP_COUNT + 1][32] = { + "BASE", "NIC", "UNKNOWN"}; + +static struct octeon_device *octeon_device[MAX_OCTEON_DEVICES]; +static u32 octeon_device_count; + +static struct octeon_core_setup core_setup[MAX_OCTEON_DEVICES]; + +static void oct_set_config_info(int oct_id, int conf_type) +{ + if (conf_type < 0 || conf_type > (NUM_OCTEON_CONFS - 1)) + conf_type = OCTEON_CONFIG_TYPE_DEFAULT; + oct_conf_info[oct_id].conf_type = conf_type; +} + +void octeon_init_device_list(int conf_type) +{ + int i; + + memset(octeon_device, 0, (sizeof(void *) * MAX_OCTEON_DEVICES)); + for (i = 0; i < MAX_OCTEON_DEVICES; i++) + oct_set_config_info(i, conf_type); +} + +static void *__retrieve_octeon_config_info(struct octeon_device *oct, + u16 card_type) +{ + u32 oct_id = oct->octeon_id; + void *ret = NULL; + + switch (oct_conf_info[oct_id].conf_type) { + case OCTEON_CONFIG_TYPE_DEFAULT: + if (oct->chip_id == OCTEON_CN66XX) { + ret = (void *)&default_cn66xx_conf; + } else if ((oct->chip_id == OCTEON_CN68XX) && + (card_type == LIO_210NV)) { + ret = (void *)&default_cn68xx_210nv_conf; + } else if ((oct->chip_id == OCTEON_CN68XX) && + (card_type == LIO_410NV)) { + ret = (void *)&default_cn68xx_conf; + } + break; + default: + break; + } + return ret; +} + +static int __verify_octeon_config_info(struct octeon_device *oct, void *conf) +{ + switch (oct->chip_id) { + case OCTEON_CN66XX: + case OCTEON_CN68XX: + return lio_validate_cn6xxx_config_info(oct, conf); + + default: + break; + } + + return 1; +} + +void *oct_get_config_info(struct octeon_device *oct, u16 card_type) +{ + void *conf = NULL; + + conf = __retrieve_octeon_config_info(oct, card_type); + if (!conf) + return NULL; + + if (__verify_octeon_config_info(oct, conf)) { + dev_err(&oct->pci_dev->dev, "Configuration verification failed\n"); + return NULL; + } + + return conf; +} + +char *lio_get_state_string(atomic_t *state_ptr) +{ + s32 istate = (s32)atomic_read(state_ptr); + + if (istate > OCT_DEV_STATES || istate < 0) + return oct_dev_state_str[OCT_DEV_STATE_INVALID]; + return oct_dev_state_str[istate]; +} + +static char *get_oct_app_string(u32 app_mode) +{ + if (app_mode <= CVM_DRV_APP_END) + return oct_dev_app_str[app_mode - CVM_DRV_APP_START]; + return oct_dev_app_str[CVM_DRV_INVALID_APP - CVM_DRV_APP_START]; +} + +int octeon_download_firmware(struct octeon_device *oct, const u8 *data, + size_t size) +{ + int ret = 0; + u8 *p; + u8 *buffer; + u32 crc32_result; + u64 load_addr; + u32 image_len; + struct octeon_firmware_file_header *h; + u32 i; + + if (size < sizeof(struct octeon_firmware_file_header)) { + dev_err(&oct->pci_dev->dev, "Firmware file too small (%d < %d).\n", + (u32)size, + (u32)sizeof(struct octeon_firmware_file_header)); + return -EINVAL; + } + + h = (struct octeon_firmware_file_header *)data; + + if (be32_to_cpu(h->magic) != LIO_NIC_MAGIC) { + dev_err(&oct->pci_dev->dev, "Unrecognized firmware file.\n"); + return -EINVAL; + } + + crc32_result = + crc32(~0, data, + sizeof(struct octeon_firmware_file_header) - + sizeof(u32)) ^ ~0U; + if (crc32_result != be32_to_cpu(h->crc32)) { + dev_err(&oct->pci_dev->dev, "Firmware CRC mismatch (0x%08x != 0x%08x).\n", + crc32_result, be32_to_cpu(h->crc32)); + return -EINVAL; + } + + if (memcmp(LIQUIDIO_VERSION, h->version, strlen(LIQUIDIO_VERSION))) { + dev_err(&oct->pci_dev->dev, "Unmatched firmware version. Expected %s, got %s.\n", + LIQUIDIO_VERSION, h->version); + return -EINVAL; + } + + if (be32_to_cpu(h->num_images) > LIO_MAX_IMAGES) { + dev_err(&oct->pci_dev->dev, "Too many images in firmware file (%d).\n", + be32_to_cpu(h->num_images)); + return -EINVAL; + } + + dev_info(&oct->pci_dev->dev, "Firmware version: %s\n", h->version); + snprintf(oct->fw_info.liquidio_firmware_version, 32, "LIQUIDIO: %s", + h->version); + + buffer = kmalloc(size, GFP_KERNEL); + if (!buffer) + return -ENOMEM; + + memcpy(buffer, data, size); + + p = buffer + sizeof(struct octeon_firmware_file_header); + + /* load all images */ + for (i = 0; i < be32_to_cpu(h->num_images); i++) { + load_addr = be64_to_cpu(h->desc[i].addr); + image_len = be32_to_cpu(h->desc[i].len); + + /* validate the image */ + crc32_result = crc32(~0, p, image_len) ^ ~0U; + if (crc32_result != be32_to_cpu(h->desc[i].crc32)) { + dev_err(&oct->pci_dev->dev, + "Firmware CRC mismatch in image %d (0x%08x != 0x%08x).\n", + i, crc32_result, + be32_to_cpu(h->desc[i].crc32)); + ret = -EINVAL; + goto done_downloading; + } + + /* download the image */ + octeon_pci_write_core_mem(oct, load_addr, p, image_len); + + p += image_len; + dev_dbg(&oct->pci_dev->dev, + "Downloaded image %d (%d bytes) to address 0x%016llx\n", + i, image_len, load_addr); + } + + /* Invoke the bootcmd */ + ret = octeon_console_send_cmd(oct, h->bootcmd, 50); + +done_downloading: + kfree(buffer); + + return ret; +} + +void octeon_free_device_mem(struct octeon_device *oct) +{ + u32 i; + + for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES; i++) { + /* could check mask as well */ + vfree(oct->droq[i]); + } + + for (i = 0; i < MAX_OCTEON_INSTR_QUEUES; i++) { + /* could check mask as well */ + vfree(oct->instr_queue[i]); + } + + i = oct->octeon_id; + vfree(oct); + + octeon_device[i] = NULL; + octeon_device_count--; +} + +static struct octeon_device *octeon_allocate_device_mem(u32 pci_id, + u32 priv_size) +{ + struct octeon_device *oct; + u8 *buf = NULL; + u32 octdevsize = 0, configsize = 0, size; + + switch (pci_id) { + case OCTEON_CN68XX: + case OCTEON_CN66XX: + configsize = sizeof(struct octeon_cn6xxx); + break; + + default: + pr_err("%s: Unknown PCI Device: 0x%x\n", + __func__, + pci_id); + return NULL; + } + + if (configsize & 0x7) + configsize += (8 - (configsize & 0x7)); + + octdevsize = sizeof(struct octeon_device); + if (octdevsize & 0x7) + octdevsize += (8 - (octdevsize & 0x7)); + + if (priv_size & 0x7) + priv_size += (8 - (priv_size & 0x7)); + + size = octdevsize + priv_size + configsize + + (sizeof(struct octeon_dispatch) * DISPATCH_LIST_SIZE); + + buf = vmalloc(size); + if (!buf) + return NULL; + + memset(buf, 0, size); + + oct = (struct octeon_device *)buf; + oct->priv = (void *)(buf + octdevsize); + oct->chip = (void *)(buf + octdevsize + priv_size); + oct->dispatch.dlist = (struct octeon_dispatch *) + (buf + octdevsize + priv_size + configsize); + + return oct; +} + +struct octeon_device *octeon_allocate_device(u32 pci_id, + u32 priv_size) +{ + u32 oct_idx = 0; + struct octeon_device *oct = NULL; + + for (oct_idx = 0; oct_idx < MAX_OCTEON_DEVICES; oct_idx++) + if (!octeon_device[oct_idx]) + break; + + if (oct_idx == MAX_OCTEON_DEVICES) + return NULL; + + oct = octeon_allocate_device_mem(pci_id, priv_size); + if (!oct) + return NULL; + + spin_lock_init(&oct->pci_win_lock); + spin_lock_init(&oct->mem_access_lock); + + octeon_device_count++; + octeon_device[oct_idx] = oct; + + oct->octeon_id = oct_idx; + snprintf((oct->device_name), sizeof(oct->device_name), + "LiquidIO%d", (oct->octeon_id)); + + return oct; +} + +int octeon_setup_instr_queues(struct octeon_device *oct) +{ + u32 i, num_iqs = 0; + u32 num_descs = 0; + + /* this causes queue 0 to be default queue */ + if (OCTEON_CN6XXX(oct)) { + num_iqs = 1; + num_descs = + CFG_GET_NUM_DEF_TX_DESCS(CHIP_FIELD(oct, cn6xxx, conf)); + } + + oct->num_iqs = 0; + + for (i = 0; i < num_iqs; i++) { + oct->instr_queue[i] = + vmalloc(sizeof(struct octeon_instr_queue)); + if (!oct->instr_queue[i]) + return 1; + + memset(oct->instr_queue[i], 0, + sizeof(struct octeon_instr_queue)); + + oct->instr_queue[i]->app_ctx = (void *)(size_t)i; + if (octeon_init_instr_queue(oct, i, num_descs)) + return 1; + + oct->num_iqs++; + } + + return 0; +} + +int octeon_setup_output_queues(struct octeon_device *oct) +{ + u32 i, num_oqs = 0; + u32 num_descs = 0; + u32 desc_size = 0; + + /* this causes queue 0 to be default queue */ + if (OCTEON_CN6XXX(oct)) { + /* CFG_GET_OQ_MAX_BASE_Q(CHIP_FIELD(oct, cn6xxx, conf)); */ + num_oqs = 1; + num_descs = + CFG_GET_NUM_DEF_RX_DESCS(CHIP_FIELD(oct, cn6xxx, conf)); + desc_size = + CFG_GET_DEF_RX_BUF_SIZE(CHIP_FIELD(oct, cn6xxx, conf)); + } + + oct->num_oqs = 0; + + for (i = 0; i < num_oqs; i++) { + oct->droq[i] = vmalloc(sizeof(*oct->droq[i])); + if (!oct->droq[i]) + return 1; + + memset(oct->droq[i], 0, sizeof(struct octeon_droq)); + + if (octeon_init_droq(oct, i, num_descs, desc_size, NULL)) + return 1; + + oct->num_oqs++; + } + + return 0; +} + +void octeon_set_io_queues_off(struct octeon_device *oct) +{ + /* Disable the i/p and o/p queues for this Octeon. */ + + octeon_write_csr(oct, CN6XXX_SLI_PKT_INSTR_ENB, 0); + octeon_write_csr(oct, CN6XXX_SLI_PKT_OUT_ENB, 0); +} + +void octeon_set_droq_pkt_op(struct octeon_device *oct, + u32 q_no, + u32 enable) +{ + u32 reg_val = 0; + + /* Disable the i/p and o/p queues for this Octeon. */ + reg_val = octeon_read_csr(oct, CN6XXX_SLI_PKT_OUT_ENB); + + if (enable) + reg_val = reg_val | (1 << q_no); + else + reg_val = reg_val & (~(1 << q_no)); + + octeon_write_csr(oct, CN6XXX_SLI_PKT_OUT_ENB, reg_val); +} + +int octeon_init_dispatch_list(struct octeon_device *oct) +{ + u32 i; + + oct->dispatch.count = 0; + + for (i = 0; i < DISPATCH_LIST_SIZE; i++) { + oct->dispatch.dlist[i].opcode = 0; + INIT_LIST_HEAD(&oct->dispatch.dlist[i].list); + } + + for (i = 0; i <= REQTYPE_LAST; i++) + octeon_register_reqtype_free_fn(oct, i, NULL); + + spin_lock_init(&oct->dispatch.lock); + + return 0; +} + +void octeon_delete_dispatch_list(struct octeon_device *oct) +{ + u32 i; + struct list_head freelist, *temp, *tmp2; + + INIT_LIST_HEAD(&freelist); + + spin_lock_bh(&oct->dispatch.lock); + + for (i = 0; i < DISPATCH_LIST_SIZE; i++) { + struct list_head *dispatch; + + dispatch = &oct->dispatch.dlist[i].list; + while (dispatch->next != dispatch) { + temp = dispatch->next; + list_del(temp); + list_add_tail(temp, &freelist); + } + + oct->dispatch.dlist[i].opcode = 0; + } + + oct->dispatch.count = 0; + + spin_unlock_bh(&oct->dispatch.lock); + + list_for_each_safe(temp, tmp2, &freelist) { + list_del(temp); + vfree(temp); + } +} + +octeon_dispatch_fn_t +octeon_get_dispatch(struct octeon_device *octeon_dev, u16 opcode, + u16 subcode) +{ + u32 idx; + struct list_head *dispatch; + octeon_dispatch_fn_t fn = NULL; + u16 combined_opcode = OPCODE_SUBCODE(opcode, subcode); + + idx = combined_opcode & OCTEON_OPCODE_MASK; + + spin_lock_bh(&octeon_dev->dispatch.lock); + + if (octeon_dev->dispatch.count == 0) { + spin_unlock_bh(&octeon_dev->dispatch.lock); + return NULL; + } + + if (!(octeon_dev->dispatch.dlist[idx].opcode)) { + spin_unlock_bh(&octeon_dev->dispatch.lock); + return NULL; + } + + if (octeon_dev->dispatch.dlist[idx].opcode == combined_opcode) { + fn = octeon_dev->dispatch.dlist[idx].dispatch_fn; + } else { + list_for_each(dispatch, + &octeon_dev->dispatch.dlist[idx].list) { + if (((struct octeon_dispatch *)dispatch)->opcode == + combined_opcode) { + fn = ((struct octeon_dispatch *) + dispatch)->dispatch_fn; + break; + } + } + } + + spin_unlock_bh(&octeon_dev->dispatch.lock); + return fn; +} + +/* octeon_register_dispatch_fn + * Parameters: + * octeon_id - id of the octeon device. + * opcode - opcode for which driver should call the registered function + * subcode - subcode for which driver should call the registered function + * fn - The function to call when a packet with "opcode" arrives in + * octeon output queues. + * fn_arg - The argument to be passed when calling function "fn". + * Description: + * Registers a function and its argument to be called when a packet + * arrives in Octeon output queues with "opcode". + * Returns: + * Success: 0 + * Failure: 1 + * Locks: + * No locks are held. + */ +int +octeon_register_dispatch_fn(struct octeon_device *oct, + u16 opcode, + u16 subcode, + octeon_dispatch_fn_t fn, void *fn_arg) +{ + u32 idx; + octeon_dispatch_fn_t pfn; + u16 combined_opcode = OPCODE_SUBCODE(opcode, subcode); + + idx = combined_opcode & OCTEON_OPCODE_MASK; + + spin_lock_bh(&oct->dispatch.lock); + /* Add dispatch function to first level of lookup table */ + if (oct->dispatch.dlist[idx].opcode == 0) { + oct->dispatch.dlist[idx].opcode = combined_opcode; + oct->dispatch.dlist[idx].dispatch_fn = fn; + oct->dispatch.dlist[idx].arg = fn_arg; + oct->dispatch.count++; + spin_unlock_bh(&oct->dispatch.lock); + return 0; + } + + spin_unlock_bh(&oct->dispatch.lock); + + /* Check if there was a function already registered for this + * opcode/subcode. + */ + pfn = octeon_get_dispatch(oct, opcode, subcode); + if (!pfn) { + struct octeon_dispatch *dispatch; + + dev_dbg(&oct->pci_dev->dev, + "Adding opcode to dispatch list linked list\n"); + dispatch = (struct octeon_dispatch *) + vmalloc(sizeof(struct octeon_dispatch)); + if (!dispatch) { + dev_err(&oct->pci_dev->dev, + "No memory to add dispatch function\n"); + return 1; + } + dispatch->opcode = combined_opcode; + dispatch->dispatch_fn = fn; + dispatch->arg = fn_arg; + + /* Add dispatch function to linked list of fn ptrs + * at the hashed index. + */ + spin_lock_bh(&oct->dispatch.lock); + list_add(&dispatch->list, &oct->dispatch.dlist[idx].list); + oct->dispatch.count++; + spin_unlock_bh(&oct->dispatch.lock); + + } else { + dev_err(&oct->pci_dev->dev, + "Found previously registered dispatch fn for opcode/subcode: %x/%x\n", + opcode, subcode); + return 1; + } + + return 0; +} + +/* octeon_unregister_dispatch_fn + * Parameters: + * oct - octeon device + * opcode - driver should unregister the function for this opcode + * subcode - driver should unregister the function for this subcode + * Description: + * Unregister the function set for this opcode+subcode. + * Returns: + * Success: 0 + * Failure: 1 + * Locks: + * No locks are held. + */ +int +octeon_unregister_dispatch_fn(struct octeon_device *oct, u16 opcode, + u16 subcode) +{ + int retval = 0; + u32 idx; + struct list_head *dispatch, *dfree = NULL, *tmp2; + u16 combined_opcode = OPCODE_SUBCODE(opcode, subcode); + + idx = combined_opcode & OCTEON_OPCODE_MASK; + + spin_lock_bh(&oct->dispatch.lock); + + if (oct->dispatch.count == 0) { + spin_unlock_bh(&oct->dispatch.lock); + dev_err(&oct->pci_dev->dev, + "No dispatch functions registered for this device\n"); + return 1; + } + + if (oct->dispatch.dlist[idx].opcode == combined_opcode) { + dispatch = &oct->dispatch.dlist[idx].list; + if (dispatch->next != dispatch) { + dispatch = dispatch->next; + oct->dispatch.dlist[idx].opcode = + ((struct octeon_dispatch *)dispatch)->opcode; + oct->dispatch.dlist[idx].dispatch_fn = + ((struct octeon_dispatch *) + dispatch)->dispatch_fn; + oct->dispatch.dlist[idx].arg = + ((struct octeon_dispatch *)dispatch)->arg; + list_del(dispatch); + dfree = dispatch; + } else { + oct->dispatch.dlist[idx].opcode = 0; + oct->dispatch.dlist[idx].dispatch_fn = NULL; + oct->dispatch.dlist[idx].arg = NULL; + } + } else { + retval = 1; + list_for_each_safe(dispatch, tmp2, + &(oct->dispatch.dlist[idx]. + list)) { + if (((struct octeon_dispatch *)dispatch)->opcode == + combined_opcode) { + list_del(dispatch); + dfree = dispatch; + retval = 0; + } + } + } + + if (!retval) + oct->dispatch.count--; + + spin_unlock_bh(&oct->dispatch.lock); + vfree(dfree); + return retval; +} + +int octeon_core_drv_init(struct octeon_recv_info *recv_info, void *buf) +{ + u32 i; + char app_name[16]; + struct octeon_device *oct = (struct octeon_device *)buf; + struct octeon_recv_pkt *recv_pkt = recv_info->recv_pkt; + struct octeon_core_setup *cs = NULL; + u32 num_nic_ports = 0; + + if (OCTEON_CN6XXX(oct)) + num_nic_ports = + CFG_GET_NUM_NIC_PORTS(CHIP_FIELD(oct, cn6xxx, conf)); + + if (atomic_read(&oct->status) >= OCT_DEV_RUNNING) { + dev_err(&oct->pci_dev->dev, "Received CORE OK when device state is 0x%x\n", + atomic_read(&oct->status)); + goto core_drv_init_err; + } + + strncpy(app_name, + get_oct_app_string( + (u32)recv_pkt->rh.r_core_drv_init.app_mode), + sizeof(app_name) - 1); + oct->app_mode = (u32)recv_pkt->rh.r_core_drv_init.app_mode; + if (recv_pkt->rh.r_core_drv_init.app_mode == CVM_DRV_NIC_APP) { + oct->fw_info.max_nic_ports = + (u32)recv_pkt->rh.r_core_drv_init.max_nic_ports; + oct->fw_info.num_gmx_ports = + (u32)recv_pkt->rh.r_core_drv_init.num_gmx_ports; + } + + if (oct->fw_info.max_nic_ports < num_nic_ports) { + dev_err(&oct->pci_dev->dev, + "Config has more ports than firmware allows (%d > %d).\n", + num_nic_ports, oct->fw_info.max_nic_ports); + goto core_drv_init_err; + } + oct->fw_info.app_cap_flags = recv_pkt->rh.r_core_drv_init.app_cap_flags; + oct->fw_info.app_mode = (u32)recv_pkt->rh.r_core_drv_init.app_mode; + + atomic_set(&oct->status, OCT_DEV_CORE_OK); + + cs = &core_setup[oct->octeon_id]; + + if (recv_pkt->buffer_size[0] != sizeof(*cs)) { + dev_dbg(&oct->pci_dev->dev, "Core setup bytes expected %u found %d\n", + (u32)sizeof(*cs), + recv_pkt->buffer_size[0]); + } + + memcpy(cs, get_rbd(recv_pkt->buffer_ptr[0]), sizeof(*cs)); + strncpy(oct->boardinfo.name, cs->boardname, OCT_BOARD_NAME); + strncpy(oct->boardinfo.serial_number, cs->board_serial_number, + OCT_SERIAL_LEN); + + octeon_swap_8B_data((u64 *)cs, (sizeof(*cs) >> 3)); + + oct->boardinfo.major = cs->board_rev_major; + oct->boardinfo.minor = cs->board_rev_minor; + + dev_info(&oct->pci_dev->dev, + "Running %s (%llu Hz)\n", + app_name, CVM_CAST64(cs->corefreq)); + +core_drv_init_err: + for (i = 0; i < recv_pkt->buffer_count; i++) + recv_buffer_free(recv_pkt->buffer_ptr[i]); + octeon_free_recv_info(recv_info); + return 0; +} + +int octeon_get_tx_qsize(struct octeon_device *oct, u32 q_no) + +{ + if (oct && (q_no < MAX_OCTEON_INSTR_QUEUES) && + (oct->io_qmask.iq & (1UL << q_no))) + return oct->instr_queue[q_no]->max_count; + + return -1; +} + +int octeon_get_rx_qsize(struct octeon_device *oct, u32 q_no) +{ + if (oct && (q_no < MAX_OCTEON_OUTPUT_QUEUES) && + (oct->io_qmask.oq & (1UL << q_no))) + return oct->droq[q_no]->max_count; + return -1; +} + +/* Retruns the host firmware handshake OCTEON specific configuration */ +struct octeon_config *octeon_get_conf(struct octeon_device *oct) +{ + struct octeon_config *default_oct_conf = NULL; + + /* check the OCTEON Device model & return the corresponding octeon + * configuration + */ + + if (OCTEON_CN6XXX(oct)) { + default_oct_conf = + (struct octeon_config *)(CHIP_FIELD(oct, cn6xxx, conf)); + } + + return default_oct_conf; +} + +/* scratch register address is same in all the OCT-II and CN70XX models */ +#define CNXX_SLI_SCRATCH1 0x3C0 + +/** Get the octeon device pointer. + * @param octeon_id - The id for which the octeon device pointer is required. + * @return Success: Octeon device pointer. + * @return Failure: NULL. + */ +struct octeon_device *lio_get_device(u32 octeon_id) +{ + if (octeon_id >= MAX_OCTEON_DEVICES) + return NULL; + else + return octeon_device[octeon_id]; +} + +u64 lio_pci_readq(struct octeon_device *oct, u64 addr) +{ + u64 val64; + unsigned long flags; + u32 val32, addrhi; + + spin_lock_irqsave(&oct->pci_win_lock, flags); + + /* The windowed read happens when the LSB of the addr is written. + * So write MSB first + */ + addrhi = (addr >> 32); + if ((oct->chip_id == OCTEON_CN66XX) || (oct->chip_id == OCTEON_CN68XX)) + addrhi |= 0x00060000; + writel(addrhi, oct->reg_list.pci_win_rd_addr_hi); + + /* Read back to preserve ordering of writes */ + val32 = readl(oct->reg_list.pci_win_rd_addr_hi); + + writel(addr & 0xffffffff, oct->reg_list.pci_win_rd_addr_lo); + val32 = readl(oct->reg_list.pci_win_rd_addr_lo); + + val64 = readq(oct->reg_list.pci_win_rd_data); + + spin_unlock_irqrestore(&oct->pci_win_lock, flags); + + return val64; +} + +void lio_pci_writeq(struct octeon_device *oct, + u64 val, + u64 addr) +{ + u32 val32; + unsigned long flags; + + spin_lock_irqsave(&oct->pci_win_lock, flags); + + writeq(addr, oct->reg_list.pci_win_wr_addr); + + /* The write happens when the LSB is written. So write MSB first. */ + writel(val >> 32, oct->reg_list.pci_win_wr_data_hi); + /* Read the MSB to ensure ordering of writes. */ + val32 = readl(oct->reg_list.pci_win_wr_data_hi); + + writel(val & 0xffffffff, oct->reg_list.pci_win_wr_data_lo); + + spin_unlock_irqrestore(&oct->pci_win_lock, flags); +} + +int octeon_mem_access_ok(struct octeon_device *oct) +{ + u64 access_okay = 0; + + /* Check to make sure a DDR interface is enabled */ + u64 lmc0_reset_ctl = lio_pci_readq(oct, CN6XXX_LMC0_RESET_CTL); + + access_okay = (lmc0_reset_ctl & CN6XXX_LMC0_RESET_CTL_DDR3RST_MASK); + + return access_okay ? 0 : 1; +} + +int octeon_wait_for_ddr_init(struct octeon_device *oct, u32 *timeout) +{ + int ret = 1; + u32 ms; + + if (!timeout) + return ret; + + while (*timeout == 0) + schedule_timeout_uninterruptible(HZ / 10); + + for (ms = 0; (ret != 0) && ((*timeout == 0) || (ms <= *timeout)); + ms += HZ / 10) { + ret = octeon_mem_access_ok(oct); + + /* wait 100 ms */ + if (ret) + schedule_timeout_uninterruptible(HZ / 10); + } + + return ret; +} + +/** Get the octeon id assigned to the octeon device passed as argument. + * This function is exported to other modules. + * @param dev - octeon device pointer passed as a void *. + * @return octeon device id + */ +int lio_get_device_id(void *dev) +{ + struct octeon_device *octeon_dev = (struct octeon_device *)dev; + u32 i; + + for (i = 0; i < MAX_OCTEON_DEVICES; i++) + if (octeon_device[i] == octeon_dev) + return octeon_dev->octeon_id; + return -1; +} diff --git a/kernel/drivers/net/ethernet/cavium/liquidio/octeon_device.h b/kernel/drivers/net/ethernet/cavium/liquidio/octeon_device.h new file mode 100644 index 000000000..36e1f85df --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/liquidio/octeon_device.h @@ -0,0 +1,649 @@ +/********************************************************************** +* Author: Cavium, Inc. +* +* Contact: support@cavium.com +* Please include "LiquidIO" in the subject. +* +* Copyright (c) 2003-2015 Cavium, Inc. +* +* This file is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License, Version 2, as +* published by the Free Software Foundation. +* +* This file is distributed in the hope that it will be useful, but +* AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty +* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or +* NONINFRINGEMENT. See the GNU General Public License for more +* details. +* +* This file may also be available under a different license from Cavium. +* Contact Cavium, Inc. for more information +**********************************************************************/ + +/*! \file octeon_device.h + * \brief Host Driver: This file defines the octeon device structure. + */ + +#ifndef _OCTEON_DEVICE_H_ +#define _OCTEON_DEVICE_H_ + +/** PCI VendorId Device Id */ +#define OCTEON_CN68XX_PCIID 0x91177d +#define OCTEON_CN66XX_PCIID 0x92177d + +/** Driver identifies chips by these Ids, created by clubbing together + * DeviceId+RevisionId; Where Revision Id is not used to distinguish + * between chips, a value of 0 is used for revision id. + */ +#define OCTEON_CN68XX 0x0091 +#define OCTEON_CN66XX 0x0092 + +/** Endian-swap modes supported by Octeon. */ +enum octeon_pci_swap_mode { + OCTEON_PCI_PASSTHROUGH = 0, + OCTEON_PCI_64BIT_SWAP = 1, + OCTEON_PCI_32BIT_BYTE_SWAP = 2, + OCTEON_PCI_32BIT_LW_SWAP = 3 +}; + +/*--------------- PCI BAR1 index registers -------------*/ + +/* BAR1 Mask */ +#define PCI_BAR1_ENABLE_CA 1 +#define PCI_BAR1_ENDIAN_MODE OCTEON_PCI_64BIT_SWAP +#define PCI_BAR1_ENTRY_VALID 1 +#define PCI_BAR1_MASK ((PCI_BAR1_ENABLE_CA << 3) \ + | (PCI_BAR1_ENDIAN_MODE << 1) \ + | PCI_BAR1_ENTRY_VALID) + +/** Octeon Device state. + * Each octeon device goes through each of these states + * as it is initialized. + */ +#define OCT_DEV_BEGIN_STATE 0x0 +#define OCT_DEV_PCI_MAP_DONE 0x1 +#define OCT_DEV_DISPATCH_INIT_DONE 0x2 +#define OCT_DEV_INSTR_QUEUE_INIT_DONE 0x3 +#define OCT_DEV_SC_BUFF_POOL_INIT_DONE 0x4 +#define OCT_DEV_RESP_LIST_INIT_DONE 0x5 +#define OCT_DEV_DROQ_INIT_DONE 0x6 +#define OCT_DEV_IO_QUEUES_DONE 0x7 +#define OCT_DEV_CONSOLE_INIT_DONE 0x8 +#define OCT_DEV_HOST_OK 0x9 +#define OCT_DEV_CORE_OK 0xa +#define OCT_DEV_RUNNING 0xb +#define OCT_DEV_IN_RESET 0xc +#define OCT_DEV_STATE_INVALID 0xd + +#define OCT_DEV_STATES OCT_DEV_STATE_INVALID + +/** Octeon Device interrupts + * These interrupt bits are set in int_status filed of + * octeon_device structure + */ +#define OCT_DEV_INTR_DMA0_FORCE 0x01 +#define OCT_DEV_INTR_DMA1_FORCE 0x02 +#define OCT_DEV_INTR_PKT_DATA 0x04 + +#define LIO_RESET_SECS (3) + +/*---------------------------DISPATCH LIST-------------------------------*/ + +/** The dispatch list entry. + * The driver keeps a record of functions registered for each + * response header opcode in this structure. Since the opcode is + * hashed to index into the driver's list, more than one opcode + * can hash to the same entry, in which case the list field points + * to a linked list with the other entries. + */ +struct octeon_dispatch { + /** List head for this entry */ + struct list_head list; + + /** The opcode for which the dispatch function & arg should be used */ + u16 opcode; + + /** The function to be called for a packet received by the driver */ + octeon_dispatch_fn_t dispatch_fn; + + /* The application specified argument to be passed to the above + * function along with the received packet + */ + void *arg; +}; + +/** The dispatch list structure. */ +struct octeon_dispatch_list { + /** access to dispatch list must be atomic */ + spinlock_t lock; + + /** Count of dispatch functions currently registered */ + u32 count; + + /** The list of dispatch functions */ + struct octeon_dispatch *dlist; +}; + +/*----------------------- THE OCTEON DEVICE ---------------------------*/ + +#define OCT_MEM_REGIONS 3 +/** PCI address space mapping information. + * Each of the 3 address spaces given by BAR0, BAR2 and BAR4 of + * Octeon gets mapped to different physical address spaces in + * the kernel. + */ +struct octeon_mmio { + /** PCI address to which the BAR is mapped. */ + u64 start; + + /** Length of this PCI address space. */ + u32 len; + + /** Length that has been mapped to phys. address space. */ + u32 mapped_len; + + /** The physical address to which the PCI address space is mapped. */ + u8 __iomem *hw_addr; + + /** Flag indicating the mapping was successful. */ + u32 done; +}; + +#define MAX_OCTEON_MAPS 32 + +struct octeon_io_enable { + u32 iq; + u32 oq; + u32 iq64B; +}; + +struct octeon_reg_list { + u32 __iomem *pci_win_wr_addr_hi; + u32 __iomem *pci_win_wr_addr_lo; + u64 __iomem *pci_win_wr_addr; + + u32 __iomem *pci_win_rd_addr_hi; + u32 __iomem *pci_win_rd_addr_lo; + u64 __iomem *pci_win_rd_addr; + + u32 __iomem *pci_win_wr_data_hi; + u32 __iomem *pci_win_wr_data_lo; + u64 __iomem *pci_win_wr_data; + + u32 __iomem *pci_win_rd_data_hi; + u32 __iomem *pci_win_rd_data_lo; + u64 __iomem *pci_win_rd_data; +}; + +#define OCTEON_CONSOLE_MAX_READ_BYTES 512 +struct octeon_console { + u32 active; + u32 waiting; + u64 addr; + u32 buffer_size; + u64 input_base_addr; + u64 output_base_addr; + char leftover[OCTEON_CONSOLE_MAX_READ_BYTES]; +}; + +struct octeon_board_info { + char name[OCT_BOARD_NAME]; + char serial_number[OCT_SERIAL_LEN]; + u64 major; + u64 minor; +}; + +struct octeon_fn_list { + void (*setup_iq_regs)(struct octeon_device *, u32); + void (*setup_oq_regs)(struct octeon_device *, u32); + + irqreturn_t (*process_interrupt_regs)(void *); + int (*soft_reset)(struct octeon_device *); + int (*setup_device_regs)(struct octeon_device *); + void (*reinit_regs)(struct octeon_device *); + void (*bar1_idx_setup)(struct octeon_device *, u64, u32, int); + void (*bar1_idx_write)(struct octeon_device *, u32, u32); + u32 (*bar1_idx_read)(struct octeon_device *, u32); + u32 (*update_iq_read_idx)(struct octeon_device *, + struct octeon_instr_queue *); + + void (*enable_oq_pkt_time_intr)(struct octeon_device *, u32); + void (*disable_oq_pkt_time_intr)(struct octeon_device *, u32); + + void (*enable_interrupt)(void *); + void (*disable_interrupt)(void *); + + void (*enable_io_queues)(struct octeon_device *); + void (*disable_io_queues)(struct octeon_device *); +}; + +/* Must be multiple of 8, changing breaks ABI */ +#define CVMX_BOOTMEM_NAME_LEN 128 + +/* Structure for named memory blocks + * Number of descriptors + * available can be changed without affecting compatiblity, + * but name length changes require a bump in the bootmem + * descriptor version + * Note: This structure must be naturally 64 bit aligned, as a single + * memory image will be used by both 32 and 64 bit programs. + */ +struct cvmx_bootmem_named_block_desc { + /** Base address of named block */ + u64 base_addr; + + /** Size actually allocated for named block */ + u64 size; + + /** name of named block */ + char name[CVMX_BOOTMEM_NAME_LEN]; +}; + +struct oct_fw_info { + u32 max_nic_ports; /** max nic ports for the device */ + u32 num_gmx_ports; /** num gmx ports */ + u64 app_cap_flags; /** firmware cap flags */ + + /** The core application is running in this mode. + * See octeon-drv-opcodes.h for values. + */ + u32 app_mode; + char liquidio_firmware_version[32]; +}; + +/* wrappers around work structs */ +struct cavium_wk { + struct delayed_work work; + void *ctxptr; + size_t ctxul; +}; + +struct cavium_wq { + struct workqueue_struct *wq; + struct cavium_wk wk; +}; + +struct octdev_props { + /* Each interface in the Octeon device has a network + * device pointer (used for OS specific calls). + */ + struct net_device *netdev; +}; + +/** The Octeon device. + * Each Octeon device has this structure to represent all its + * components. + */ +struct octeon_device { + /** Lock for PCI window configuration accesses */ + spinlock_t pci_win_lock; + + /** Lock for memory accesses */ + spinlock_t mem_access_lock; + + /** PCI device pointer */ + struct pci_dev *pci_dev; + + /** Chip specific information. */ + void *chip; + + /** Number of interfaces detected in this octeon device. */ + u32 ifcount; + + struct octdev_props props[MAX_OCTEON_LINKS]; + + /** Octeon Chip type. */ + u16 chip_id; + u16 rev_id; + + /** This device's id - set by the driver. */ + u32 octeon_id; + + /** This device's PCIe port used for traffic. */ + u16 pcie_port; + + u16 flags; +#define LIO_FLAG_MSI_ENABLED (u32)(1 << 1) +#define LIO_FLAG_MSIX_ENABLED (u32)(1 << 2) + + /** The state of this device */ + atomic_t status; + + /** memory mapped io range */ + struct octeon_mmio mmio[OCT_MEM_REGIONS]; + + struct octeon_reg_list reg_list; + + struct octeon_fn_list fn_list; + + struct octeon_board_info boardinfo; + + u32 num_iqs; + + /* The pool containing pre allocated buffers used for soft commands */ + struct octeon_sc_buffer_pool sc_buf_pool; + + /** The input instruction queues */ + struct octeon_instr_queue *instr_queue[MAX_OCTEON_INSTR_QUEUES]; + + /** The doubly-linked list of instruction response */ + struct octeon_response_list response_list[MAX_RESPONSE_LISTS]; + + u32 num_oqs; + + /** The DROQ output queues */ + struct octeon_droq *droq[MAX_OCTEON_OUTPUT_QUEUES]; + + struct octeon_io_enable io_qmask; + + /** List of dispatch functions */ + struct octeon_dispatch_list dispatch; + + /* Interrupt Moderation */ + struct oct_intrmod_cfg intrmod; + + u32 int_status; + + u64 droq_intr; + + /** Physical location of the cvmx_bootmem_desc_t in octeon memory */ + u64 bootmem_desc_addr; + + /** Placeholder memory for named blocks. + * Assumes single-threaded access + */ + struct cvmx_bootmem_named_block_desc bootmem_named_block_desc; + + /** Address of consoles descriptor */ + u64 console_desc_addr; + + /** Number of consoles available. 0 means they are inaccessible */ + u32 num_consoles; + + /* Console caches */ + struct octeon_console console[MAX_OCTEON_MAPS]; + + /* Coprocessor clock rate. */ + u64 coproc_clock_rate; + + /** The core application is running in this mode. See liquidio_common.h + * for values. + */ + u32 app_mode; + + struct oct_fw_info fw_info; + + /** The name given to this device. */ + char device_name[32]; + + /** Application Context */ + void *app_ctx; + + struct cavium_wq dma_comp_wq; + + struct cavium_wq check_db_wq[MAX_OCTEON_INSTR_QUEUES]; + + struct cavium_wk nic_poll_work; + + struct cavium_wk console_poll_work[MAX_OCTEON_MAPS]; + + void *priv; +}; + +#define OCTEON_CN6XXX(oct) ((oct->chip_id == OCTEON_CN66XX) || \ + (oct->chip_id == OCTEON_CN68XX)) +#define CHIP_FIELD(oct, TYPE, field) \ + (((struct octeon_ ## TYPE *)(oct->chip))->field) + +struct oct_intrmod_cmd { + struct octeon_device *oct_dev; + struct octeon_soft_command *sc; + struct oct_intrmod_cfg *cfg; +}; + +/*------------------ Function Prototypes ----------------------*/ + +/** Initialize device list memory */ +void octeon_init_device_list(int conf_type); + +/** Free memory for Input and Output queue structures for a octeon device */ +void octeon_free_device_mem(struct octeon_device *); + +/* Look up a free entry in the octeon_device table and allocate resources + * for the octeon_device structure for an octeon device. Called at init + * time. + */ +struct octeon_device *octeon_allocate_device(u32 pci_id, + u32 priv_size); + +/** Initialize the driver's dispatch list which is a mix of a hash table + * and a linked list. This is done at driver load time. + * @param octeon_dev - pointer to the octeon device structure. + * @return 0 on success, else -ve error value + */ +int octeon_init_dispatch_list(struct octeon_device *octeon_dev); + +/** Delete the driver's dispatch list and all registered entries. + * This is done at driver unload time. + * @param octeon_dev - pointer to the octeon device structure. + */ +void octeon_delete_dispatch_list(struct octeon_device *octeon_dev); + +/** Initialize the core device fields with the info returned by the FW. + * @param recv_info - Receive info structure + * @param buf - Receive buffer + */ +int octeon_core_drv_init(struct octeon_recv_info *recv_info, void *buf); + +/** Gets the dispatch function registered to receive packets with a + * given opcode/subcode. + * @param octeon_dev - the octeon device pointer. + * @param opcode - the opcode for which the dispatch function + * is to checked. + * @param subcode - the subcode for which the dispatch function + * is to checked. + * + * @return Success: octeon_dispatch_fn_t (dispatch function pointer) + * @return Failure: NULL + * + * Looks up the dispatch list to get the dispatch function for a + * given opcode. + */ +octeon_dispatch_fn_t +octeon_get_dispatch(struct octeon_device *octeon_dev, u16 opcode, + u16 subcode); + +/** Get the octeon device pointer. + * @param octeon_id - The id for which the octeon device pointer is required. + * @return Success: Octeon device pointer. + * @return Failure: NULL. + */ +struct octeon_device *lio_get_device(u32 octeon_id); + +/** Get the octeon id assigned to the octeon device passed as argument. + * This function is exported to other modules. + * @param dev - octeon device pointer passed as a void *. + * @return octeon device id + */ +int lio_get_device_id(void *dev); + +static inline u16 OCTEON_MAJOR_REV(struct octeon_device *oct) +{ + u16 rev = (oct->rev_id & 0xC) >> 2; + + return (rev == 0) ? 1 : rev; +} + +static inline u16 OCTEON_MINOR_REV(struct octeon_device *oct) +{ + return oct->rev_id & 0x3; +} + +/** Read windowed register. + * @param oct - pointer to the Octeon device. + * @param addr - Address of the register to read. + * + * This routine is called to read from the indirectly accessed + * Octeon registers that are visible through a PCI BAR0 mapped window + * register. + * @return - 64 bit value read from the register. + */ + +u64 lio_pci_readq(struct octeon_device *oct, u64 addr); + +/** Write windowed register. + * @param oct - pointer to the Octeon device. + * @param val - Value to write + * @param addr - Address of the register to write + * + * This routine is called to write to the indirectly accessed + * Octeon registers that are visible through a PCI BAR0 mapped window + * register. + * @return Nothing. + */ +void lio_pci_writeq(struct octeon_device *oct, u64 val, u64 addr); + +/* Routines for reading and writing CSRs */ +#define octeon_write_csr(oct_dev, reg_off, value) \ + writel(value, oct_dev->mmio[0].hw_addr + reg_off) + +#define octeon_write_csr64(oct_dev, reg_off, val64) \ + writeq(val64, oct_dev->mmio[0].hw_addr + reg_off) + +#define octeon_read_csr(oct_dev, reg_off) \ + readl(oct_dev->mmio[0].hw_addr + reg_off) + +#define octeon_read_csr64(oct_dev, reg_off) \ + readq(oct_dev->mmio[0].hw_addr + reg_off) + +/** + * Checks if memory access is okay + * + * @param oct which octeon to send to + * @return Zero on success, negative on failure. + */ +int octeon_mem_access_ok(struct octeon_device *oct); + +/** + * Waits for DDR initialization. + * + * @param oct which octeon to send to + * @param timeout_in_ms pointer to how long to wait until DDR is initialized + * in ms. + * If contents are 0, it waits until contents are non-zero + * before starting to check. + * @return Zero on success, negative on failure. + */ +int octeon_wait_for_ddr_init(struct octeon_device *oct, + u32 *timeout_in_ms); + +/** + * Wait for u-boot to boot and be waiting for a command. + * + * @param wait_time_hundredths + * Maximum time to wait + * + * @return Zero on success, negative on failure. + */ +int octeon_wait_for_bootloader(struct octeon_device *oct, + u32 wait_time_hundredths); + +/** + * Initialize console access + * + * @param oct which octeon initialize + * @return Zero on success, negative on failure. + */ +int octeon_init_consoles(struct octeon_device *oct); + +/** + * Adds access to a console to the device. + * + * @param oct which octeon to add to + * @param console_num which console + * @return Zero on success, negative on failure. + */ +int octeon_add_console(struct octeon_device *oct, u32 console_num); + +/** write or read from a console */ +int octeon_console_write(struct octeon_device *oct, u32 console_num, + char *buffer, u32 write_request_size, u32 flags); +int octeon_console_write_avail(struct octeon_device *oct, u32 console_num); +int octeon_console_read(struct octeon_device *oct, u32 console_num, + char *buffer, u32 buf_size, u32 flags); +int octeon_console_read_avail(struct octeon_device *oct, u32 console_num); + +/** Removes all attached consoles. */ +void octeon_remove_consoles(struct octeon_device *oct); + +/** + * Send a string to u-boot on console 0 as a command. + * + * @param oct which octeon to send to + * @param cmd_str String to send + * @param wait_hundredths Time to wait for u-boot to accept the command. + * + * @return Zero on success, negative on failure. + */ +int octeon_console_send_cmd(struct octeon_device *oct, char *cmd_str, + u32 wait_hundredths); + +/** Parses, validates, and downloads firmware, then boots associated cores. + * @param oct which octeon to download firmware to + * @param data - The complete firmware file image + * @param size - The size of the data + * + * @return 0 if success. + * -EINVAL if file is incompatible or badly formatted. + * -ENODEV if no handler was found for the application type or an + * invalid octeon id was passed. + */ +int octeon_download_firmware(struct octeon_device *oct, const u8 *data, + size_t size); + +char *lio_get_state_string(atomic_t *state_ptr); + +/** Sets up instruction queues for the device + * @param oct which octeon to setup + * + * @return 0 if success. 1 if fails + */ +int octeon_setup_instr_queues(struct octeon_device *oct); + +/** Sets up output queues for the device + * @param oct which octeon to setup + * + * @return 0 if success. 1 if fails + */ +int octeon_setup_output_queues(struct octeon_device *oct); + +int octeon_get_tx_qsize(struct octeon_device *oct, u32 q_no); + +int octeon_get_rx_qsize(struct octeon_device *oct, u32 q_no); + +/** Turns off the input and output queues for the device + * @param oct which octeon to disable + */ +void octeon_set_io_queues_off(struct octeon_device *oct); + +/** Turns on or off the given output queue for the device + * @param oct which octeon to change + * @param q_no which queue + * @param enable 1 to enable, 0 to disable + */ +void octeon_set_droq_pkt_op(struct octeon_device *oct, u32 q_no, u32 enable); + +/** Retrieve the config for the device + * @param oct which octeon + * @param card_type type of card + * + * @returns pointer to configuration + */ +void *oct_get_config_info(struct octeon_device *oct, u16 card_type); + +/** Gets the octeon device configuration + * @return - pointer to the octeon configuration struture + */ +struct octeon_config *octeon_get_conf(struct octeon_device *oct); + +#endif diff --git a/kernel/drivers/net/ethernet/cavium/liquidio/octeon_droq.c b/kernel/drivers/net/ethernet/cavium/liquidio/octeon_droq.c new file mode 100644 index 000000000..4dba86eaa --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/liquidio/octeon_droq.c @@ -0,0 +1,987 @@ +/********************************************************************** +* Author: Cavium, Inc. +* +* Contact: support@cavium.com +* Please include "LiquidIO" in the subject. +* +* Copyright (c) 2003-2015 Cavium, Inc. +* +* This file is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License, Version 2, as +* published by the Free Software Foundation. +* +* This file is distributed in the hope that it will be useful, but +* AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty +* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or +* NONINFRINGEMENT. See the GNU General Public License for more +* details. +* +* This file may also be available under a different license from Cavium. +* Contact Cavium, Inc. for more information +**********************************************************************/ +#include <linux/version.h> +#include <linux/types.h> +#include <linux/list.h> +#include <linux/pci.h> +#include <linux/kthread.h> +#include <linux/netdevice.h> +#include <linux/vmalloc.h> +#include "octeon_config.h" +#include "liquidio_common.h" +#include "octeon_droq.h" +#include "octeon_iq.h" +#include "response_manager.h" +#include "octeon_device.h" +#include "octeon_nic.h" +#include "octeon_main.h" +#include "octeon_network.h" +#include "cn66xx_regs.h" +#include "cn66xx_device.h" +#include "cn68xx_regs.h" +#include "cn68xx_device.h" +#include "liquidio_image.h" +#include "octeon_mem_ops.h" + +/* #define CAVIUM_ONLY_PERF_MODE */ + +#define CVM_MIN(d1, d2) (((d1) < (d2)) ? (d1) : (d2)) +#define CVM_MAX(d1, d2) (((d1) > (d2)) ? (d1) : (d2)) + +struct niclist { + struct list_head list; + void *ptr; +}; + +struct __dispatch { + struct list_head list; + struct octeon_recv_info *rinfo; + octeon_dispatch_fn_t disp_fn; +}; + +/** Get the argument that the user set when registering dispatch + * function for a given opcode/subcode. + * @param octeon_dev - the octeon device pointer. + * @param opcode - the opcode for which the dispatch argument + * is to be checked. + * @param subcode - the subcode for which the dispatch argument + * is to be checked. + * @return Success: void * (argument to the dispatch function) + * @return Failure: NULL + * + */ +static inline void *octeon_get_dispatch_arg(struct octeon_device *octeon_dev, + u16 opcode, u16 subcode) +{ + int idx; + struct list_head *dispatch; + void *fn_arg = NULL; + u16 combined_opcode = OPCODE_SUBCODE(opcode, subcode); + + idx = combined_opcode & OCTEON_OPCODE_MASK; + + spin_lock_bh(&octeon_dev->dispatch.lock); + + if (octeon_dev->dispatch.count == 0) { + spin_unlock_bh(&octeon_dev->dispatch.lock); + return NULL; + } + + if (octeon_dev->dispatch.dlist[idx].opcode == combined_opcode) { + fn_arg = octeon_dev->dispatch.dlist[idx].arg; + } else { + list_for_each(dispatch, + &octeon_dev->dispatch.dlist[idx].list) { + if (((struct octeon_dispatch *)dispatch)->opcode == + combined_opcode) { + fn_arg = ((struct octeon_dispatch *) + dispatch)->arg; + break; + } + } + } + + spin_unlock_bh(&octeon_dev->dispatch.lock); + return fn_arg; +} + +u32 octeon_droq_check_hw_for_pkts(struct octeon_device *oct, + struct octeon_droq *droq) +{ + u32 pkt_count = 0; + + pkt_count = readl(droq->pkts_sent_reg); + if (pkt_count) { + atomic_add(pkt_count, &droq->pkts_pending); + writel(pkt_count, droq->pkts_sent_reg); + } + + return pkt_count; +} + +static void octeon_droq_compute_max_packet_bufs(struct octeon_droq *droq) +{ + u32 count = 0; + + /* max_empty_descs is the max. no. of descs that can have no buffers. + * If the empty desc count goes beyond this value, we cannot safely + * read in a 64K packet sent by Octeon + * (64K is max pkt size from Octeon) + */ + droq->max_empty_descs = 0; + + do { + droq->max_empty_descs++; + count += droq->buffer_size; + } while (count < (64 * 1024)); + + droq->max_empty_descs = droq->max_count - droq->max_empty_descs; +} + +static void octeon_droq_reset_indices(struct octeon_droq *droq) +{ + droq->read_idx = 0; + droq->write_idx = 0; + droq->refill_idx = 0; + droq->refill_count = 0; + atomic_set(&droq->pkts_pending, 0); +} + +static void +octeon_droq_destroy_ring_buffers(struct octeon_device *oct, + struct octeon_droq *droq) +{ + u32 i; + + for (i = 0; i < droq->max_count; i++) { + if (droq->recv_buf_list[i].buffer) { + if (droq->desc_ring) { + lio_unmap_ring_info(oct->pci_dev, + (u64)droq-> + desc_ring[i].info_ptr, + OCT_DROQ_INFO_SIZE); + lio_unmap_ring(oct->pci_dev, + (u64)droq->desc_ring[i]. + buffer_ptr, + droq->buffer_size); + } + recv_buffer_free(droq->recv_buf_list[i].buffer); + droq->recv_buf_list[i].buffer = NULL; + } + } + + octeon_droq_reset_indices(droq); +} + +static int +octeon_droq_setup_ring_buffers(struct octeon_device *oct, + struct octeon_droq *droq) +{ + u32 i; + void *buf; + struct octeon_droq_desc *desc_ring = droq->desc_ring; + + for (i = 0; i < droq->max_count; i++) { + buf = recv_buffer_alloc(oct, droq->q_no, droq->buffer_size); + + if (!buf) { + dev_err(&oct->pci_dev->dev, "%s buffer alloc failed\n", + __func__); + return -ENOMEM; + } + + droq->recv_buf_list[i].buffer = buf; + droq->recv_buf_list[i].data = get_rbd(buf); + + droq->info_list[i].length = 0; + + /* map ring buffers into memory */ + desc_ring[i].info_ptr = lio_map_ring_info(droq, i); + desc_ring[i].buffer_ptr = + lio_map_ring(oct->pci_dev, + droq->recv_buf_list[i].buffer, + droq->buffer_size); + } + + octeon_droq_reset_indices(droq); + + octeon_droq_compute_max_packet_bufs(droq); + + return 0; +} + +int octeon_delete_droq(struct octeon_device *oct, u32 q_no) +{ + struct octeon_droq *droq = oct->droq[q_no]; + + dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no); + + octeon_droq_destroy_ring_buffers(oct, droq); + vfree(droq->recv_buf_list); + + if (droq->info_base_addr) + cnnic_free_aligned_dma(oct->pci_dev, droq->info_list, + droq->info_alloc_size, + droq->info_base_addr, + droq->info_list_dma); + + if (droq->desc_ring) + lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE), + droq->desc_ring, droq->desc_ring_dma); + + memset(droq, 0, OCT_DROQ_SIZE); + + return 0; +} + +int octeon_init_droq(struct octeon_device *oct, + u32 q_no, + u32 num_descs, + u32 desc_size, + void *app_ctx) +{ + struct octeon_droq *droq; + u32 desc_ring_size = 0, c_num_descs = 0, c_buf_size = 0; + u32 c_pkts_per_intr = 0, c_refill_threshold = 0; + + dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no); + + droq = oct->droq[q_no]; + memset(droq, 0, OCT_DROQ_SIZE); + + droq->oct_dev = oct; + droq->q_no = q_no; + if (app_ctx) + droq->app_ctx = app_ctx; + else + droq->app_ctx = (void *)(size_t)q_no; + + c_num_descs = num_descs; + c_buf_size = desc_size; + if (OCTEON_CN6XXX(oct)) { + struct octeon_config *conf6x = CHIP_FIELD(oct, cn6xxx, conf); + + c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf6x); + c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf6x); + } + + droq->max_count = c_num_descs; + droq->buffer_size = c_buf_size; + + desc_ring_size = droq->max_count * OCT_DROQ_DESC_SIZE; + droq->desc_ring = lio_dma_alloc(oct, desc_ring_size, + (dma_addr_t *)&droq->desc_ring_dma); + + if (!droq->desc_ring) { + dev_err(&oct->pci_dev->dev, + "Output queue %d ring alloc failed\n", q_no); + return 1; + } + + dev_dbg(&oct->pci_dev->dev, "droq[%d]: desc_ring: virt: 0x%p, dma: %lx\n", + q_no, droq->desc_ring, droq->desc_ring_dma); + dev_dbg(&oct->pci_dev->dev, "droq[%d]: num_desc: %d\n", q_no, + droq->max_count); + + droq->info_list = + cnnic_alloc_aligned_dma(oct->pci_dev, + (droq->max_count * OCT_DROQ_INFO_SIZE), + &droq->info_alloc_size, + &droq->info_base_addr, + &droq->info_list_dma); + + if (!droq->info_list) { + dev_err(&oct->pci_dev->dev, "Cannot allocate memory for info list.\n"); + lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE), + droq->desc_ring, droq->desc_ring_dma); + return 1; + } + + droq->recv_buf_list = (struct octeon_recv_buffer *) + vmalloc(droq->max_count * + OCT_DROQ_RECVBUF_SIZE); + if (!droq->recv_buf_list) { + dev_err(&oct->pci_dev->dev, "Output queue recv buf list alloc failed\n"); + goto init_droq_fail; + } + + if (octeon_droq_setup_ring_buffers(oct, droq)) + goto init_droq_fail; + + droq->pkts_per_intr = c_pkts_per_intr; + droq->refill_threshold = c_refill_threshold; + + dev_dbg(&oct->pci_dev->dev, "DROQ INIT: max_empty_descs: %d\n", + droq->max_empty_descs); + + spin_lock_init(&droq->lock); + + INIT_LIST_HEAD(&droq->dispatch_list); + + /* For 56xx Pass1, this function won't be called, so no checks. */ + oct->fn_list.setup_oq_regs(oct, q_no); + + oct->io_qmask.oq |= (1 << q_no); + + return 0; + +init_droq_fail: + octeon_delete_droq(oct, q_no); + return 1; +} + +/* octeon_create_recv_info + * Parameters: + * octeon_dev - pointer to the octeon device structure + * droq - droq in which the packet arrived. + * buf_cnt - no. of buffers used by the packet. + * idx - index in the descriptor for the first buffer in the packet. + * Description: + * Allocates a recv_info_t and copies the buffer addresses for packet data + * into the recv_pkt space which starts at an 8B offset from recv_info_t. + * Flags the descriptors for refill later. If available descriptors go + * below the threshold to receive a 64K pkt, new buffers are first allocated + * before the recv_pkt_t is created. + * This routine will be called in interrupt context. + * Returns: + * Success: Pointer to recv_info_t + * Failure: NULL. + * Locks: + * The droq->lock is held when this routine is called. + */ +static inline struct octeon_recv_info *octeon_create_recv_info( + struct octeon_device *octeon_dev, + struct octeon_droq *droq, + u32 buf_cnt, + u32 idx) +{ + struct octeon_droq_info *info; + struct octeon_recv_pkt *recv_pkt; + struct octeon_recv_info *recv_info; + u32 i, bytes_left; + + info = &droq->info_list[idx]; + + recv_info = octeon_alloc_recv_info(sizeof(struct __dispatch)); + if (!recv_info) + return NULL; + + recv_pkt = recv_info->recv_pkt; + recv_pkt->rh = info->rh; + recv_pkt->length = (u32)info->length; + recv_pkt->buffer_count = (u16)buf_cnt; + recv_pkt->octeon_id = (u16)octeon_dev->octeon_id; + + i = 0; + bytes_left = (u32)info->length; + + while (buf_cnt) { + lio_unmap_ring(octeon_dev->pci_dev, + (u64)droq->desc_ring[idx].buffer_ptr, + droq->buffer_size); + + recv_pkt->buffer_size[i] = + (bytes_left >= + droq->buffer_size) ? droq->buffer_size : bytes_left; + + recv_pkt->buffer_ptr[i] = droq->recv_buf_list[idx].buffer; + droq->recv_buf_list[idx].buffer = NULL; + + INCR_INDEX_BY1(idx, droq->max_count); + bytes_left -= droq->buffer_size; + i++; + buf_cnt--; + } + + return recv_info; +} + +/* If we were not able to refill all buffers, try to move around + * the buffers that were not dispatched. + */ +static inline u32 +octeon_droq_refill_pullup_descs(struct octeon_droq *droq, + struct octeon_droq_desc *desc_ring) +{ + u32 desc_refilled = 0; + + u32 refill_index = droq->refill_idx; + + while (refill_index != droq->read_idx) { + if (droq->recv_buf_list[refill_index].buffer) { + droq->recv_buf_list[droq->refill_idx].buffer = + droq->recv_buf_list[refill_index].buffer; + droq->recv_buf_list[droq->refill_idx].data = + droq->recv_buf_list[refill_index].data; + desc_ring[droq->refill_idx].buffer_ptr = + desc_ring[refill_index].buffer_ptr; + droq->recv_buf_list[refill_index].buffer = NULL; + desc_ring[refill_index].buffer_ptr = 0; + do { + INCR_INDEX_BY1(droq->refill_idx, + droq->max_count); + desc_refilled++; + droq->refill_count--; + } while (droq->recv_buf_list[droq->refill_idx]. + buffer); + } + INCR_INDEX_BY1(refill_index, droq->max_count); + } /* while */ + return desc_refilled; +} + +/* octeon_droq_refill + * Parameters: + * droq - droq in which descriptors require new buffers. + * Description: + * Called during normal DROQ processing in interrupt mode or by the poll + * thread to refill the descriptors from which buffers were dispatched + * to upper layers. Attempts to allocate new buffers. If that fails, moves + * up buffers (that were not dispatched) to form a contiguous ring. + * Returns: + * No of descriptors refilled. + * Locks: + * This routine is called with droq->lock held. + */ +static u32 +octeon_droq_refill(struct octeon_device *octeon_dev, struct octeon_droq *droq) +{ + struct octeon_droq_desc *desc_ring; + void *buf = NULL; + u8 *data; + u32 desc_refilled = 0; + + desc_ring = droq->desc_ring; + + while (droq->refill_count && (desc_refilled < droq->max_count)) { + /* If a valid buffer exists (happens if there is no dispatch), + * reuse + * the buffer, else allocate. + */ + if (!droq->recv_buf_list[droq->refill_idx].buffer) { + buf = recv_buffer_alloc(octeon_dev, droq->q_no, + droq->buffer_size); + /* If a buffer could not be allocated, no point in + * continuing + */ + if (!buf) + break; + droq->recv_buf_list[droq->refill_idx].buffer = + buf; + data = get_rbd(buf); + } else { + data = get_rbd(droq->recv_buf_list + [droq->refill_idx].buffer); + } + + droq->recv_buf_list[droq->refill_idx].data = data; + + desc_ring[droq->refill_idx].buffer_ptr = + lio_map_ring(octeon_dev->pci_dev, + droq->recv_buf_list[droq-> + refill_idx].buffer, + droq->buffer_size); + + /* Reset any previous values in the length field. */ + droq->info_list[droq->refill_idx].length = 0; + + INCR_INDEX_BY1(droq->refill_idx, droq->max_count); + desc_refilled++; + droq->refill_count--; + } + + if (droq->refill_count) + desc_refilled += + octeon_droq_refill_pullup_descs(droq, desc_ring); + + /* if droq->refill_count + * The refill count would not change in pass two. We only moved buffers + * to close the gap in the ring, but we would still have the same no. of + * buffers to refill. + */ + return desc_refilled; +} + +static inline u32 +octeon_droq_get_bufcount(u32 buf_size, u32 total_len) +{ + u32 buf_cnt = 0; + + while (total_len > (buf_size * buf_cnt)) + buf_cnt++; + return buf_cnt; +} + +static int +octeon_droq_dispatch_pkt(struct octeon_device *oct, + struct octeon_droq *droq, + union octeon_rh *rh, + struct octeon_droq_info *info) +{ + u32 cnt; + octeon_dispatch_fn_t disp_fn; + struct octeon_recv_info *rinfo; + + cnt = octeon_droq_get_bufcount(droq->buffer_size, (u32)info->length); + + disp_fn = octeon_get_dispatch(oct, (u16)rh->r.opcode, + (u16)rh->r.subcode); + if (disp_fn) { + rinfo = octeon_create_recv_info(oct, droq, cnt, droq->read_idx); + if (rinfo) { + struct __dispatch *rdisp = rinfo->rsvd; + + rdisp->rinfo = rinfo; + rdisp->disp_fn = disp_fn; + rinfo->recv_pkt->rh = *rh; + list_add_tail(&rdisp->list, + &droq->dispatch_list); + } else { + droq->stats.dropped_nomem++; + } + } else { + dev_err(&oct->pci_dev->dev, "DROQ: No dispatch function\n"); + droq->stats.dropped_nodispatch++; + } /* else (dispatch_fn ... */ + + return cnt; +} + +static inline void octeon_droq_drop_packets(struct octeon_device *oct, + struct octeon_droq *droq, + u32 cnt) +{ + u32 i = 0, buf_cnt; + struct octeon_droq_info *info; + + for (i = 0; i < cnt; i++) { + info = &droq->info_list[droq->read_idx]; + octeon_swap_8B_data((u64 *)info, 2); + + if (info->length) { + info->length -= OCT_RH_SIZE; + droq->stats.bytes_received += info->length; + buf_cnt = octeon_droq_get_bufcount(droq->buffer_size, + (u32)info->length); + } else { + dev_err(&oct->pci_dev->dev, "DROQ: In drop: pkt with len 0\n"); + buf_cnt = 1; + } + + INCR_INDEX(droq->read_idx, buf_cnt, droq->max_count); + droq->refill_count += buf_cnt; + } +} + +static u32 +octeon_droq_fast_process_packets(struct octeon_device *oct, + struct octeon_droq *droq, + u32 pkts_to_process) +{ + struct octeon_droq_info *info; + union octeon_rh *rh; + u32 pkt, total_len = 0, pkt_count; + + pkt_count = pkts_to_process; + + for (pkt = 0; pkt < pkt_count; pkt++) { + u32 pkt_len = 0; + struct sk_buff *nicbuf = NULL; + + info = &droq->info_list[droq->read_idx]; + octeon_swap_8B_data((u64 *)info, 2); + + if (!info->length) { + dev_err(&oct->pci_dev->dev, + "DROQ[%d] idx: %d len:0, pkt_cnt: %d\n", + droq->q_no, droq->read_idx, pkt_count); + print_hex_dump_bytes("", DUMP_PREFIX_ADDRESS, + (u8 *)info, + OCT_DROQ_INFO_SIZE); + break; + } + + /* Len of resp hdr in included in the received data len. */ + info->length -= OCT_RH_SIZE; + rh = &info->rh; + + total_len += (u32)info->length; + + if (OPCODE_SLOW_PATH(rh)) { + u32 buf_cnt; + + buf_cnt = octeon_droq_dispatch_pkt(oct, droq, rh, info); + INCR_INDEX(droq->read_idx, buf_cnt, droq->max_count); + droq->refill_count += buf_cnt; + } else { + if (info->length <= droq->buffer_size) { + lio_unmap_ring(oct->pci_dev, + (u64)droq->desc_ring[ + droq->read_idx].buffer_ptr, + droq->buffer_size); + pkt_len = (u32)info->length; + nicbuf = droq->recv_buf_list[ + droq->read_idx].buffer; + droq->recv_buf_list[droq->read_idx].buffer = + NULL; + INCR_INDEX_BY1(droq->read_idx, droq->max_count); + skb_put(nicbuf, pkt_len); + droq->refill_count++; + } else { + nicbuf = octeon_fast_packet_alloc(oct, droq, + droq->q_no, + (u32) + info->length); + pkt_len = 0; + /* nicbuf allocation can fail. We'll handle it + * inside the loop. + */ + while (pkt_len < info->length) { + int cpy_len; + + cpy_len = ((pkt_len + + droq->buffer_size) > + info->length) ? + ((u32)info->length - pkt_len) : + droq->buffer_size; + + if (nicbuf) { + lio_unmap_ring(oct->pci_dev, + (u64) + droq->desc_ring + [droq->read_idx]. + buffer_ptr, + droq-> + buffer_size); + octeon_fast_packet_next(droq, + nicbuf, + cpy_len, + droq-> + read_idx + ); + } + + pkt_len += cpy_len; + INCR_INDEX_BY1(droq->read_idx, + droq->max_count); + droq->refill_count++; + } + } + + if (nicbuf) { + if (droq->ops.fptr) + droq->ops.fptr(oct->octeon_id, + nicbuf, pkt_len, + rh, &droq->napi); + else + recv_buffer_free(nicbuf); + } + } + + if (droq->refill_count >= droq->refill_threshold) { + int desc_refilled = octeon_droq_refill(oct, droq); + + /* Flush the droq descriptor data to memory to be sure + * that when we update the credits the data in memory + * is accurate. + */ + wmb(); + writel((desc_refilled), droq->pkts_credit_reg); + /* make sure mmio write completes */ + mmiowb(); + } + + } /* for ( each packet )... */ + + /* Increment refill_count by the number of buffers processed. */ + droq->stats.pkts_received += pkt; + droq->stats.bytes_received += total_len; + + if ((droq->ops.drop_on_max) && (pkts_to_process - pkt)) { + octeon_droq_drop_packets(oct, droq, (pkts_to_process - pkt)); + + droq->stats.dropped_toomany += (pkts_to_process - pkt); + return pkts_to_process; + } + + return pkt; +} + +int +octeon_droq_process_packets(struct octeon_device *oct, + struct octeon_droq *droq, + u32 budget) +{ + u32 pkt_count = 0, pkts_processed = 0; + struct list_head *tmp, *tmp2; + + pkt_count = atomic_read(&droq->pkts_pending); + if (!pkt_count) + return 0; + + if (pkt_count > budget) + pkt_count = budget; + + /* Grab the lock */ + spin_lock(&droq->lock); + + pkts_processed = octeon_droq_fast_process_packets(oct, droq, pkt_count); + + atomic_sub(pkts_processed, &droq->pkts_pending); + + /* Release the spin lock */ + spin_unlock(&droq->lock); + + list_for_each_safe(tmp, tmp2, &droq->dispatch_list) { + struct __dispatch *rdisp = (struct __dispatch *)tmp; + + list_del(tmp); + rdisp->disp_fn(rdisp->rinfo, + octeon_get_dispatch_arg + (oct, + (u16)rdisp->rinfo->recv_pkt->rh.r.opcode, + (u16)rdisp->rinfo->recv_pkt->rh.r.subcode)); + } + + /* If there are packets pending. schedule tasklet again */ + if (atomic_read(&droq->pkts_pending)) + return 1; + + return 0; +} + +/** + * Utility function to poll for packets. check_hw_for_packets must be + * called before calling this routine. + */ + +static int +octeon_droq_process_poll_pkts(struct octeon_device *oct, + struct octeon_droq *droq, u32 budget) +{ + struct list_head *tmp, *tmp2; + u32 pkts_available = 0, pkts_processed = 0; + u32 total_pkts_processed = 0; + + if (budget > droq->max_count) + budget = droq->max_count; + + spin_lock(&droq->lock); + + while (total_pkts_processed < budget) { + pkts_available = + CVM_MIN((budget - total_pkts_processed), + (u32)(atomic_read(&droq->pkts_pending))); + + if (pkts_available == 0) + break; + + pkts_processed = + octeon_droq_fast_process_packets(oct, droq, + pkts_available); + + atomic_sub(pkts_processed, &droq->pkts_pending); + + total_pkts_processed += pkts_processed; + + octeon_droq_check_hw_for_pkts(oct, droq); + } + + spin_unlock(&droq->lock); + + list_for_each_safe(tmp, tmp2, &droq->dispatch_list) { + struct __dispatch *rdisp = (struct __dispatch *)tmp; + + list_del(tmp); + rdisp->disp_fn(rdisp->rinfo, + octeon_get_dispatch_arg + (oct, + (u16)rdisp->rinfo->recv_pkt->rh.r.opcode, + (u16)rdisp->rinfo->recv_pkt->rh.r.subcode)); + } + + return total_pkts_processed; +} + +int +octeon_process_droq_poll_cmd(struct octeon_device *oct, u32 q_no, int cmd, + u32 arg) +{ + struct octeon_droq *droq; + struct octeon_config *oct_cfg = NULL; + + oct_cfg = octeon_get_conf(oct); + + if (!oct_cfg) + return -EINVAL; + + if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) { + dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n", + __func__, q_no, (oct->num_oqs - 1)); + return -EINVAL; + } + + droq = oct->droq[q_no]; + + if (cmd == POLL_EVENT_PROCESS_PKTS) + return octeon_droq_process_poll_pkts(oct, droq, arg); + + if (cmd == POLL_EVENT_PENDING_PKTS) { + u32 pkt_cnt = atomic_read(&droq->pkts_pending); + + return octeon_droq_process_packets(oct, droq, pkt_cnt); + } + + if (cmd == POLL_EVENT_ENABLE_INTR) { + u32 value; + unsigned long flags; + + /* Enable Pkt Interrupt */ + switch (oct->chip_id) { + case OCTEON_CN66XX: + case OCTEON_CN68XX: { + struct octeon_cn6xxx *cn6xxx = + (struct octeon_cn6xxx *)oct->chip; + spin_lock_irqsave + (&cn6xxx->lock_for_droq_int_enb_reg, flags); + value = + octeon_read_csr(oct, + CN6XXX_SLI_PKT_TIME_INT_ENB); + value |= (1 << q_no); + octeon_write_csr(oct, + CN6XXX_SLI_PKT_TIME_INT_ENB, + value); + value = + octeon_read_csr(oct, + CN6XXX_SLI_PKT_CNT_INT_ENB); + value |= (1 << q_no); + octeon_write_csr(oct, + CN6XXX_SLI_PKT_CNT_INT_ENB, + value); + + /* don't bother flushing the enables */ + + spin_unlock_irqrestore + (&cn6xxx->lock_for_droq_int_enb_reg, flags); + return 0; + } + break; + } + + return 0; + } + + dev_err(&oct->pci_dev->dev, "%s Unknown command: %d\n", __func__, cmd); + return -EINVAL; +} + +int octeon_register_droq_ops(struct octeon_device *oct, u32 q_no, + struct octeon_droq_ops *ops) +{ + struct octeon_droq *droq; + unsigned long flags; + struct octeon_config *oct_cfg = NULL; + + oct_cfg = octeon_get_conf(oct); + + if (!oct_cfg) + return -EINVAL; + + if (!(ops)) { + dev_err(&oct->pci_dev->dev, "%s: droq_ops pointer is NULL\n", + __func__); + return -EINVAL; + } + + if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) { + dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n", + __func__, q_no, (oct->num_oqs - 1)); + return -EINVAL; + } + + droq = oct->droq[q_no]; + + spin_lock_irqsave(&droq->lock, flags); + + memcpy(&droq->ops, ops, sizeof(struct octeon_droq_ops)); + + spin_unlock_irqrestore(&droq->lock, flags); + + return 0; +} + +int octeon_unregister_droq_ops(struct octeon_device *oct, u32 q_no) +{ + unsigned long flags; + struct octeon_droq *droq; + struct octeon_config *oct_cfg = NULL; + + oct_cfg = octeon_get_conf(oct); + + if (!oct_cfg) + return -EINVAL; + + if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) { + dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n", + __func__, q_no, oct->num_oqs - 1); + return -EINVAL; + } + + droq = oct->droq[q_no]; + + if (!droq) { + dev_info(&oct->pci_dev->dev, + "Droq id (%d) not available.\n", q_no); + return 0; + } + + spin_lock_irqsave(&droq->lock, flags); + + droq->ops.fptr = NULL; + droq->ops.drop_on_max = 0; + + spin_unlock_irqrestore(&droq->lock, flags); + + return 0; +} + +int octeon_create_droq(struct octeon_device *oct, + u32 q_no, u32 num_descs, + u32 desc_size, void *app_ctx) +{ + struct octeon_droq *droq; + + if (oct->droq[q_no]) { + dev_dbg(&oct->pci_dev->dev, "Droq already in use. Cannot create droq %d again\n", + q_no); + return 1; + } + + /* Allocate the DS for the new droq. */ + droq = vmalloc(sizeof(*droq)); + if (!droq) + goto create_droq_fail; + memset(droq, 0, sizeof(struct octeon_droq)); + + /*Disable the pkt o/p for this Q */ + octeon_set_droq_pkt_op(oct, q_no, 0); + oct->droq[q_no] = droq; + + /* Initialize the Droq */ + octeon_init_droq(oct, q_no, num_descs, desc_size, app_ctx); + + oct->num_oqs++; + + dev_dbg(&oct->pci_dev->dev, "%s: Total number of OQ: %d\n", __func__, + oct->num_oqs); + + /* Global Droq register settings */ + + /* As of now not required, as setting are done for all 32 Droqs at + * the same time. + */ + return 0; + +create_droq_fail: + octeon_delete_droq(oct, q_no); + return -1; +} diff --git a/kernel/drivers/net/ethernet/cavium/liquidio/octeon_droq.h b/kernel/drivers/net/ethernet/cavium/liquidio/octeon_droq.h new file mode 100644 index 000000000..7940ccee1 --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/liquidio/octeon_droq.h @@ -0,0 +1,426 @@ +/********************************************************************** + * Author: Cavium, Inc. + * + * Contact: support@cavium.com + * Please include "LiquidIO" in the subject. + * + * Copyright (c) 2003-2015 Cavium, Inc. + * + * This file is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License, Version 2, as + * published by the Free Software Foundation. + * + * This file is distributed in the hope that it will be useful, but + * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or + * NONINFRINGEMENT. See the GNU General Public License for more + * details. + * + * This file may also be available under a different license from Cavium. + * Contact Cavium, Inc. for more information + **********************************************************************/ + +/*! \file octeon_droq.h + * \brief Implementation of Octeon Output queues. "Output" is with + * respect to the Octeon device on the NIC. From this driver's point of + * view they are ingress queues. + */ + +#ifndef __OCTEON_DROQ_H__ +#define __OCTEON_DROQ_H__ + +/* Default number of packets that will be processed in one iteration. */ +#define MAX_PACKET_BUDGET 0xFFFFFFFF + +/** Octeon descriptor format. + * The descriptor ring is made of descriptors which have 2 64-bit values: + * -# Physical (bus) address of the data buffer. + * -# Physical (bus) address of a octeon_droq_info structure. + * The Octeon device DMA's incoming packets and its information at the address + * given by these descriptor fields. + */ +struct octeon_droq_desc { + /** The buffer pointer */ + u64 buffer_ptr; + + /** The Info pointer */ + u64 info_ptr; +}; + +#define OCT_DROQ_DESC_SIZE (sizeof(struct octeon_droq_desc)) + +/** Information about packet DMA'ed by Octeon. + * The format of the information available at Info Pointer after Octeon + * has posted a packet. Not all descriptors have valid information. Only + * the Info field of the first descriptor for a packet has information + * about the packet. + */ +struct octeon_droq_info { + /** The Output Receive Header. */ + union octeon_rh rh; + + /** The Length of the packet. */ + u64 length; +}; + +#define OCT_DROQ_INFO_SIZE (sizeof(struct octeon_droq_info)) + +/** Pointer to data buffer. + * Driver keeps a pointer to the data buffer that it made available to + * the Octeon device. Since the descriptor ring keeps physical (bus) + * addresses, this field is required for the driver to keep track of + * the virtual address pointers. +*/ +struct octeon_recv_buffer { + /** Packet buffer, including metadata. */ + void *buffer; + + /** Data in the packet buffer. */ + u8 *data; +}; + +#define OCT_DROQ_RECVBUF_SIZE (sizeof(struct octeon_recv_buffer)) + +/** Output Queue statistics. Each output queue has four stats fields. */ +struct oct_droq_stats { + /** Number of packets received in this queue. */ + u64 pkts_received; + + /** Bytes received by this queue. */ + u64 bytes_received; + + /** Packets dropped due to no dispatch function. */ + u64 dropped_nodispatch; + + /** Packets dropped due to no memory available. */ + u64 dropped_nomem; + + /** Packets dropped due to large number of pkts to process. */ + u64 dropped_toomany; + + /** Number of packets sent to stack from this queue. */ + u64 rx_pkts_received; + + /** Number of Bytes sent to stack from this queue. */ + u64 rx_bytes_received; + + /** Num of Packets dropped due to receive path failures. */ + u64 rx_dropped; +}; + +#define POLL_EVENT_INTR_ARRIVED 1 +#define POLL_EVENT_PROCESS_PKTS 2 +#define POLL_EVENT_PENDING_PKTS 3 +#define POLL_EVENT_ENABLE_INTR 4 + +/* The maximum number of buffers that can be dispatched from the + * output/dma queue. Set to 64 assuming 1K buffers in DROQ and the fact that + * max packet size from DROQ is 64K. + */ +#define MAX_RECV_BUFS 64 + +/** Receive Packet format used when dispatching output queue packets + * with non-raw opcodes. + * The received packet will be sent to the upper layers using this + * structure which is passed as a parameter to the dispatch function + */ +struct octeon_recv_pkt { + /** Number of buffers in this received packet */ + u16 buffer_count; + + /** Id of the device that is sending the packet up */ + u16 octeon_id; + + /** Length of data in the packet buffer */ + u32 length; + + /** The receive header */ + union octeon_rh rh; + + /** Pointer to the OS-specific packet buffer */ + void *buffer_ptr[MAX_RECV_BUFS]; + + /** Size of the buffers pointed to by ptr's in buffer_ptr */ + u32 buffer_size[MAX_RECV_BUFS]; +}; + +#define OCT_RECV_PKT_SIZE (sizeof(struct octeon_recv_pkt)) + +/** The first parameter of a dispatch function. + * For a raw mode opcode, the driver dispatches with the device + * pointer in this structure. + * For non-raw mode opcode, the driver dispatches the recv_pkt + * created to contain the buffers with data received from Octeon. + * --------------------- + * | *recv_pkt ----|--- + * |-------------------| | + * | 0 or more bytes | | + * | reserved by driver| | + * |-------------------|<-/ + * | octeon_recv_pkt | + * | | + * |___________________| + */ +struct octeon_recv_info { + void *rsvd; + struct octeon_recv_pkt *recv_pkt; +}; + +#define OCT_RECV_INFO_SIZE (sizeof(struct octeon_recv_info)) + +/** Allocate a recv_info structure. The recv_pkt pointer in the recv_info + * structure is filled in before this call returns. + * @param extra_bytes - extra bytes to be allocated at the end of the recv info + * structure. + * @return - pointer to a newly allocated recv_info structure. + */ +static inline struct octeon_recv_info *octeon_alloc_recv_info(int extra_bytes) +{ + struct octeon_recv_info *recv_info; + u8 *buf; + + buf = kmalloc(OCT_RECV_PKT_SIZE + OCT_RECV_INFO_SIZE + + extra_bytes, GFP_ATOMIC); + if (!buf) + return NULL; + + recv_info = (struct octeon_recv_info *)buf; + recv_info->recv_pkt = + (struct octeon_recv_pkt *)(buf + OCT_RECV_INFO_SIZE); + recv_info->rsvd = NULL; + if (extra_bytes) + recv_info->rsvd = buf + OCT_RECV_INFO_SIZE + OCT_RECV_PKT_SIZE; + + return recv_info; +} + +/** Free a recv_info structure. + * @param recv_info - Pointer to receive_info to be freed + */ +static inline void octeon_free_recv_info(struct octeon_recv_info *recv_info) +{ + kfree(recv_info); +} + +typedef int (*octeon_dispatch_fn_t)(struct octeon_recv_info *, void *); + +/** Used by NIC module to register packet handler and to get device + * information for each octeon device. + */ +struct octeon_droq_ops { + /** This registered function will be called by the driver with + * the octeon id, pointer to buffer from droq and length of + * data in the buffer. The receive header gives the port + * number to the caller. Function pointer is set by caller. + */ + void (*fptr)(u32, void *, u32, union octeon_rh *, void *); + + /* This function will be called by the driver for all NAPI related + * events. The first param is the octeon id. The second param is the + * output queue number. The third is the NAPI event that occurred. + */ + void (*napi_fn)(void *); + + u32 poll_mode; + + /** Flag indicating if the DROQ handler should drop packets that + * it cannot handle in one iteration. Set by caller. + */ + u32 drop_on_max; +}; + +/** The Descriptor Ring Output Queue structure. + * This structure has all the information required to implement a + * Octeon DROQ. + */ +struct octeon_droq { + /** A spinlock to protect access to this ring. */ + spinlock_t lock; + + u32 q_no; + + struct octeon_droq_ops ops; + + struct octeon_device *oct_dev; + + /** The 8B aligned descriptor ring starts at this address. */ + struct octeon_droq_desc *desc_ring; + + /** Index in the ring where the driver should read the next packet */ + u32 read_idx; + + /** Index in the ring where Octeon will write the next packet */ + u32 write_idx; + + /** Index in the ring where the driver will refill the descriptor's + * buffer + */ + u32 refill_idx; + + /** Packets pending to be processed */ + atomic_t pkts_pending; + + /** Number of descriptors in this ring. */ + u32 max_count; + + /** The number of descriptors pending refill. */ + u32 refill_count; + + u32 pkts_per_intr; + u32 refill_threshold; + + /** The max number of descriptors in DROQ without a buffer. + * This field is used to keep track of empty space threshold. If the + * refill_count reaches this value, the DROQ cannot accept a max-sized + * (64K) packet. + */ + u32 max_empty_descs; + + /** The 8B aligned info ptrs begin from this address. */ + struct octeon_droq_info *info_list; + + /** The receive buffer list. This list has the virtual addresses of the + * buffers. + */ + struct octeon_recv_buffer *recv_buf_list; + + /** The size of each buffer pointed by the buffer pointer. */ + u32 buffer_size; + + /** Pointer to the mapped packet credit register. + * Host writes number of info/buffer ptrs available to this register + */ + void __iomem *pkts_credit_reg; + + /** Pointer to the mapped packet sent register. + * Octeon writes the number of packets DMA'ed to host memory + * in this register. + */ + void __iomem *pkts_sent_reg; + + struct list_head dispatch_list; + + /** Statistics for this DROQ. */ + struct oct_droq_stats stats; + + /** DMA mapped address of the DROQ descriptor ring. */ + size_t desc_ring_dma; + + /** Info ptr list are allocated at this virtual address. */ + size_t info_base_addr; + + /** DMA mapped address of the info list */ + size_t info_list_dma; + + /** Allocated size of info list. */ + u32 info_alloc_size; + + /** application context */ + void *app_ctx; + + struct napi_struct napi; + + u32 cpu_id; + + struct call_single_data csd; +}; + +#define OCT_DROQ_SIZE (sizeof(struct octeon_droq)) + +/** + * Allocates space for the descriptor ring for the droq and sets the + * base addr, num desc etc in Octeon registers. + * + * @param oct_dev - pointer to the octeon device structure + * @param q_no - droq no. ranges from 0 - 3. + * @param app_ctx - pointer to application context + * @return Success: 0 Failure: 1 +*/ +int octeon_init_droq(struct octeon_device *oct_dev, + u32 q_no, + u32 num_descs, + u32 desc_size, + void *app_ctx); + +/** + * Frees the space for descriptor ring for the droq. + * + * @param oct_dev - pointer to the octeon device structure + * @param q_no - droq no. ranges from 0 - 3. + * @return: Success: 0 Failure: 1 +*/ +int octeon_delete_droq(struct octeon_device *oct_dev, u32 q_no); + +/** Register a change in droq operations. The ops field has a pointer to a + * function which will called by the DROQ handler for all packets arriving + * on output queues given by q_no irrespective of the type of packet. + * The ops field also has a flag which if set tells the DROQ handler to + * drop packets if it receives more than what it can process in one + * invocation of the handler. + * @param oct - octeon device + * @param q_no - octeon output queue number (0 <= q_no <= MAX_OCTEON_DROQ-1 + * @param ops - the droq_ops settings for this queue + * @return - 0 on success, -ENODEV or -EINVAL on error. + */ +int +octeon_register_droq_ops(struct octeon_device *oct, + u32 q_no, + struct octeon_droq_ops *ops); + +/** Resets the function pointer and flag settings made by + * octeon_register_droq_ops(). After this routine is called, the DROQ handler + * will lookup dispatch function for each arriving packet on the output queue + * given by q_no. + * @param oct - octeon device + * @param q_no - octeon output queue number (0 <= q_no <= MAX_OCTEON_DROQ-1 + * @return - 0 on success, -ENODEV or -EINVAL on error. + */ +int octeon_unregister_droq_ops(struct octeon_device *oct, u32 q_no); + +/** Register a dispatch function for a opcode/subcode. The driver will call + * this dispatch function when it receives a packet with the given + * opcode/subcode in its output queues along with the user specified + * argument. + * @param oct - the octeon device to register with. + * @param opcode - the opcode for which the dispatch will be registered. + * @param subcode - the subcode for which the dispatch will be registered + * @param fn - the dispatch function. + * @param fn_arg - user specified that will be passed along with the + * dispatch function by the driver. + * @return Success: 0; Failure: 1 + */ +int octeon_register_dispatch_fn(struct octeon_device *oct, + u16 opcode, + u16 subcode, + octeon_dispatch_fn_t fn, void *fn_arg); + +/** Remove registration for an opcode/subcode. This will delete the mapping for + * an opcode/subcode. The dispatch function will be unregistered and will no + * longer be called if a packet with the opcode/subcode arrives in the driver + * output queues. + * @param oct - the octeon device to unregister from. + * @param opcode - the opcode to be unregistered. + * @param subcode - the subcode to be unregistered. + * + * @return Success: 0; Failure: 1 + */ +int octeon_unregister_dispatch_fn(struct octeon_device *oct, + u16 opcode, + u16 subcode); + +void octeon_droq_print_stats(void); + +u32 octeon_droq_check_hw_for_pkts(struct octeon_device *oct, + struct octeon_droq *droq); + +int octeon_create_droq(struct octeon_device *oct, u32 q_no, + u32 num_descs, u32 desc_size, void *app_ctx); + +int octeon_droq_process_packets(struct octeon_device *oct, + struct octeon_droq *droq, + u32 budget); + +int octeon_process_droq_poll_cmd(struct octeon_device *oct, u32 q_no, + int cmd, u32 arg); + +#endif /*__OCTEON_DROQ_H__ */ diff --git a/kernel/drivers/net/ethernet/cavium/liquidio/octeon_iq.h b/kernel/drivers/net/ethernet/cavium/liquidio/octeon_iq.h new file mode 100644 index 000000000..592fe49b5 --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/liquidio/octeon_iq.h @@ -0,0 +1,319 @@ +/********************************************************************** + * Author: Cavium, Inc. + * + * Contact: support@cavium.com + * Please include "LiquidIO" in the subject. + * + * Copyright (c) 2003-2015 Cavium, Inc. + * + * This file is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License, Version 2, as + * published by the Free Software Foundation. + * + * This file is distributed in the hope that it will be useful, but + * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or + * NONINFRINGEMENT. See the GNU General Public License for more + * details. + * + * This file may also be available under a different license from Cavium. + * Contact Cavium, Inc. for more information + **********************************************************************/ + +/*! \file octeon_iq.h + * \brief Host Driver: Implementation of Octeon input queues. "Input" is + * with respect to the Octeon device on the NIC. From this driver's + * point of view they are egress queues. + */ + +#ifndef __OCTEON_IQ_H__ +#define __OCTEON_IQ_H__ + +#define IQ_STATUS_RUNNING 1 + +#define IQ_SEND_OK 0 +#define IQ_SEND_STOP 1 +#define IQ_SEND_FAILED -1 + +/*------------------------- INSTRUCTION QUEUE --------------------------*/ + +/* \cond */ + +#define REQTYPE_NONE 0 +#define REQTYPE_NORESP_NET 1 +#define REQTYPE_NORESP_NET_SG 2 +#define REQTYPE_RESP_NET 3 +#define REQTYPE_RESP_NET_SG 4 +#define REQTYPE_SOFT_COMMAND 5 +#define REQTYPE_LAST 5 + +struct octeon_request_list { + u32 reqtype; + void *buf; +}; + +/* \endcond */ + +/** Input Queue statistics. Each input queue has four stats fields. */ +struct oct_iq_stats { + u64 instr_posted; /**< Instructions posted to this queue. */ + u64 instr_processed; /**< Instructions processed in this queue. */ + u64 instr_dropped; /**< Instructions that could not be processed */ + u64 bytes_sent; /**< Bytes sent through this queue. */ + u64 sgentry_sent;/**< Gather entries sent through this queue. */ + u64 tx_done;/**< Num of packets sent to network. */ + u64 tx_iq_busy;/**< Numof times this iq was found to be full. */ + u64 tx_dropped;/**< Numof pkts dropped dueto xmitpath errors. */ + u64 tx_tot_bytes;/**< Total count of bytes sento to network. */ +}; + +#define OCT_IQ_STATS_SIZE (sizeof(struct oct_iq_stats)) + +/** The instruction (input) queue. + * The input queue is used to post raw (instruction) mode data or packet + * data to Octeon device from the host. Each input queue (upto 4) for + * a Octeon device has one such structure to represent it. +*/ +struct octeon_instr_queue { + /** A spinlock to protect access to the input ring. */ + spinlock_t lock; + + /** Flag that indicates if the queue uses 64 byte commands. */ + u32 iqcmd_64B:1; + + /** Queue Number. */ + u32 iq_no:5; + + u32 rsvd:17; + + /* Controls the periodic flushing of iq */ + u32 do_auto_flush:1; + + u32 status:8; + + /** Maximum no. of instructions in this queue. */ + u32 max_count; + + /** Index in input ring where the driver should write the next packet */ + u32 host_write_index; + + /** Index in input ring where Octeon is expected to read the next + * packet. + */ + u32 octeon_read_index; + + /** This index aids in finding the window in the queue where Octeon + * has read the commands. + */ + u32 flush_index; + + /** This field keeps track of the instructions pending in this queue. */ + atomic_t instr_pending; + + u32 reset_instr_cnt; + + /** Pointer to the Virtual Base addr of the input ring. */ + u8 *base_addr; + + struct octeon_request_list *request_list; + + /** Octeon doorbell register for the ring. */ + void __iomem *doorbell_reg; + + /** Octeon instruction count register for this ring. */ + void __iomem *inst_cnt_reg; + + /** Number of instructions pending to be posted to Octeon. */ + u32 fill_cnt; + + /** The max. number of instructions that can be held pending by the + * driver. + */ + u32 fill_threshold; + + /** The last time that the doorbell was rung. */ + u64 last_db_time; + + /** The doorbell timeout. If the doorbell was not rung for this time and + * fill_cnt is non-zero, ring the doorbell again. + */ + u32 db_timeout; + + /** Statistics for this input queue. */ + struct oct_iq_stats stats; + + /** DMA mapped base address of the input descriptor ring. */ + u64 base_addr_dma; + + /** Application context */ + void *app_ctx; +}; + +/*---------------------- INSTRUCTION FORMAT ----------------------------*/ + +/** 32-byte instruction format. + * Format of instruction for a 32-byte mode input queue. + */ +struct octeon_instr_32B { + /** Pointer where the input data is available. */ + u64 dptr; + + /** Instruction Header. */ + u64 ih; + + /** Pointer where the response for a RAW mode packet will be written + * by Octeon. + */ + u64 rptr; + + /** Input Request Header. Additional info about the input. */ + u64 irh; + +}; + +#define OCT_32B_INSTR_SIZE (sizeof(struct octeon_instr_32B)) + +/** 64-byte instruction format. + * Format of instruction for a 64-byte mode input queue. + */ +struct octeon_instr_64B { + /** Pointer where the input data is available. */ + u64 dptr; + + /** Instruction Header. */ + u64 ih; + + /** Input Request Header. */ + u64 irh; + + /** opcode/subcode specific parameters */ + u64 ossp[2]; + + /** Return Data Parameters */ + u64 rdp; + + /** Pointer where the response for a RAW mode packet will be written + * by Octeon. + */ + u64 rptr; + + u64 reserved; + +}; + +#define OCT_64B_INSTR_SIZE (sizeof(struct octeon_instr_64B)) + +/** The size of each buffer in soft command buffer pool + */ +#define SOFT_COMMAND_BUFFER_SIZE 1024 + +struct octeon_soft_command { + /** Soft command buffer info. */ + struct list_head node; + u64 dma_addr; + u32 size; + + /** Command and return status */ + struct octeon_instr_64B cmd; +#define COMPLETION_WORD_INIT 0xffffffffffffffffULL + u64 *status_word; + + /** Data buffer info */ + void *virtdptr; + u64 dmadptr; + u32 datasize; + + /** Return buffer info */ + void *virtrptr; + u64 dmarptr; + u32 rdatasize; + + /** Context buffer info */ + void *ctxptr; + u32 ctxsize; + + /** Time out and callback */ + size_t wait_time; + size_t timeout; + u32 iq_no; + void (*callback)(struct octeon_device *, u32, void *); + void *callback_arg; +}; + +/** Maximum number of buffers to allocate into soft command buffer pool + */ +#define MAX_SOFT_COMMAND_BUFFERS 16 + +/** Head of a soft command buffer pool. + */ +struct octeon_sc_buffer_pool { + /** List structure to add delete pending entries to */ + struct list_head head; + + /** A lock for this response list */ + spinlock_t lock; + + atomic_t alloc_buf_count; +}; + +int octeon_setup_sc_buffer_pool(struct octeon_device *oct); +int octeon_free_sc_buffer_pool(struct octeon_device *oct); +struct octeon_soft_command * + octeon_alloc_soft_command(struct octeon_device *oct, + u32 datasize, u32 rdatasize, + u32 ctxsize); +void octeon_free_soft_command(struct octeon_device *oct, + struct octeon_soft_command *sc); + +/** + * octeon_init_instr_queue() + * @param octeon_dev - pointer to the octeon device structure. + * @param iq_no - queue to be initialized (0 <= q_no <= 3). + * + * Called at driver init time for each input queue. iq_conf has the + * configuration parameters for the queue. + * + * @return Success: 0 Failure: 1 + */ +int octeon_init_instr_queue(struct octeon_device *octeon_dev, u32 iq_no, + u32 num_descs); + +/** + * octeon_delete_instr_queue() + * @param octeon_dev - pointer to the octeon device structure. + * @param iq_no - queue to be deleted (0 <= q_no <= 3). + * + * Called at driver unload time for each input queue. Deletes all + * allocated resources for the input queue. + * + * @return Success: 0 Failure: 1 + */ +int octeon_delete_instr_queue(struct octeon_device *octeon_dev, u32 iq_no); + +int lio_wait_for_instr_fetch(struct octeon_device *oct); + +int +octeon_register_reqtype_free_fn(struct octeon_device *oct, int reqtype, + void (*fn)(void *)); + +int +lio_process_iq_request_list(struct octeon_device *oct, + struct octeon_instr_queue *iq); + +int octeon_send_command(struct octeon_device *oct, u32 iq_no, + u32 force_db, void *cmd, void *buf, + u32 datasize, u32 reqtype); + +void octeon_prepare_soft_command(struct octeon_device *oct, + struct octeon_soft_command *sc, + u8 opcode, u8 subcode, + u32 irh_ossp, u64 ossp0, + u64 ossp1); + +int octeon_send_soft_command(struct octeon_device *oct, + struct octeon_soft_command *sc); + +int octeon_setup_iq(struct octeon_device *oct, u32 iq_no, + u32 num_descs, void *app_ctx); + +#endif /* __OCTEON_IQ_H__ */ diff --git a/kernel/drivers/net/ethernet/cavium/liquidio/octeon_main.h b/kernel/drivers/net/ethernet/cavium/liquidio/octeon_main.h new file mode 100644 index 000000000..cbd081981 --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/liquidio/octeon_main.h @@ -0,0 +1,237 @@ +/********************************************************************** + * Author: Cavium, Inc. + * + * Contact: support@cavium.com + * Please include "LiquidIO" in the subject. + * + * Copyright (c) 2003-2015 Cavium, Inc. + * + * This file is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License, Version 2, as + * published by the Free Software Foundation. + * + * This file is distributed in the hope that it will be useful, but + * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or + * NONINFRINGEMENT. See the GNU General Public License for more + * details. + * + * This file may also be available under a different license from Cavium. + * Contact Cavium, Inc. for more information + **********************************************************************/ + +/*! \file octeon_main.h + * \brief Host Driver: This file is included by all host driver source files + * to include common definitions. + */ + +#ifndef _OCTEON_MAIN_H_ +#define _OCTEON_MAIN_H_ + +#if BITS_PER_LONG == 32 +#define CVM_CAST64(v) ((long long)(v)) +#elif BITS_PER_LONG == 64 +#define CVM_CAST64(v) ((long long)(long)(v)) +#else +#error "Unknown system architecture" +#endif + +#define DRV_NAME "LiquidIO" + +/** + * \brief determines if a given console has debug enabled. + * @param console console to check + * @returns 1 = enabled. 0 otherwise + */ +int octeon_console_debug_enabled(u32 console); + +/* BQL-related functions */ +void octeon_report_sent_bytes_to_bql(void *buf, int reqtype); +void octeon_update_tx_completion_counters(void *buf, int reqtype, + unsigned int *pkts_compl, + unsigned int *bytes_compl); +void octeon_report_tx_completion_to_bql(void *txq, unsigned int pkts_compl, + unsigned int bytes_compl); + +/** Swap 8B blocks */ +static inline void octeon_swap_8B_data(u64 *data, u32 blocks) +{ + while (blocks) { + cpu_to_be64s(data); + blocks--; + data++; + } +} + +/** + * \brief unmaps a PCI BAR + * @param oct Pointer to Octeon device + * @param baridx bar index + */ +static inline void octeon_unmap_pci_barx(struct octeon_device *oct, int baridx) +{ + dev_dbg(&oct->pci_dev->dev, "Freeing PCI mapped regions for Bar%d\n", + baridx); + + if (oct->mmio[baridx].done) + iounmap(oct->mmio[baridx].hw_addr); + + if (oct->mmio[baridx].start) + pci_release_region(oct->pci_dev, baridx * 2); +} + +/** + * \brief maps a PCI BAR + * @param oct Pointer to Octeon device + * @param baridx bar index + * @param max_map_len maximum length of mapped memory + */ +static inline int octeon_map_pci_barx(struct octeon_device *oct, + int baridx, int max_map_len) +{ + u32 mapped_len = 0; + + if (pci_request_region(oct->pci_dev, baridx * 2, DRV_NAME)) { + dev_err(&oct->pci_dev->dev, "pci_request_region failed for bar %d\n", + baridx); + return 1; + } + + oct->mmio[baridx].start = pci_resource_start(oct->pci_dev, baridx * 2); + oct->mmio[baridx].len = pci_resource_len(oct->pci_dev, baridx * 2); + + mapped_len = oct->mmio[baridx].len; + if (!mapped_len) + return 1; + + if (max_map_len && (mapped_len > max_map_len)) + mapped_len = max_map_len; + + oct->mmio[baridx].hw_addr = + ioremap(oct->mmio[baridx].start, mapped_len); + oct->mmio[baridx].mapped_len = mapped_len; + + dev_dbg(&oct->pci_dev->dev, "BAR%d start: 0x%llx mapped %u of %u bytes\n", + baridx, oct->mmio[baridx].start, mapped_len, + oct->mmio[baridx].len); + + if (!oct->mmio[baridx].hw_addr) { + dev_err(&oct->pci_dev->dev, "error ioremap for bar %d\n", + baridx); + return 1; + } + oct->mmio[baridx].done = 1; + + return 0; +} + +static inline void * +cnnic_alloc_aligned_dma(struct pci_dev *pci_dev, + u32 size, + u32 *alloc_size, + size_t *orig_ptr, + size_t *dma_addr __attribute__((unused))) +{ + int retries = 0; + void *ptr = NULL; + +#define OCTEON_MAX_ALLOC_RETRIES 1 + do { + ptr = + (void *)__get_free_pages(GFP_KERNEL, + get_order(size)); + if ((unsigned long)ptr & 0x07) { + free_pages((unsigned long)ptr, get_order(size)); + ptr = NULL; + /* Increment the size required if the first + * attempt failed. + */ + if (!retries) + size += 7; + } + retries++; + } while ((retries <= OCTEON_MAX_ALLOC_RETRIES) && !ptr); + + *alloc_size = size; + *orig_ptr = (unsigned long)ptr; + if ((unsigned long)ptr & 0x07) + ptr = (void *)(((unsigned long)ptr + 7) & ~(7UL)); + return ptr; +} + +#define cnnic_free_aligned_dma(pci_dev, ptr, size, orig_ptr, dma_addr) \ + free_pages(orig_ptr, get_order(size)) + +static inline void +sleep_cond(wait_queue_head_t *wait_queue, int *condition) +{ + wait_queue_t we; + + init_waitqueue_entry(&we, current); + add_wait_queue(wait_queue, &we); + while (!(ACCESS_ONCE(*condition))) { + set_current_state(TASK_INTERRUPTIBLE); + if (signal_pending(current)) + goto out; + schedule(); + } +out: + set_current_state(TASK_RUNNING); + remove_wait_queue(wait_queue, &we); +} + +static inline void +sleep_atomic_cond(wait_queue_head_t *waitq, atomic_t *pcond) +{ + wait_queue_t we; + + init_waitqueue_entry(&we, current); + add_wait_queue(waitq, &we); + while (!atomic_read(pcond)) { + set_current_state(TASK_INTERRUPTIBLE); + if (signal_pending(current)) + goto out; + schedule(); + } +out: + set_current_state(TASK_RUNNING); + remove_wait_queue(waitq, &we); +} + +/* Gives up the CPU for a timeout period. + * Check that the condition is not true before we go to sleep for a + * timeout period. + */ +static inline void +sleep_timeout_cond(wait_queue_head_t *wait_queue, + int *condition, + int timeout) +{ + wait_queue_t we; + + init_waitqueue_entry(&we, current); + add_wait_queue(wait_queue, &we); + set_current_state(TASK_INTERRUPTIBLE); + if (!(*condition)) + schedule_timeout(timeout); + set_current_state(TASK_RUNNING); + remove_wait_queue(wait_queue, &we); +} + +#ifndef ROUNDUP4 +#define ROUNDUP4(val) (((val) + 3) & 0xfffffffc) +#endif + +#ifndef ROUNDUP8 +#define ROUNDUP8(val) (((val) + 7) & 0xfffffff8) +#endif + +#ifndef ROUNDUP16 +#define ROUNDUP16(val) (((val) + 15) & 0xfffffff0) +#endif + +#ifndef ROUNDUP128 +#define ROUNDUP128(val) (((val) + 127) & 0xffffff80) +#endif + +#endif /* _OCTEON_MAIN_H_ */ diff --git a/kernel/drivers/net/ethernet/cavium/liquidio/octeon_mem_ops.c b/kernel/drivers/net/ethernet/cavium/liquidio/octeon_mem_ops.c new file mode 100644 index 000000000..5aecef870 --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/liquidio/octeon_mem_ops.c @@ -0,0 +1,199 @@ +/********************************************************************** + * Author: Cavium, Inc. + * + * Contact: support@cavium.com + * Please include "LiquidIO" in the subject. + * + * Copyright (c) 2003-2015 Cavium, Inc. + * + * This file is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License, Version 2, as + * published by the Free Software Foundation. + * + * This file is distributed in the hope that it will be useful, but + * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or + * NONINFRINGEMENT. See the GNU General Public License for more + * details. + * + * This file may also be available under a different license from Cavium. + * Contact Cavium, Inc. for more information + **********************************************************************/ +#include <linux/version.h> +#include <linux/types.h> +#include <linux/list.h> +#include <linux/interrupt.h> +#include <linux/pci.h> +#include <linux/kthread.h> +#include <linux/netdevice.h> +#include "octeon_config.h" +#include "liquidio_common.h" +#include "octeon_droq.h" +#include "octeon_iq.h" +#include "response_manager.h" +#include "octeon_device.h" +#include "octeon_nic.h" +#include "octeon_main.h" +#include "octeon_network.h" +#include "cn66xx_regs.h" +#include "cn66xx_device.h" +#include "cn68xx_regs.h" +#include "cn68xx_device.h" +#include "liquidio_image.h" +#include "octeon_mem_ops.h" + +#define MEMOPS_IDX MAX_BAR1_MAP_INDEX + +static inline void +octeon_toggle_bar1_swapmode(struct octeon_device *oct __attribute__((unused)), + u32 idx __attribute__((unused))) +{ +#ifdef __BIG_ENDIAN_BITFIELD + u32 mask; + + mask = oct->fn_list.bar1_idx_read(oct, idx); + mask = (mask & 0x2) ? (mask & ~2) : (mask | 2); + oct->fn_list.bar1_idx_write(oct, idx, mask); +#endif +} + +static void +octeon_pci_fastwrite(struct octeon_device *oct, u8 __iomem *mapped_addr, + u8 *hostbuf, u32 len) +{ + while ((len) && ((unsigned long)mapped_addr) & 7) { + writeb(*(hostbuf++), mapped_addr++); + len--; + } + + octeon_toggle_bar1_swapmode(oct, MEMOPS_IDX); + + while (len >= 8) { + writeq(*((u64 *)hostbuf), mapped_addr); + mapped_addr += 8; + hostbuf += 8; + len -= 8; + } + + octeon_toggle_bar1_swapmode(oct, MEMOPS_IDX); + + while (len--) + writeb(*(hostbuf++), mapped_addr++); +} + +static void +octeon_pci_fastread(struct octeon_device *oct, u8 __iomem *mapped_addr, + u8 *hostbuf, u32 len) +{ + while ((len) && ((unsigned long)mapped_addr) & 7) { + *(hostbuf++) = readb(mapped_addr++); + len--; + } + + octeon_toggle_bar1_swapmode(oct, MEMOPS_IDX); + + while (len >= 8) { + *((u64 *)hostbuf) = readq(mapped_addr); + mapped_addr += 8; + hostbuf += 8; + len -= 8; + } + + octeon_toggle_bar1_swapmode(oct, MEMOPS_IDX); + + while (len--) + *(hostbuf++) = readb(mapped_addr++); +} + +/* Core mem read/write with temporary bar1 settings. */ +/* op = 1 to read, op = 0 to write. */ +static void +__octeon_pci_rw_core_mem(struct octeon_device *oct, u64 addr, + u8 *hostbuf, u32 len, u32 op) +{ + u32 copy_len = 0, index_reg_val = 0; + unsigned long flags; + u8 __iomem *mapped_addr; + + spin_lock_irqsave(&oct->mem_access_lock, flags); + + /* Save the original index reg value. */ + index_reg_val = oct->fn_list.bar1_idx_read(oct, MEMOPS_IDX); + do { + oct->fn_list.bar1_idx_setup(oct, addr, MEMOPS_IDX, 1); + mapped_addr = oct->mmio[1].hw_addr + + (MEMOPS_IDX << 22) + (addr & 0x3fffff); + + /* If operation crosses a 4MB boundary, split the transfer + * at the 4MB + * boundary. + */ + if (((addr + len - 1) & ~(0x3fffff)) != (addr & ~(0x3fffff))) { + copy_len = (u32)(((addr & ~(0x3fffff)) + + (MEMOPS_IDX << 22)) - addr); + } else { + copy_len = len; + } + + if (op) { /* read from core */ + octeon_pci_fastread(oct, mapped_addr, hostbuf, + copy_len); + } else { + octeon_pci_fastwrite(oct, mapped_addr, hostbuf, + copy_len); + } + + len -= copy_len; + addr += copy_len; + hostbuf += copy_len; + + } while (len); + + oct->fn_list.bar1_idx_write(oct, MEMOPS_IDX, index_reg_val); + + spin_unlock_irqrestore(&oct->mem_access_lock, flags); +} + +void +octeon_pci_read_core_mem(struct octeon_device *oct, + u64 coreaddr, + u8 *buf, + u32 len) +{ + __octeon_pci_rw_core_mem(oct, coreaddr, buf, len, 1); +} + +void +octeon_pci_write_core_mem(struct octeon_device *oct, + u64 coreaddr, + u8 *buf, + u32 len) +{ + __octeon_pci_rw_core_mem(oct, coreaddr, buf, len, 0); +} + +u64 octeon_read_device_mem64(struct octeon_device *oct, u64 coreaddr) +{ + __be64 ret; + + __octeon_pci_rw_core_mem(oct, coreaddr, (u8 *)&ret, 8, 1); + + return be64_to_cpu(ret); +} + +u32 octeon_read_device_mem32(struct octeon_device *oct, u64 coreaddr) +{ + __be32 ret; + + __octeon_pci_rw_core_mem(oct, coreaddr, (u8 *)&ret, 4, 1); + + return be32_to_cpu(ret); +} + +void octeon_write_device_mem32(struct octeon_device *oct, u64 coreaddr, + u32 val) +{ + __be32 t = cpu_to_be32(val); + + __octeon_pci_rw_core_mem(oct, coreaddr, (u8 *)&t, 4, 0); +} diff --git a/kernel/drivers/net/ethernet/cavium/liquidio/octeon_mem_ops.h b/kernel/drivers/net/ethernet/cavium/liquidio/octeon_mem_ops.h new file mode 100644 index 000000000..11b183377 --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/liquidio/octeon_mem_ops.h @@ -0,0 +1,75 @@ +/********************************************************************** + * Author: Cavium, Inc. + * + * Contact: support@cavium.com + * Please include "LiquidIO" in the subject. + * + * Copyright (c) 2003-2015 Cavium, Inc. + * + * This file is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License, Version 2, as + * published by the Free Software Foundation. + * + * This file is distributed in the hope that it will be useful, but + * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or + * NONINFRINGEMENT. See the GNU General Public License for more + * details. + * + * This file may also be available under a different license from Cavium. + * Contact Cavium, Inc. for more information + **********************************************************************/ + +/*! \file octeon_mem_ops.h + * \brief Host Driver: Routines used to read/write Octeon memory. + */ + +#ifndef __OCTEON_MEM_OPS_H__ +#define __OCTEON_MEM_OPS_H__ + +/** Read a 64-bit value from a BAR1 mapped core memory address. + * @param oct - pointer to the octeon device. + * @param core_addr - the address to read from. + * + * The range_idx gives the BAR1 index register for the range of address + * in which core_addr is mapped. + * + * @return 64-bit value read from Core memory + */ +u64 octeon_read_device_mem64(struct octeon_device *oct, u64 core_addr); + +/** Read a 32-bit value from a BAR1 mapped core memory address. + * @param oct - pointer to the octeon device. + * @param core_addr - the address to read from. + * + * @return 32-bit value read from Core memory + */ +u32 octeon_read_device_mem32(struct octeon_device *oct, u64 core_addr); + +/** Write a 32-bit value to a BAR1 mapped core memory address. + * @param oct - pointer to the octeon device. + * @param core_addr - the address to write to. + * @param val - 32-bit value to write. + */ +void +octeon_write_device_mem32(struct octeon_device *oct, + u64 core_addr, + u32 val); + +/** Read multiple bytes from Octeon memory. + */ +void +octeon_pci_read_core_mem(struct octeon_device *oct, + u64 coreaddr, + u8 *buf, + u32 len); + +/** Write multiple bytes into Octeon memory. + */ +void +octeon_pci_write_core_mem(struct octeon_device *oct, + u64 coreaddr, + u8 *buf, + u32 len); + +#endif diff --git a/kernel/drivers/net/ethernet/cavium/liquidio/octeon_network.h b/kernel/drivers/net/ethernet/cavium/liquidio/octeon_network.h new file mode 100644 index 000000000..b3abe5818 --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/liquidio/octeon_network.h @@ -0,0 +1,224 @@ +/********************************************************************** + * Author: Cavium, Inc. + * + * Contact: support@cavium.com + * Please include "LiquidIO" in the subject. + * + * Copyright (c) 2003-2015 Cavium, Inc. + * + * This file is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License, Version 2, as + * published by the Free Software Foundation. + * + * This file is distributed in the hope that it will be useful, but + * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or + * NONINFRINGEMENT. See the GNU General Public License for more + * details. + * + * This file may also be available under a different license from Cavium. + * Contact Cavium, Inc. for more information + **********************************************************************/ + +/*! \file octeon_network.h + * \brief Host NIC Driver: Structure and Macro definitions used by NIC Module. + */ + +#ifndef __OCTEON_NETWORK_H__ +#define __OCTEON_NETWORK_H__ +#include <linux/version.h> +#include <linux/dma-mapping.h> +#include <linux/ptp_clock_kernel.h> + +/** LiquidIO per-interface network private data */ +struct lio { + /** State of the interface. Rx/Tx happens only in the RUNNING state. */ + atomic_t ifstate; + + /** Octeon Interface index number. This device will be represented as + * oct<ifidx> in the system. + */ + int ifidx; + + /** Octeon Input queue to use to transmit for this network interface. */ + int txq; + + /** Octeon Output queue from which pkts arrive + * for this network interface. + */ + int rxq; + + /** Guards the glist */ + spinlock_t lock; + + /** Linked list of gather components */ + struct list_head glist; + + /** Pointer to the NIC properties for the Octeon device this network + * interface is associated with. + */ + struct octdev_props *octprops; + + /** Pointer to the octeon device structure. */ + struct octeon_device *oct_dev; + + struct net_device *netdev; + + /** Link information sent by the core application for this interface. */ + struct oct_link_info linfo; + + /** Size of Tx queue for this octeon device. */ + u32 tx_qsize; + + /** Size of Rx queue for this octeon device. */ + u32 rx_qsize; + + /** Size of MTU this octeon device. */ + u32 mtu; + + /** msg level flag per interface. */ + u32 msg_enable; + + /** Copy of Interface capabilities: TSO, TSO6, LRO, Chescksums . */ + u64 dev_capability; + + /** Copy of beacaon reg in phy */ + u32 phy_beacon_val; + + /** Copy of ctrl reg in phy */ + u32 led_ctrl_val; + + /* PTP clock information */ + struct ptp_clock_info ptp_info; + struct ptp_clock *ptp_clock; + s64 ptp_adjust; + + /* for atomic access to Octeon PTP reg and data struct */ + spinlock_t ptp_lock; + + /* Interface info */ + u32 intf_open; + + /* work queue for txq status */ + struct cavium_wq txq_status_wq; + +}; + +#define LIO_SIZE (sizeof(struct lio)) +#define GET_LIO(netdev) ((struct lio *)netdev_priv(netdev)) + +/** + * \brief Enable or disable feature + * @param netdev pointer to network device + * @param cmd Command that just requires acknowledgment + */ +int liquidio_set_feature(struct net_device *netdev, int cmd); + +/** + * \brief Link control command completion callback + * @param nctrl_ptr pointer to control packet structure + * + * This routine is called by the callback function when a ctrl pkt sent to + * core app completes. The nctrl_ptr contains a copy of the command type + * and data sent to the core app. This routine is only called if the ctrl + * pkt was sent successfully to the core app. + */ +void liquidio_link_ctrl_cmd_completion(void *nctrl_ptr); + +/** + * \brief Register ethtool operations + * @param netdev pointer to network device + */ +void liquidio_set_ethtool_ops(struct net_device *netdev); + +static inline void +*recv_buffer_alloc(struct octeon_device *oct __attribute__((unused)), + u32 q_no __attribute__((unused)), u32 size) +{ +#define SKB_ADJ_MASK 0x3F +#define SKB_ADJ (SKB_ADJ_MASK + 1) + + struct sk_buff *skb = dev_alloc_skb(size + SKB_ADJ); + + if ((unsigned long)skb->data & SKB_ADJ_MASK) { + u32 r = SKB_ADJ - ((unsigned long)skb->data & SKB_ADJ_MASK); + + skb_reserve(skb, r); + } + + return (void *)skb; +} + +static inline void recv_buffer_free(void *buffer) +{ + dev_kfree_skb_any((struct sk_buff *)buffer); +} + +#define lio_dma_alloc(oct, size, dma_addr) \ + dma_alloc_coherent(&oct->pci_dev->dev, size, dma_addr, GFP_KERNEL) +#define lio_dma_free(oct, size, virt_addr, dma_addr) \ + dma_free_coherent(&oct->pci_dev->dev, size, virt_addr, dma_addr) + +#define get_rbd(ptr) (((struct sk_buff *)(ptr))->data) + +static inline u64 +lio_map_ring_info(struct octeon_droq *droq, u32 i) +{ + dma_addr_t dma_addr; + struct octeon_device *oct = droq->oct_dev; + + dma_addr = dma_map_single(&oct->pci_dev->dev, &droq->info_list[i], + OCT_DROQ_INFO_SIZE, DMA_FROM_DEVICE); + + BUG_ON(dma_mapping_error(&oct->pci_dev->dev, dma_addr)); + + return (u64)dma_addr; +} + +static inline void +lio_unmap_ring_info(struct pci_dev *pci_dev, + u64 info_ptr, u32 size) +{ + dma_unmap_single(&pci_dev->dev, info_ptr, size, DMA_FROM_DEVICE); +} + +static inline u64 +lio_map_ring(struct pci_dev *pci_dev, + void *buf, u32 size) +{ + dma_addr_t dma_addr; + + dma_addr = dma_map_single(&pci_dev->dev, get_rbd(buf), size, + DMA_FROM_DEVICE); + + BUG_ON(dma_mapping_error(&pci_dev->dev, dma_addr)); + + return (u64)dma_addr; +} + +static inline void +lio_unmap_ring(struct pci_dev *pci_dev, + u64 buf_ptr, u32 size) +{ + dma_unmap_single(&pci_dev->dev, + buf_ptr, size, + DMA_FROM_DEVICE); +} + +static inline void *octeon_fast_packet_alloc(struct octeon_device *oct, + struct octeon_droq *droq, + u32 q_no, u32 size) +{ + return recv_buffer_alloc(oct, q_no, size); +} + +static inline void octeon_fast_packet_next(struct octeon_droq *droq, + struct sk_buff *nicbuf, + int copy_len, + int idx) +{ + memcpy(skb_put(nicbuf, copy_len), + get_rbd(droq->recv_buf_list[idx].buffer), copy_len); +} + +#endif diff --git a/kernel/drivers/net/ethernet/cavium/liquidio/octeon_nic.c b/kernel/drivers/net/ethernet/cavium/liquidio/octeon_nic.c new file mode 100644 index 000000000..1a0191549 --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/liquidio/octeon_nic.c @@ -0,0 +1,189 @@ +/********************************************************************** + * Author: Cavium, Inc. + * + * Contact: support@cavium.com + * Please include "LiquidIO" in the subject. + * + * Copyright (c) 2003-2015 Cavium, Inc. + * + * This file is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License, Version 2, as + * published by the Free Software Foundation. + * + * This file is distributed in the hope that it will be useful, but + * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or + * NONINFRINGEMENT. See the GNU General Public License for more + * details. + * + * This file may also be available under a different license from Cavium. + * Contact Cavium, Inc. for more information + **********************************************************************/ +#include <linux/version.h> +#include <linux/types.h> +#include <linux/list.h> +#include <linux/interrupt.h> +#include <linux/pci.h> +#include <linux/kthread.h> +#include <linux/netdevice.h> +#include "octeon_config.h" +#include "liquidio_common.h" +#include "octeon_droq.h" +#include "octeon_iq.h" +#include "response_manager.h" +#include "octeon_device.h" +#include "octeon_nic.h" +#include "octeon_main.h" +#include "octeon_network.h" +#include "cn66xx_regs.h" +#include "cn66xx_device.h" +#include "cn68xx_regs.h" +#include "cn68xx_device.h" +#include "liquidio_image.h" +#include "octeon_mem_ops.h" + +void * +octeon_alloc_soft_command_resp(struct octeon_device *oct, + struct octeon_instr_64B *cmd, + size_t rdatasize) +{ + struct octeon_soft_command *sc; + struct octeon_instr_ih *ih; + struct octeon_instr_irh *irh; + struct octeon_instr_rdp *rdp; + + sc = (struct octeon_soft_command *) + octeon_alloc_soft_command(oct, 0, rdatasize, 0); + + if (!sc) + return NULL; + + /* Copy existing command structure into the soft command */ + memcpy(&sc->cmd, cmd, sizeof(struct octeon_instr_64B)); + + /* Add in the response related fields. Opcode and Param are already + * there. + */ + ih = (struct octeon_instr_ih *)&sc->cmd.ih; + ih->fsz = 40; /* irh + ossp[0] + ossp[1] + rdp + rptr = 40 bytes */ + + irh = (struct octeon_instr_irh *)&sc->cmd.irh; + irh->rflag = 1; /* a response is required */ + irh->len = 4; /* means four 64-bit words immediately follow irh */ + + rdp = (struct octeon_instr_rdp *)&sc->cmd.rdp; + rdp->pcie_port = oct->pcie_port; + rdp->rlen = rdatasize; + + *sc->status_word = COMPLETION_WORD_INIT; + + sc->wait_time = 1000; + sc->timeout = jiffies + sc->wait_time; + + return sc; +} + +int octnet_send_nic_data_pkt(struct octeon_device *oct, + struct octnic_data_pkt *ndata, + u32 xmit_more) +{ + int ring_doorbell; + + ring_doorbell = !xmit_more; + + return octeon_send_command(oct, ndata->q_no, ring_doorbell, &ndata->cmd, + ndata->buf, ndata->datasize, + ndata->reqtype); +} + +static void octnet_link_ctrl_callback(struct octeon_device *oct, + u32 status, + void *sc_ptr) +{ + struct octeon_soft_command *sc = (struct octeon_soft_command *)sc_ptr; + struct octnic_ctrl_pkt *nctrl; + + nctrl = (struct octnic_ctrl_pkt *)sc->ctxptr; + + /* Call the callback function if status is OK. + * Status is OK only if a response was expected and core returned + * success. + * If no response was expected, status is OK if the command was posted + * successfully. + */ + if (!status && nctrl->cb_fn) + nctrl->cb_fn(nctrl); + + octeon_free_soft_command(oct, sc); +} + +static inline struct octeon_soft_command +*octnic_alloc_ctrl_pkt_sc(struct octeon_device *oct, + struct octnic_ctrl_pkt *nctrl, + struct octnic_ctrl_params nparams) +{ + struct octeon_soft_command *sc = NULL; + u8 *data; + size_t rdatasize; + u32 uddsize = 0, datasize = 0; + + uddsize = (u32)(nctrl->ncmd.s.more * 8); + + datasize = OCTNET_CMD_SIZE + uddsize; + rdatasize = (nctrl->wait_time) ? 16 : 0; + + sc = (struct octeon_soft_command *) + octeon_alloc_soft_command(oct, datasize, rdatasize, + sizeof(struct octnic_ctrl_pkt)); + + if (!sc) + return NULL; + + memcpy(sc->ctxptr, nctrl, sizeof(struct octnic_ctrl_pkt)); + + data = (u8 *)sc->virtdptr; + + memcpy(data, &nctrl->ncmd, OCTNET_CMD_SIZE); + + octeon_swap_8B_data((u64 *)data, (OCTNET_CMD_SIZE >> 3)); + + if (uddsize) { + /* Endian-Swap for UDD should have been done by caller. */ + memcpy(data + OCTNET_CMD_SIZE, nctrl->udd, uddsize); + } + + octeon_prepare_soft_command(oct, sc, OPCODE_NIC, OPCODE_NIC_CMD, + 0, 0, 0); + + sc->callback = octnet_link_ctrl_callback; + sc->callback_arg = sc; + sc->wait_time = nctrl->wait_time; + + return sc; +} + +int +octnet_send_nic_ctrl_pkt(struct octeon_device *oct, + struct octnic_ctrl_pkt *nctrl, + struct octnic_ctrl_params nparams) +{ + int retval; + struct octeon_soft_command *sc = NULL; + + sc = octnic_alloc_ctrl_pkt_sc(oct, nctrl, nparams); + if (!sc) { + dev_err(&oct->pci_dev->dev, "%s soft command alloc failed\n", + __func__); + return -1; + } + + retval = octeon_send_soft_command(oct, sc); + if (retval) { + octeon_free_soft_command(oct, sc); + dev_err(&oct->pci_dev->dev, "%s soft command send failed status: %x\n", + __func__, retval); + return -1; + } + + return retval; +} diff --git a/kernel/drivers/net/ethernet/cavium/liquidio/octeon_nic.h b/kernel/drivers/net/ethernet/cavium/liquidio/octeon_nic.h new file mode 100644 index 000000000..0238857c8 --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/liquidio/octeon_nic.h @@ -0,0 +1,227 @@ +/********************************************************************** + * Author: Cavium, Inc. + * + * Contact: support@cavium.com + * Please include "LiquidIO" in the subject. + * + * Copyright (c) 2003-2015 Cavium, Inc. + * + * This file is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License, Version 2, as + * published by the Free Software Foundation. + * + * This file is distributed in the hope that it will be useful, but + * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or + * NONINFRINGEMENT. See the GNU General Public License for more + * details. + * + * This file may also be available under a different license from Cavium. + * Contact Cavium, Inc. for more information + **********************************************************************/ + +/*! \file octeon_nic.h + * \brief Host NIC Driver: Routine to send network data & + * control packet to Octeon. + */ + +#ifndef __OCTEON_NIC_H__ +#define __OCTEON_NIC_H__ + +/* Maximum number of 8-byte words can be sent in a NIC control message. + */ +#define MAX_NCTRL_UDD 32 + +typedef void (*octnic_ctrl_pkt_cb_fn_t) (void *); + +/* Structure of control information passed by the NIC module to the OSI + * layer when sending control commands to Octeon device software. + */ +struct octnic_ctrl_pkt { + /** Command to be passed to the Octeon device software. */ + union octnet_cmd ncmd; + + /** Send buffer */ + void *data; + u64 dmadata; + + /** Response buffer */ + void *rdata; + u64 dmardata; + + /** Additional data that may be needed by some commands. */ + u64 udd[MAX_NCTRL_UDD]; + + /** Time to wait for Octeon software to respond to this control command. + * If wait_time is 0, OSI assumes no response is expected. + */ + size_t wait_time; + + /** The network device that issued the control command. */ + u64 netpndev; + + /** Callback function called when the command has been fetched */ + octnic_ctrl_pkt_cb_fn_t cb_fn; +}; + +#define MAX_UDD_SIZE(nctrl) (sizeof(nctrl->udd)) + +/** Structure of data information passed by the NIC module to the OSI + * layer when forwarding data to Octeon device software. + */ +struct octnic_data_pkt { + /** Pointer to information maintained by NIC module for this packet. The + * OSI layer passes this as-is to the driver. + */ + void *buf; + + /** Type of buffer passed in "buf" above. */ + u32 reqtype; + + /** Total data bytes to be transferred in this command. */ + u32 datasize; + + /** Command to be passed to the Octeon device software. */ + struct octeon_instr_64B cmd; + + /** Input queue to use to send this command. */ + u32 q_no; + +}; + +/** Structure passed by NIC module to OSI layer to prepare a command to send + * network data to Octeon. + */ +union octnic_cmd_setup { + struct { + u32 ifidx:8; + u32 cksum_offset:7; + u32 gather:1; + u32 timestamp:1; + u32 ipv4opts_ipv6exthdr:2; + u32 ip_csum:1; + u32 tnl_csum:1; + + u32 rsvd:11; + union { + u32 datasize; + u32 gatherptrs; + } u; + } s; + + u64 u64; + +}; + +struct octnic_ctrl_params { + u32 resp_order; +}; + +static inline int octnet_iq_is_full(struct octeon_device *oct, u32 q_no) +{ + return ((u32)atomic_read(&oct->instr_queue[q_no]->instr_pending) + >= (oct->instr_queue[q_no]->max_count - 2)); +} + +/** Utility function to prepare a 64B NIC instruction based on a setup command + * @param cmd - pointer to instruction to be filled in. + * @param setup - pointer to the setup structure + * @param q_no - which queue for back pressure + * + * Assumes the cmd instruction is pre-allocated, but no fields are filled in. + */ +static inline void +octnet_prepare_pci_cmd(struct octeon_instr_64B *cmd, + union octnic_cmd_setup *setup, u32 tag) +{ + struct octeon_instr_ih *ih; + struct octeon_instr_irh *irh; + union octnic_packet_params packet_params; + + memset(cmd, 0, sizeof(struct octeon_instr_64B)); + + ih = (struct octeon_instr_ih *)&cmd->ih; + + /* assume that rflag is cleared so therefore front data will only have + * irh and ossp[1] and ossp[2] for a total of 24 bytes + */ + ih->fsz = 24; + + ih->tagtype = ORDERED_TAG; + ih->grp = DEFAULT_POW_GRP; + + if (tag) + ih->tag = tag; + else + ih->tag = LIO_DATA(setup->s.ifidx); + + ih->raw = 1; + ih->qos = (setup->s.ifidx & 3) + 4; /* map qos based on interface */ + + if (!setup->s.gather) { + ih->dlengsz = setup->s.u.datasize; + } else { + ih->gather = 1; + ih->dlengsz = setup->s.u.gatherptrs; + } + + irh = (struct octeon_instr_irh *)&cmd->irh; + + irh->opcode = OPCODE_NIC; + irh->subcode = OPCODE_NIC_NW_DATA; + + packet_params.u32 = 0; + + if (setup->s.cksum_offset) { + packet_params.s.csoffset = setup->s.cksum_offset; + packet_params.s.ipv4opts_ipv6exthdr = + setup->s.ipv4opts_ipv6exthdr; + } + + packet_params.s.ip_csum = setup->s.ip_csum; + packet_params.s.tnl_csum = setup->s.tnl_csum; + packet_params.s.ifidx = setup->s.ifidx; + packet_params.s.tsflag = setup->s.timestamp; + + irh->ossp = packet_params.u32; +} + +/** Allocate and a soft command with space for a response immediately following + * the commnad. + * @param oct - octeon device pointer + * @param cmd - pointer to the command structure, pre-filled for everything + * except the response. + * @param rdatasize - size in bytes of the response. + * + * @returns pointer to allocated buffer with command copied into it, and + * response space immediately following. + */ +void * +octeon_alloc_soft_command_resp(struct octeon_device *oct, + struct octeon_instr_64B *cmd, + size_t rdatasize); + +/** Send a NIC data packet to the device + * @param oct - octeon device pointer + * @param ndata - control structure with queueing, and buffer information + * + * @returns IQ_FAILED if it failed to add to the input queue. IQ_STOP if it the + * queue should be stopped, and IQ_SEND_OK if it sent okay. + */ +int octnet_send_nic_data_pkt(struct octeon_device *oct, + struct octnic_data_pkt *ndata, u32 xmit_more); + +/** Send a NIC control packet to the device + * @param oct - octeon device pointer + * @param nctrl - control structure with command, timout, and callback info + * @param nparams - response control structure + * + * @returns IQ_FAILED if it failed to add to the input queue. IQ_STOP if it the + * queue should be stopped, and IQ_SEND_OK if it sent okay. + */ +int +octnet_send_nic_ctrl_pkt(struct octeon_device *oct, + struct octnic_ctrl_pkt *nctrl, + struct octnic_ctrl_params nparams); + +#endif diff --git a/kernel/drivers/net/ethernet/cavium/liquidio/request_manager.c b/kernel/drivers/net/ethernet/cavium/liquidio/request_manager.c new file mode 100644 index 000000000..a2a24652c --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/liquidio/request_manager.c @@ -0,0 +1,765 @@ +/********************************************************************** + * Author: Cavium, Inc. + * + * Contact: support@cavium.com + * Please include "LiquidIO" in the subject. + * + * Copyright (c) 2003-2015 Cavium, Inc. + * + * This file is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License, Version 2, as + * published by the Free Software Foundation. + * + * This file is distributed in the hope that it will be useful, but + * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or + * NONINFRINGEMENT. See the GNU General Public License for more + * details. + * + * This file may also be available under a different license from Cavium. + * Contact Cavium, Inc. for more information + **********************************************************************/ +#include <linux/version.h> +#include <linux/types.h> +#include <linux/list.h> +#include <linux/interrupt.h> +#include <linux/pci.h> +#include <linux/kthread.h> +#include <linux/netdevice.h> +#include <linux/vmalloc.h> +#include "octeon_config.h" +#include "liquidio_common.h" +#include "octeon_droq.h" +#include "octeon_iq.h" +#include "response_manager.h" +#include "octeon_device.h" +#include "octeon_nic.h" +#include "octeon_main.h" +#include "octeon_network.h" +#include "cn66xx_regs.h" +#include "cn66xx_device.h" +#include "cn68xx_regs.h" +#include "cn68xx_device.h" +#include "liquidio_image.h" + +#define INCR_INSTRQUEUE_PKT_COUNT(octeon_dev_ptr, iq_no, field, count) \ + (octeon_dev_ptr->instr_queue[iq_no]->stats.field += count) + +struct iq_post_status { + int status; + int index; +}; + +static void check_db_timeout(struct work_struct *work); +static void __check_db_timeout(struct octeon_device *oct, unsigned long iq_no); + +static void (*reqtype_free_fn[MAX_OCTEON_DEVICES][REQTYPE_LAST + 1]) (void *); + +static inline int IQ_INSTR_MODE_64B(struct octeon_device *oct, int iq_no) +{ + struct octeon_instr_queue *iq = + (struct octeon_instr_queue *)oct->instr_queue[iq_no]; + return iq->iqcmd_64B; +} + +#define IQ_INSTR_MODE_32B(oct, iq_no) (!IQ_INSTR_MODE_64B(oct, iq_no)) + +/* Define this to return the request status comaptible to old code */ +/*#define OCTEON_USE_OLD_REQ_STATUS*/ + +/* Return 0 on success, 1 on failure */ +int octeon_init_instr_queue(struct octeon_device *oct, + u32 iq_no, u32 num_descs) +{ + struct octeon_instr_queue *iq; + struct octeon_iq_config *conf = NULL; + u32 q_size; + struct cavium_wq *db_wq; + + if (OCTEON_CN6XXX(oct)) + conf = &(CFG_GET_IQ_CFG(CHIP_FIELD(oct, cn6xxx, conf))); + + if (!conf) { + dev_err(&oct->pci_dev->dev, "Unsupported Chip %x\n", + oct->chip_id); + return 1; + } + + if (num_descs & (num_descs - 1)) { + dev_err(&oct->pci_dev->dev, + "Number of descriptors for instr queue %d not in power of 2.\n", + iq_no); + return 1; + } + + q_size = (u32)conf->instr_type * num_descs; + + iq = oct->instr_queue[iq_no]; + + iq->base_addr = lio_dma_alloc(oct, q_size, + (dma_addr_t *)&iq->base_addr_dma); + if (!iq->base_addr) { + dev_err(&oct->pci_dev->dev, "Cannot allocate memory for instr queue %d\n", + iq_no); + return 1; + } + + iq->max_count = num_descs; + + /* Initialize a list to holds requests that have been posted to Octeon + * but has yet to be fetched by octeon + */ + iq->request_list = vmalloc(sizeof(*iq->request_list) * num_descs); + if (!iq->request_list) { + lio_dma_free(oct, q_size, iq->base_addr, iq->base_addr_dma); + dev_err(&oct->pci_dev->dev, "Alloc failed for IQ[%d] nr free list\n", + iq_no); + return 1; + } + + memset(iq->request_list, 0, sizeof(*iq->request_list) * num_descs); + + dev_dbg(&oct->pci_dev->dev, "IQ[%d]: base: %p basedma: %llx count: %d\n", + iq_no, iq->base_addr, iq->base_addr_dma, iq->max_count); + + iq->iq_no = iq_no; + iq->fill_threshold = (u32)conf->db_min; + iq->fill_cnt = 0; + iq->host_write_index = 0; + iq->octeon_read_index = 0; + iq->flush_index = 0; + iq->last_db_time = 0; + iq->do_auto_flush = 1; + iq->db_timeout = (u32)conf->db_timeout; + atomic_set(&iq->instr_pending, 0); + + /* Initialize the spinlock for this instruction queue */ + spin_lock_init(&iq->lock); + + oct->io_qmask.iq |= (1 << iq_no); + + /* Set the 32B/64B mode for each input queue */ + oct->io_qmask.iq64B |= ((conf->instr_type == 64) << iq_no); + iq->iqcmd_64B = (conf->instr_type == 64); + + oct->fn_list.setup_iq_regs(oct, iq_no); + + oct->check_db_wq[iq_no].wq = create_workqueue("check_iq_db"); + if (!oct->check_db_wq[iq_no].wq) { + lio_dma_free(oct, q_size, iq->base_addr, iq->base_addr_dma); + dev_err(&oct->pci_dev->dev, "check db wq create failed for iq %d\n", + iq_no); + return 1; + } + + db_wq = &oct->check_db_wq[iq_no]; + + INIT_DELAYED_WORK(&db_wq->wk.work, check_db_timeout); + db_wq->wk.ctxptr = oct; + db_wq->wk.ctxul = iq_no; + queue_delayed_work(db_wq->wq, &db_wq->wk.work, msecs_to_jiffies(1)); + + return 0; +} + +int octeon_delete_instr_queue(struct octeon_device *oct, u32 iq_no) +{ + u64 desc_size = 0, q_size; + struct octeon_instr_queue *iq = oct->instr_queue[iq_no]; + + cancel_delayed_work_sync(&oct->check_db_wq[iq_no].wk.work); + flush_workqueue(oct->check_db_wq[iq_no].wq); + destroy_workqueue(oct->check_db_wq[iq_no].wq); + + if (OCTEON_CN6XXX(oct)) + desc_size = + CFG_GET_IQ_INSTR_TYPE(CHIP_FIELD(oct, cn6xxx, conf)); + + vfree(iq->request_list); + + if (iq->base_addr) { + q_size = iq->max_count * desc_size; + lio_dma_free(oct, (u32)q_size, iq->base_addr, + iq->base_addr_dma); + return 0; + } + return 1; +} + +/* Return 0 on success, 1 on failure */ +int octeon_setup_iq(struct octeon_device *oct, + u32 iq_no, + u32 num_descs, + void *app_ctx) +{ + if (oct->instr_queue[iq_no]) { + dev_dbg(&oct->pci_dev->dev, "IQ is in use. Cannot create the IQ: %d again\n", + iq_no); + oct->instr_queue[iq_no]->app_ctx = app_ctx; + return 0; + } + oct->instr_queue[iq_no] = + vmalloc(sizeof(struct octeon_instr_queue)); + if (!oct->instr_queue[iq_no]) + return 1; + + memset(oct->instr_queue[iq_no], 0, + sizeof(struct octeon_instr_queue)); + + oct->instr_queue[iq_no]->app_ctx = app_ctx; + if (octeon_init_instr_queue(oct, iq_no, num_descs)) { + vfree(oct->instr_queue[iq_no]); + oct->instr_queue[iq_no] = NULL; + return 1; + } + + oct->num_iqs++; + oct->fn_list.enable_io_queues(oct); + return 0; +} + +int lio_wait_for_instr_fetch(struct octeon_device *oct) +{ + int i, retry = 1000, pending, instr_cnt = 0; + + do { + instr_cnt = 0; + + /*for (i = 0; i < oct->num_iqs; i++) {*/ + for (i = 0; i < MAX_OCTEON_INSTR_QUEUES; i++) { + if (!(oct->io_qmask.iq & (1UL << i))) + continue; + pending = + atomic_read(&oct-> + instr_queue[i]->instr_pending); + if (pending) + __check_db_timeout(oct, i); + instr_cnt += pending; + } + + if (instr_cnt == 0) + break; + + schedule_timeout_uninterruptible(1); + + } while (retry-- && instr_cnt); + + return instr_cnt; +} + +static inline void +ring_doorbell(struct octeon_device *oct, struct octeon_instr_queue *iq) +{ + if (atomic_read(&oct->status) == OCT_DEV_RUNNING) { + writel(iq->fill_cnt, iq->doorbell_reg); + /* make sure doorbell write goes through */ + mmiowb(); + iq->fill_cnt = 0; + iq->last_db_time = jiffies; + return; + } +} + +static inline void __copy_cmd_into_iq(struct octeon_instr_queue *iq, + u8 *cmd) +{ + u8 *iqptr, cmdsize; + + cmdsize = ((iq->iqcmd_64B) ? 64 : 32); + iqptr = iq->base_addr + (cmdsize * iq->host_write_index); + + memcpy(iqptr, cmd, cmdsize); +} + +static inline int +__post_command(struct octeon_device *octeon_dev __attribute__((unused)), + struct octeon_instr_queue *iq, + u32 force_db __attribute__((unused)), u8 *cmd) +{ + u32 index = -1; + + /* This ensures that the read index does not wrap around to the same + * position if queue gets full before Octeon could fetch any instr. + */ + if (atomic_read(&iq->instr_pending) >= (s32)(iq->max_count - 1)) + return -1; + + __copy_cmd_into_iq(iq, cmd); + + /* "index" is returned, host_write_index is modified. */ + index = iq->host_write_index; + INCR_INDEX_BY1(iq->host_write_index, iq->max_count); + iq->fill_cnt++; + + /* Flush the command into memory. We need to be sure the data is in + * memory before indicating that the instruction is pending. + */ + wmb(); + + atomic_inc(&iq->instr_pending); + + return index; +} + +static inline struct iq_post_status +__post_command2(struct octeon_device *octeon_dev __attribute__((unused)), + struct octeon_instr_queue *iq, + u32 force_db __attribute__((unused)), u8 *cmd) +{ + struct iq_post_status st; + + st.status = IQ_SEND_OK; + + /* This ensures that the read index does not wrap around to the same + * position if queue gets full before Octeon could fetch any instr. + */ + if (atomic_read(&iq->instr_pending) >= (s32)(iq->max_count - 1)) { + st.status = IQ_SEND_FAILED; + st.index = -1; + return st; + } + + if (atomic_read(&iq->instr_pending) >= (s32)(iq->max_count - 2)) + st.status = IQ_SEND_STOP; + + __copy_cmd_into_iq(iq, cmd); + + /* "index" is returned, host_write_index is modified. */ + st.index = iq->host_write_index; + INCR_INDEX_BY1(iq->host_write_index, iq->max_count); + iq->fill_cnt++; + + /* Flush the command into memory. We need to be sure the data is in + * memory before indicating that the instruction is pending. + */ + wmb(); + + atomic_inc(&iq->instr_pending); + + return st; +} + +int +octeon_register_reqtype_free_fn(struct octeon_device *oct, int reqtype, + void (*fn)(void *)) +{ + if (reqtype > REQTYPE_LAST) { + dev_err(&oct->pci_dev->dev, "%s: Invalid reqtype: %d\n", + __func__, reqtype); + return -EINVAL; + } + + reqtype_free_fn[oct->octeon_id][reqtype] = fn; + + return 0; +} + +static inline void +__add_to_request_list(struct octeon_instr_queue *iq, + int idx, void *buf, int reqtype) +{ + iq->request_list[idx].buf = buf; + iq->request_list[idx].reqtype = reqtype; +} + +int +lio_process_iq_request_list(struct octeon_device *oct, + struct octeon_instr_queue *iq) +{ + int reqtype; + void *buf; + u32 old = iq->flush_index; + u32 inst_count = 0; + unsigned pkts_compl = 0, bytes_compl = 0; + struct octeon_soft_command *sc; + struct octeon_instr_irh *irh; + + while (old != iq->octeon_read_index) { + reqtype = iq->request_list[old].reqtype; + buf = iq->request_list[old].buf; + + if (reqtype == REQTYPE_NONE) + goto skip_this; + + octeon_update_tx_completion_counters(buf, reqtype, &pkts_compl, + &bytes_compl); + + switch (reqtype) { + case REQTYPE_NORESP_NET: + case REQTYPE_NORESP_NET_SG: + case REQTYPE_RESP_NET_SG: + reqtype_free_fn[oct->octeon_id][reqtype](buf); + break; + case REQTYPE_RESP_NET: + case REQTYPE_SOFT_COMMAND: + sc = buf; + + irh = (struct octeon_instr_irh *)&sc->cmd.irh; + if (irh->rflag) { + /* We're expecting a response from Octeon. + * It's up to lio_process_ordered_list() to + * process sc. Add sc to the ordered soft + * command response list because we expect + * a response from Octeon. + */ + spin_lock_bh(&oct->response_list + [OCTEON_ORDERED_SC_LIST].lock); + atomic_inc(&oct->response_list + [OCTEON_ORDERED_SC_LIST]. + pending_req_count); + list_add_tail(&sc->node, &oct->response_list + [OCTEON_ORDERED_SC_LIST].head); + spin_unlock_bh(&oct->response_list + [OCTEON_ORDERED_SC_LIST].lock); + } else { + if (sc->callback) { + sc->callback(oct, OCTEON_REQUEST_DONE, + sc->callback_arg); + } + } + break; + default: + dev_err(&oct->pci_dev->dev, + "%s Unknown reqtype: %d buf: %p at idx %d\n", + __func__, reqtype, buf, old); + } + + iq->request_list[old].buf = NULL; + iq->request_list[old].reqtype = 0; + + skip_this: + inst_count++; + INCR_INDEX_BY1(old, iq->max_count); + } + if (bytes_compl) + octeon_report_tx_completion_to_bql(iq->app_ctx, pkts_compl, + bytes_compl); + iq->flush_index = old; + + return inst_count; +} + +static inline void +update_iq_indices(struct octeon_device *oct, struct octeon_instr_queue *iq) +{ + u32 inst_processed = 0; + + /* Calculate how many commands Octeon has read and move the read index + * accordingly. + */ + iq->octeon_read_index = oct->fn_list.update_iq_read_idx(oct, iq); + + /* Move the NORESPONSE requests to the per-device completion list. */ + if (iq->flush_index != iq->octeon_read_index) + inst_processed = lio_process_iq_request_list(oct, iq); + + if (inst_processed) { + atomic_sub(inst_processed, &iq->instr_pending); + iq->stats.instr_processed += inst_processed; + } +} + +static void +octeon_flush_iq(struct octeon_device *oct, struct octeon_instr_queue *iq, + u32 pending_thresh) +{ + if (atomic_read(&iq->instr_pending) >= (s32)pending_thresh) { + spin_lock_bh(&iq->lock); + update_iq_indices(oct, iq); + spin_unlock_bh(&iq->lock); + } +} + +static void __check_db_timeout(struct octeon_device *oct, unsigned long iq_no) +{ + struct octeon_instr_queue *iq; + u64 next_time; + + if (!oct) + return; + iq = oct->instr_queue[iq_no]; + if (!iq) + return; + + /* If jiffies - last_db_time < db_timeout do nothing */ + next_time = iq->last_db_time + iq->db_timeout; + if (!time_after(jiffies, (unsigned long)next_time)) + return; + iq->last_db_time = jiffies; + + /* Get the lock and prevent tasklets. This routine gets called from + * the poll thread. Instructions can now be posted in tasklet context + */ + spin_lock_bh(&iq->lock); + if (iq->fill_cnt != 0) + ring_doorbell(oct, iq); + + spin_unlock_bh(&iq->lock); + + /* Flush the instruction queue */ + if (iq->do_auto_flush) + octeon_flush_iq(oct, iq, 1); +} + +/* Called by the Poll thread at regular intervals to check the instruction + * queue for commands to be posted and for commands that were fetched by Octeon. + */ +static void check_db_timeout(struct work_struct *work) +{ + struct cavium_wk *wk = (struct cavium_wk *)work; + struct octeon_device *oct = (struct octeon_device *)wk->ctxptr; + unsigned long iq_no = wk->ctxul; + struct cavium_wq *db_wq = &oct->check_db_wq[iq_no]; + + __check_db_timeout(oct, iq_no); + queue_delayed_work(db_wq->wq, &db_wq->wk.work, msecs_to_jiffies(1)); +} + +int +octeon_send_command(struct octeon_device *oct, u32 iq_no, + u32 force_db, void *cmd, void *buf, + u32 datasize, u32 reqtype) +{ + struct iq_post_status st; + struct octeon_instr_queue *iq = oct->instr_queue[iq_no]; + + spin_lock_bh(&iq->lock); + + st = __post_command2(oct, iq, force_db, cmd); + + if (st.status != IQ_SEND_FAILED) { + octeon_report_sent_bytes_to_bql(buf, reqtype); + __add_to_request_list(iq, st.index, buf, reqtype); + INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, bytes_sent, datasize); + INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, instr_posted, 1); + + if (iq->fill_cnt >= iq->fill_threshold || force_db) + ring_doorbell(oct, iq); + } else { + INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, instr_dropped, 1); + } + + spin_unlock_bh(&iq->lock); + + if (iq->do_auto_flush) + octeon_flush_iq(oct, iq, 2); + + return st.status; +} + +void +octeon_prepare_soft_command(struct octeon_device *oct, + struct octeon_soft_command *sc, + u8 opcode, + u8 subcode, + u32 irh_ossp, + u64 ossp0, + u64 ossp1) +{ + struct octeon_config *oct_cfg; + struct octeon_instr_ih *ih; + struct octeon_instr_irh *irh; + struct octeon_instr_rdp *rdp; + + BUG_ON(opcode > 15); + BUG_ON(subcode > 127); + + oct_cfg = octeon_get_conf(oct); + + ih = (struct octeon_instr_ih *)&sc->cmd.ih; + ih->tagtype = ATOMIC_TAG; + ih->tag = LIO_CONTROL; + ih->raw = 1; + ih->grp = CFG_GET_CTRL_Q_GRP(oct_cfg); + + if (sc->datasize) { + ih->dlengsz = sc->datasize; + ih->rs = 1; + } + + irh = (struct octeon_instr_irh *)&sc->cmd.irh; + irh->opcode = opcode; + irh->subcode = subcode; + + /* opcode/subcode specific parameters (ossp) */ + irh->ossp = irh_ossp; + sc->cmd.ossp[0] = ossp0; + sc->cmd.ossp[1] = ossp1; + + if (sc->rdatasize) { + rdp = (struct octeon_instr_rdp *)&sc->cmd.rdp; + rdp->pcie_port = oct->pcie_port; + rdp->rlen = sc->rdatasize; + + irh->rflag = 1; + irh->len = 4; + ih->fsz = 40; /* irh+ossp[0]+ossp[1]+rdp+rptr = 40 bytes */ + } else { + irh->rflag = 0; + irh->len = 2; + ih->fsz = 24; /* irh + ossp[0] + ossp[1] = 24 bytes */ + } + + while (!(oct->io_qmask.iq & (1 << sc->iq_no))) + sc->iq_no++; +} + +int octeon_send_soft_command(struct octeon_device *oct, + struct octeon_soft_command *sc) +{ + struct octeon_instr_ih *ih; + struct octeon_instr_irh *irh; + struct octeon_instr_rdp *rdp; + + ih = (struct octeon_instr_ih *)&sc->cmd.ih; + if (ih->dlengsz) { + BUG_ON(!sc->dmadptr); + sc->cmd.dptr = sc->dmadptr; + } + + irh = (struct octeon_instr_irh *)&sc->cmd.irh; + if (irh->rflag) { + BUG_ON(!sc->dmarptr); + BUG_ON(!sc->status_word); + *sc->status_word = COMPLETION_WORD_INIT; + + rdp = (struct octeon_instr_rdp *)&sc->cmd.rdp; + + sc->cmd.rptr = sc->dmarptr; + } + + if (sc->wait_time) + sc->timeout = jiffies + sc->wait_time; + + return octeon_send_command(oct, sc->iq_no, 1, &sc->cmd, sc, + (u32)ih->dlengsz, REQTYPE_SOFT_COMMAND); +} + +int octeon_setup_sc_buffer_pool(struct octeon_device *oct) +{ + int i; + u64 dma_addr; + struct octeon_soft_command *sc; + + INIT_LIST_HEAD(&oct->sc_buf_pool.head); + spin_lock_init(&oct->sc_buf_pool.lock); + atomic_set(&oct->sc_buf_pool.alloc_buf_count, 0); + + for (i = 0; i < MAX_SOFT_COMMAND_BUFFERS; i++) { + sc = (struct octeon_soft_command *) + lio_dma_alloc(oct, + SOFT_COMMAND_BUFFER_SIZE, + (dma_addr_t *)&dma_addr); + if (!sc) + return 1; + + sc->dma_addr = dma_addr; + sc->size = SOFT_COMMAND_BUFFER_SIZE; + + list_add_tail(&sc->node, &oct->sc_buf_pool.head); + } + + return 0; +} + +int octeon_free_sc_buffer_pool(struct octeon_device *oct) +{ + struct list_head *tmp, *tmp2; + struct octeon_soft_command *sc; + + spin_lock(&oct->sc_buf_pool.lock); + + list_for_each_safe(tmp, tmp2, &oct->sc_buf_pool.head) { + list_del(tmp); + + sc = (struct octeon_soft_command *)tmp; + + lio_dma_free(oct, sc->size, sc, sc->dma_addr); + } + + INIT_LIST_HEAD(&oct->sc_buf_pool.head); + + spin_unlock(&oct->sc_buf_pool.lock); + + return 0; +} + +struct octeon_soft_command *octeon_alloc_soft_command(struct octeon_device *oct, + u32 datasize, + u32 rdatasize, + u32 ctxsize) +{ + u64 dma_addr; + u32 size; + u32 offset = sizeof(struct octeon_soft_command); + struct octeon_soft_command *sc = NULL; + struct list_head *tmp; + + BUG_ON((offset + datasize + rdatasize + ctxsize) > + SOFT_COMMAND_BUFFER_SIZE); + + spin_lock(&oct->sc_buf_pool.lock); + + if (list_empty(&oct->sc_buf_pool.head)) { + spin_unlock(&oct->sc_buf_pool.lock); + return NULL; + } + + list_for_each(tmp, &oct->sc_buf_pool.head) + break; + + list_del(tmp); + + atomic_inc(&oct->sc_buf_pool.alloc_buf_count); + + spin_unlock(&oct->sc_buf_pool.lock); + + sc = (struct octeon_soft_command *)tmp; + + dma_addr = sc->dma_addr; + size = sc->size; + + memset(sc, 0, sc->size); + + sc->dma_addr = dma_addr; + sc->size = size; + + if (ctxsize) { + sc->ctxptr = (u8 *)sc + offset; + sc->ctxsize = ctxsize; + } + + /* Start data at 128 byte boundary */ + offset = (offset + ctxsize + 127) & 0xffffff80; + + if (datasize) { + sc->virtdptr = (u8 *)sc + offset; + sc->dmadptr = dma_addr + offset; + sc->datasize = datasize; + } + + /* Start rdata at 128 byte boundary */ + offset = (offset + datasize + 127) & 0xffffff80; + + if (rdatasize) { + BUG_ON(rdatasize < 16); + sc->virtrptr = (u8 *)sc + offset; + sc->dmarptr = dma_addr + offset; + sc->rdatasize = rdatasize; + sc->status_word = (u64 *)((u8 *)(sc->virtrptr) + rdatasize - 8); + } + + return sc; +} + +void octeon_free_soft_command(struct octeon_device *oct, + struct octeon_soft_command *sc) +{ + spin_lock(&oct->sc_buf_pool.lock); + + list_add_tail(&sc->node, &oct->sc_buf_pool.head); + + atomic_dec(&oct->sc_buf_pool.alloc_buf_count); + + spin_unlock(&oct->sc_buf_pool.lock); +} diff --git a/kernel/drivers/net/ethernet/cavium/liquidio/response_manager.c b/kernel/drivers/net/ethernet/cavium/liquidio/response_manager.c new file mode 100644 index 000000000..091f537a9 --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/liquidio/response_manager.c @@ -0,0 +1,178 @@ +/********************************************************************** + * Author: Cavium, Inc. + * + * Contact: support@cavium.com + * Please include "LiquidIO" in the subject. + * + * Copyright (c) 2003-2015 Cavium, Inc. + * + * This file is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License, Version 2, as + * published by the Free Software Foundation. + * + * This file is distributed in the hope that it will be useful, but + * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or + * NONINFRINGEMENT. See the GNU General Public License for more + * details. + * + * This file may also be available under a different license from Cavium. + * Contact Cavium, Inc. for more information + **********************************************************************/ +#include <linux/version.h> +#include <linux/types.h> +#include <linux/list.h> +#include <linux/interrupt.h> +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include <linux/kthread.h> +#include <linux/netdevice.h> +#include "octeon_config.h" +#include "liquidio_common.h" +#include "octeon_droq.h" +#include "octeon_iq.h" +#include "response_manager.h" +#include "octeon_device.h" +#include "octeon_nic.h" +#include "octeon_main.h" +#include "octeon_network.h" +#include "cn66xx_regs.h" +#include "cn66xx_device.h" +#include "cn68xx_regs.h" +#include "cn68xx_device.h" +#include "liquidio_image.h" + +static void oct_poll_req_completion(struct work_struct *work); + +int octeon_setup_response_list(struct octeon_device *oct) +{ + int i, ret = 0; + struct cavium_wq *cwq; + + for (i = 0; i < MAX_RESPONSE_LISTS; i++) { + INIT_LIST_HEAD(&oct->response_list[i].head); + spin_lock_init(&oct->response_list[i].lock); + atomic_set(&oct->response_list[i].pending_req_count, 0); + } + + oct->dma_comp_wq.wq = create_workqueue("dma-comp"); + if (!oct->dma_comp_wq.wq) { + dev_err(&oct->pci_dev->dev, "failed to create wq thread\n"); + return -ENOMEM; + } + + cwq = &oct->dma_comp_wq; + INIT_DELAYED_WORK(&cwq->wk.work, oct_poll_req_completion); + cwq->wk.ctxptr = oct; + queue_delayed_work(cwq->wq, &cwq->wk.work, msecs_to_jiffies(100)); + + return ret; +} + +void octeon_delete_response_list(struct octeon_device *oct) +{ + cancel_delayed_work_sync(&oct->dma_comp_wq.wk.work); + flush_workqueue(oct->dma_comp_wq.wq); + destroy_workqueue(oct->dma_comp_wq.wq); +} + +int lio_process_ordered_list(struct octeon_device *octeon_dev, + u32 force_quit) +{ + struct octeon_response_list *ordered_sc_list; + struct octeon_soft_command *sc; + int request_complete = 0; + int resp_to_process = MAX_ORD_REQS_TO_PROCESS; + u32 status; + u64 status64; + struct octeon_instr_rdp *rdp; + + ordered_sc_list = &octeon_dev->response_list[OCTEON_ORDERED_SC_LIST]; + + do { + spin_lock_bh(&ordered_sc_list->lock); + + if (ordered_sc_list->head.next == &ordered_sc_list->head) { + /* ordered_sc_list is empty; there is + * nothing to process + */ + spin_unlock_bh + (&ordered_sc_list->lock); + return 1; + } + + sc = (struct octeon_soft_command *)ordered_sc_list-> + head.next; + rdp = (struct octeon_instr_rdp *)&sc->cmd.rdp; + + status = OCTEON_REQUEST_PENDING; + + /* check if octeon has finished DMA'ing a response + * to where rptr is pointing to + */ + dma_sync_single_for_cpu(&octeon_dev->pci_dev->dev, + sc->cmd.rptr, rdp->rlen, + DMA_FROM_DEVICE); + status64 = *sc->status_word; + + if (status64 != COMPLETION_WORD_INIT) { + if ((status64 & 0xff) != 0xff) { + octeon_swap_8B_data(&status64, 1); + if (((status64 & 0xff) != 0xff)) { + status = (u32)(status64 & + 0xffffffffULL); + } + } + } else if (force_quit || (sc->timeout && + time_after(jiffies, (unsigned long)sc->timeout))) { + status = OCTEON_REQUEST_TIMEOUT; + } + + if (status != OCTEON_REQUEST_PENDING) { + /* we have received a response or we have timed out */ + /* remove node from linked list */ + list_del(&sc->node); + atomic_dec(&octeon_dev->response_list + [OCTEON_ORDERED_SC_LIST]. + pending_req_count); + spin_unlock_bh + (&ordered_sc_list->lock); + + if (sc->callback) + sc->callback(octeon_dev, status, + sc->callback_arg); + + request_complete++; + + } else { + /* no response yet */ + request_complete = 0; + spin_unlock_bh + (&ordered_sc_list->lock); + } + + /* If we hit the Max Ordered requests to process every loop, + * we quit + * and let this function be invoked the next time the poll + * thread runs + * to process the remaining requests. This function can take up + * the entire CPU if there is no upper limit to the requests + * processed. + */ + if (request_complete >= resp_to_process) + break; + } while (request_complete); + + return 0; +} + +static void oct_poll_req_completion(struct work_struct *work) +{ + struct cavium_wk *wk = (struct cavium_wk *)work; + struct octeon_device *oct = (struct octeon_device *)wk->ctxptr; + struct cavium_wq *cwq = &oct->dma_comp_wq; + + lio_process_ordered_list(oct, 0); + + queue_delayed_work(cwq->wq, &cwq->wk.work, msecs_to_jiffies(100)); +} diff --git a/kernel/drivers/net/ethernet/cavium/liquidio/response_manager.h b/kernel/drivers/net/ethernet/cavium/liquidio/response_manager.h new file mode 100644 index 000000000..7a48752dc --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/liquidio/response_manager.h @@ -0,0 +1,140 @@ +/********************************************************************** + * Author: Cavium, Inc. + * + * Contact: support@cavium.com + * Please include "LiquidIO" in the subject. + * + * Copyright (c) 2003-2015 Cavium, Inc. + * + * This file is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License, Version 2, as + * published by the Free Software Foundation. + * + * This file is distributed in the hope that it will be useful, but + * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or + * NONINFRINGEMENT. See the GNU General Public License for more + * details. + * + * This file may also be available under a different license from Cavium. + * Contact Cavium, Inc. for more information + **********************************************************************/ + +/*! \file response_manager.h + * \brief Host Driver: Response queues for host instructions. + */ + +#ifndef __RESPONSE_MANAGER_H__ +#define __RESPONSE_MANAGER_H__ + +/** Maximum ordered requests to process in every invocation of + * lio_process_ordered_list(). The function will continue to process requests + * as long as it can find one that has finished processing. If it keeps + * finding requests that have completed, the function can run for ever. The + * value defined here sets an upper limit on the number of requests it can + * process before it returns control to the poll thread. + */ +#define MAX_ORD_REQS_TO_PROCESS 4096 + +/** Head of a response list. There are several response lists in the + * system. One for each response order- Unordered, ordered + * and 1 for noresponse entries on each instruction queue. + */ +struct octeon_response_list { + /** List structure to add delete pending entries to */ + struct list_head head; + + /** A lock for this response list */ + spinlock_t lock; + + atomic_t pending_req_count; +}; + +/** The type of response list. + */ +enum { + OCTEON_ORDERED_LIST = 0, + OCTEON_UNORDERED_NONBLOCKING_LIST = 1, + OCTEON_UNORDERED_BLOCKING_LIST = 2, + OCTEON_ORDERED_SC_LIST = 3 +}; + +/** Response Order values for a Octeon Request. */ +enum { + OCTEON_RESP_ORDERED = 0, + OCTEON_RESP_UNORDERED = 1, + OCTEON_RESP_NORESPONSE = 2 +}; + +/** Error codes used in Octeon Host-Core communication. + * + * 31 16 15 0 + * --------------------------------- + * | | | + * --------------------------------- + * Error codes are 32-bit wide. The upper 16-bits, called Major Error Number, + * are reserved to identify the group to which the error code belongs. The + * lower 16-bits, called Minor Error Number, carry the actual code. + * + * So error codes are (MAJOR NUMBER << 16)| MINOR_NUMBER. + */ + +/*------------ Error codes used by host driver -----------------*/ +#define DRIVER_MAJOR_ERROR_CODE 0x0000 + +/** A value of 0x00000000 indicates no error i.e. success */ +#define DRIVER_ERROR_NONE 0x00000000 + +/** (Major number: 0x0000; Minor Number: 0x0001) */ +#define DRIVER_ERROR_REQ_PENDING 0x00000001 +#define DRIVER_ERROR_REQ_TIMEOUT 0x00000003 +#define DRIVER_ERROR_REQ_EINTR 0x00000004 +#define DRIVER_ERROR_REQ_ENXIO 0x00000006 +#define DRIVER_ERROR_REQ_ENOMEM 0x0000000C +#define DRIVER_ERROR_REQ_EINVAL 0x00000016 +#define DRIVER_ERROR_REQ_FAILED 0x000000ff + +/** Status for a request. + * If a request is not queued to Octeon by the driver, the driver returns + * an error condition that's describe by one of the OCTEON_REQ_ERR_* value + * below. If the request is successfully queued, the driver will return + * a OCTEON_REQUEST_PENDING status. OCTEON_REQUEST_TIMEOUT and + * OCTEON_REQUEST_INTERRUPTED are only returned by the driver if the + * response for request failed to arrive before a time-out period or if + * the request processing * got interrupted due to a signal respectively. + */ +enum { + OCTEON_REQUEST_DONE = (DRIVER_ERROR_NONE), + OCTEON_REQUEST_PENDING = (DRIVER_ERROR_REQ_PENDING), + OCTEON_REQUEST_TIMEOUT = (DRIVER_ERROR_REQ_TIMEOUT), + OCTEON_REQUEST_INTERRUPTED = (DRIVER_ERROR_REQ_EINTR), + OCTEON_REQUEST_NO_DEVICE = (0x00000021), + OCTEON_REQUEST_NOT_RUNNING, + OCTEON_REQUEST_INVALID_IQ, + OCTEON_REQUEST_INVALID_BUFCNT, + OCTEON_REQUEST_INVALID_RESP_ORDER, + OCTEON_REQUEST_NO_MEMORY, + OCTEON_REQUEST_INVALID_BUFSIZE, + OCTEON_REQUEST_NO_PENDING_ENTRY, + OCTEON_REQUEST_NO_IQ_SPACE = (0x7FFFFFFF) + +}; + +/** Initialize the response lists. The number of response lists to create is + * given by count. + * @param octeon_dev - the octeon device structure. + */ +int octeon_setup_response_list(struct octeon_device *octeon_dev); + +void octeon_delete_response_list(struct octeon_device *octeon_dev); + +/** Check the status of first entry in the ordered list. If the instruction at + * that entry finished processing or has timed-out, the entry is cleaned. + * @param octeon_dev - the octeon device structure. + * @param force_quit - the request is forced to timeout if this is 1 + * @return 1 if the ordered list is empty, 0 otherwise. + */ +int lio_process_ordered_list(struct octeon_device *octeon_dev, + u32 force_quit); + +#endif diff --git a/kernel/drivers/net/ethernet/cavium/thunder/Makefile b/kernel/drivers/net/ethernet/cavium/thunder/Makefile new file mode 100644 index 000000000..5c4615cca --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/thunder/Makefile @@ -0,0 +1,11 @@ +# +# Makefile for Cavium's Thunder ethernet device +# + +obj-$(CONFIG_THUNDER_NIC_BGX) += thunder_bgx.o +obj-$(CONFIG_THUNDER_NIC_PF) += nicpf.o +obj-$(CONFIG_THUNDER_NIC_VF) += nicvf.o + +nicpf-y := nic_main.o +nicvf-y := nicvf_main.o nicvf_queues.o +nicvf-y += nicvf_ethtool.o diff --git a/kernel/drivers/net/ethernet/cavium/thunder/nic.h b/kernel/drivers/net/ethernet/cavium/thunder/nic.h new file mode 100644 index 000000000..39ca6744a --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/thunder/nic.h @@ -0,0 +1,503 @@ +/* + * Copyright (C) 2015 Cavium, Inc. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of version 2 of the GNU General Public License + * as published by the Free Software Foundation. + */ + +#ifndef NIC_H +#define NIC_H + +#include <linux/netdevice.h> +#include <linux/interrupt.h> +#include <linux/pci.h> +#include "thunder_bgx.h" + +/* PCI device IDs */ +#define PCI_DEVICE_ID_THUNDER_NIC_PF 0xA01E +#define PCI_DEVICE_ID_THUNDER_PASS1_NIC_VF 0x0011 +#define PCI_DEVICE_ID_THUNDER_NIC_VF 0xA034 +#define PCI_DEVICE_ID_THUNDER_BGX 0xA026 + +/* PCI BAR nos */ +#define PCI_CFG_REG_BAR_NUM 0 +#define PCI_MSIX_REG_BAR_NUM 4 + +/* NIC SRIOV VF count */ +#define MAX_NUM_VFS_SUPPORTED 128 +#define DEFAULT_NUM_VF_ENABLED 8 + +#define NIC_TNS_BYPASS_MODE 0 +#define NIC_TNS_MODE 1 + +/* NIC priv flags */ +#define NIC_SRIOV_ENABLED BIT(0) + +/* Min/Max packet size */ +#define NIC_HW_MIN_FRS 64 +#define NIC_HW_MAX_FRS 9200 /* 9216 max packet including FCS */ + +/* Max pkinds */ +#define NIC_MAX_PKIND 16 + +/* Rx Channels */ +/* Receive channel configuration in TNS bypass mode + * Below is configuration in TNS bypass mode + * BGX0-LMAC0-CHAN0 - VNIC CHAN0 + * BGX0-LMAC1-CHAN0 - VNIC CHAN16 + * ... + * BGX1-LMAC0-CHAN0 - VNIC CHAN128 + * ... + * BGX1-LMAC3-CHAN0 - VNIC CHAN174 + */ +#define NIC_INTF_COUNT 2 /* Interfaces btw VNIC and TNS/BGX */ +#define NIC_CHANS_PER_INF 128 +#define NIC_MAX_CHANS (NIC_INTF_COUNT * NIC_CHANS_PER_INF) +#define NIC_CPI_COUNT 2048 /* No of channel parse indices */ + +/* TNS bypass mode: 1-1 mapping between VNIC and BGX:LMAC */ +#define NIC_MAX_BGX MAX_BGX_PER_CN88XX +#define NIC_CPI_PER_BGX (NIC_CPI_COUNT / NIC_MAX_BGX) +#define NIC_MAX_CPI_PER_LMAC 64 /* Max when CPI_ALG is IP diffserv */ +#define NIC_RSSI_PER_BGX (NIC_RSSI_COUNT / NIC_MAX_BGX) + +/* Tx scheduling */ +#define NIC_MAX_TL4 1024 +#define NIC_MAX_TL4_SHAPERS 256 /* 1 shaper for 4 TL4s */ +#define NIC_MAX_TL3 256 +#define NIC_MAX_TL3_SHAPERS 64 /* 1 shaper for 4 TL3s */ +#define NIC_MAX_TL2 64 +#define NIC_MAX_TL2_SHAPERS 2 /* 1 shaper for 32 TL2s */ +#define NIC_MAX_TL1 2 + +/* TNS bypass mode */ +#define NIC_TL2_PER_BGX 32 +#define NIC_TL4_PER_BGX (NIC_MAX_TL4 / NIC_MAX_BGX) +#define NIC_TL4_PER_LMAC (NIC_MAX_TL4 / NIC_CHANS_PER_INF) + +/* NIC VF Interrupts */ +#define NICVF_INTR_CQ 0 +#define NICVF_INTR_SQ 1 +#define NICVF_INTR_RBDR 2 +#define NICVF_INTR_PKT_DROP 3 +#define NICVF_INTR_TCP_TIMER 4 +#define NICVF_INTR_MBOX 5 +#define NICVF_INTR_QS_ERR 6 + +#define NICVF_INTR_CQ_SHIFT 0 +#define NICVF_INTR_SQ_SHIFT 8 +#define NICVF_INTR_RBDR_SHIFT 16 +#define NICVF_INTR_PKT_DROP_SHIFT 20 +#define NICVF_INTR_TCP_TIMER_SHIFT 21 +#define NICVF_INTR_MBOX_SHIFT 22 +#define NICVF_INTR_QS_ERR_SHIFT 23 + +#define NICVF_INTR_CQ_MASK (0xFF << NICVF_INTR_CQ_SHIFT) +#define NICVF_INTR_SQ_MASK (0xFF << NICVF_INTR_SQ_SHIFT) +#define NICVF_INTR_RBDR_MASK (0x03 << NICVF_INTR_RBDR_SHIFT) +#define NICVF_INTR_PKT_DROP_MASK BIT(NICVF_INTR_PKT_DROP_SHIFT) +#define NICVF_INTR_TCP_TIMER_MASK BIT(NICVF_INTR_TCP_TIMER_SHIFT) +#define NICVF_INTR_MBOX_MASK BIT(NICVF_INTR_MBOX_SHIFT) +#define NICVF_INTR_QS_ERR_MASK BIT(NICVF_INTR_QS_ERR_SHIFT) + +/* MSI-X interrupts */ +#define NIC_PF_MSIX_VECTORS 10 +#define NIC_VF_MSIX_VECTORS 20 + +#define NIC_PF_INTR_ID_ECC0_SBE 0 +#define NIC_PF_INTR_ID_ECC0_DBE 1 +#define NIC_PF_INTR_ID_ECC1_SBE 2 +#define NIC_PF_INTR_ID_ECC1_DBE 3 +#define NIC_PF_INTR_ID_ECC2_SBE 4 +#define NIC_PF_INTR_ID_ECC2_DBE 5 +#define NIC_PF_INTR_ID_ECC3_SBE 6 +#define NIC_PF_INTR_ID_ECC3_DBE 7 +#define NIC_PF_INTR_ID_MBOX0 8 +#define NIC_PF_INTR_ID_MBOX1 9 + +/* Global timer for CQ timer thresh interrupts + * Calculated for SCLK of 700Mhz + * value written should be a 1/16th of what is expected + * + * 1 tick per 0.025usec + */ +#define NICPF_CLK_PER_INT_TICK 1 + +/* Time to wait before we decide that a SQ is stuck. + * + * Since both pkt rx and tx notifications are done with same CQ, + * when packets are being received at very high rate (eg: L2 forwarding) + * then freeing transmitted skbs will be delayed and watchdog + * will kick in, resetting interface. Hence keeping this value high. + */ +#define NICVF_TX_TIMEOUT (50 * HZ) + +struct nicvf_cq_poll { + struct nicvf *nicvf; + u8 cq_idx; /* Completion queue index */ + struct napi_struct napi; +}; + +#define NIC_RSSI_COUNT 4096 /* Total no of RSS indices */ +#define NIC_MAX_RSS_HASH_BITS 8 +#define NIC_MAX_RSS_IDR_TBL_SIZE (1 << NIC_MAX_RSS_HASH_BITS) +#define RSS_HASH_KEY_SIZE 5 /* 320 bit key */ + +struct nicvf_rss_info { + bool enable; +#define RSS_L2_EXTENDED_HASH_ENA BIT(0) +#define RSS_IP_HASH_ENA BIT(1) +#define RSS_TCP_HASH_ENA BIT(2) +#define RSS_TCP_SYN_DIS BIT(3) +#define RSS_UDP_HASH_ENA BIT(4) +#define RSS_L4_EXTENDED_HASH_ENA BIT(5) +#define RSS_ROCE_ENA BIT(6) +#define RSS_L3_BI_DIRECTION_ENA BIT(7) +#define RSS_L4_BI_DIRECTION_ENA BIT(8) + u64 cfg; + u8 hash_bits; + u16 rss_size; + u8 ind_tbl[NIC_MAX_RSS_IDR_TBL_SIZE]; + u64 key[RSS_HASH_KEY_SIZE]; +} ____cacheline_aligned_in_smp; + +enum rx_stats_reg_offset { + RX_OCTS = 0x0, + RX_UCAST = 0x1, + RX_BCAST = 0x2, + RX_MCAST = 0x3, + RX_RED = 0x4, + RX_RED_OCTS = 0x5, + RX_ORUN = 0x6, + RX_ORUN_OCTS = 0x7, + RX_FCS = 0x8, + RX_L2ERR = 0x9, + RX_DRP_BCAST = 0xa, + RX_DRP_MCAST = 0xb, + RX_DRP_L3BCAST = 0xc, + RX_DRP_L3MCAST = 0xd, + RX_STATS_ENUM_LAST, +}; + +enum tx_stats_reg_offset { + TX_OCTS = 0x0, + TX_UCAST = 0x1, + TX_BCAST = 0x2, + TX_MCAST = 0x3, + TX_DROP = 0x4, + TX_STATS_ENUM_LAST, +}; + +struct nicvf_hw_stats { + u64 rx_bytes; + u64 rx_ucast_frames; + u64 rx_bcast_frames; + u64 rx_mcast_frames; + u64 rx_fcs_errors; + u64 rx_l2_errors; + u64 rx_drop_red; + u64 rx_drop_red_bytes; + u64 rx_drop_overrun; + u64 rx_drop_overrun_bytes; + u64 rx_drop_bcast; + u64 rx_drop_mcast; + u64 rx_drop_l3_bcast; + u64 rx_drop_l3_mcast; + u64 rx_bgx_truncated_pkts; + u64 rx_jabber_errs; + u64 rx_fcs_errs; + u64 rx_bgx_errs; + u64 rx_prel2_errs; + u64 rx_l2_hdr_malformed; + u64 rx_oversize; + u64 rx_undersize; + u64 rx_l2_len_mismatch; + u64 rx_l2_pclp; + u64 rx_ip_ver_errs; + u64 rx_ip_csum_errs; + u64 rx_ip_hdr_malformed; + u64 rx_ip_payload_malformed; + u64 rx_ip_ttl_errs; + u64 rx_l3_pclp; + u64 rx_l4_malformed; + u64 rx_l4_csum_errs; + u64 rx_udp_len_errs; + u64 rx_l4_port_errs; + u64 rx_tcp_flag_errs; + u64 rx_tcp_offset_errs; + u64 rx_l4_pclp; + u64 rx_truncated_pkts; + + u64 tx_bytes_ok; + u64 tx_ucast_frames_ok; + u64 tx_bcast_frames_ok; + u64 tx_mcast_frames_ok; + u64 tx_drops; +}; + +struct nicvf_drv_stats { + /* Rx */ + u64 rx_frames_ok; + u64 rx_frames_64; + u64 rx_frames_127; + u64 rx_frames_255; + u64 rx_frames_511; + u64 rx_frames_1023; + u64 rx_frames_1518; + u64 rx_frames_jumbo; + u64 rx_drops; + + /* Tx */ + u64 tx_frames_ok; + u64 tx_drops; + u64 tx_tso; + u64 txq_stop; + u64 txq_wake; +}; + +struct nicvf { + struct nicvf *pnicvf; + struct net_device *netdev; + struct pci_dev *pdev; + u8 vf_id; + u8 node; + u8 tns_mode:1; + u8 sqs_mode:1; + u8 loopback_supported:1; + u16 mtu; + struct queue_set *qs; +#define MAX_SQS_PER_VF_SINGLE_NODE 5 +#define MAX_SQS_PER_VF 11 + u8 sqs_id; + u8 sqs_count; /* Secondary Qset count */ + struct nicvf *snicvf[MAX_SQS_PER_VF]; + u8 rx_queues; + u8 tx_queues; + u8 max_queues; + void __iomem *reg_base; + bool link_up; + u8 duplex; + u32 speed; + struct page *rb_page; + u32 rb_page_offset; + bool rb_alloc_fail; + bool rb_work_scheduled; + struct delayed_work rbdr_work; + struct tasklet_struct rbdr_task; + struct tasklet_struct qs_err_task; + struct tasklet_struct cq_task; + struct nicvf_cq_poll *napi[8]; + struct nicvf_rss_info rss_info; + u8 cpi_alg; + /* Interrupt coalescing settings */ + u32 cq_coalesce_usecs; + + u32 msg_enable; + struct nicvf_hw_stats hw_stats; + struct nicvf_drv_stats drv_stats; + struct bgx_stats bgx_stats; + struct work_struct reset_task; + + /* MSI-X */ + bool msix_enabled; + u8 num_vec; + struct msix_entry msix_entries[NIC_VF_MSIX_VECTORS]; + char irq_name[NIC_VF_MSIX_VECTORS][20]; + bool irq_allocated[NIC_VF_MSIX_VECTORS]; + + /* VF <-> PF mailbox communication */ + bool pf_acked; + bool pf_nacked; + bool set_mac_pending; +} ____cacheline_aligned_in_smp; + +/* PF <--> VF Mailbox communication + * Eight 64bit registers are shared between PF and VF. + * Separate set for each VF. + * Writing '1' into last register mbx7 means end of message. + */ + +/* PF <--> VF mailbox communication */ +#define NIC_PF_VF_MAILBOX_SIZE 2 +#define NIC_MBOX_MSG_TIMEOUT 2000 /* ms */ + +/* Mailbox message types */ +#define NIC_MBOX_MSG_READY 0x01 /* Is PF ready to rcv msgs */ +#define NIC_MBOX_MSG_ACK 0x02 /* ACK the message received */ +#define NIC_MBOX_MSG_NACK 0x03 /* NACK the message received */ +#define NIC_MBOX_MSG_QS_CFG 0x04 /* Configure Qset */ +#define NIC_MBOX_MSG_RQ_CFG 0x05 /* Configure receive queue */ +#define NIC_MBOX_MSG_SQ_CFG 0x06 /* Configure Send queue */ +#define NIC_MBOX_MSG_RQ_DROP_CFG 0x07 /* Configure receive queue */ +#define NIC_MBOX_MSG_SET_MAC 0x08 /* Add MAC ID to DMAC filter */ +#define NIC_MBOX_MSG_SET_MAX_FRS 0x09 /* Set max frame size */ +#define NIC_MBOX_MSG_CPI_CFG 0x0A /* Config CPI, RSSI */ +#define NIC_MBOX_MSG_RSS_SIZE 0x0B /* Get RSS indir_tbl size */ +#define NIC_MBOX_MSG_RSS_CFG 0x0C /* Config RSS table */ +#define NIC_MBOX_MSG_RSS_CFG_CONT 0x0D /* RSS config continuation */ +#define NIC_MBOX_MSG_RQ_BP_CFG 0x0E /* RQ backpressure config */ +#define NIC_MBOX_MSG_RQ_SW_SYNC 0x0F /* Flush inflight pkts to RQ */ +#define NIC_MBOX_MSG_BGX_STATS 0x10 /* Get stats from BGX */ +#define NIC_MBOX_MSG_BGX_LINK_CHANGE 0x11 /* BGX:LMAC link status */ +#define NIC_MBOX_MSG_ALLOC_SQS 0x12 /* Allocate secondary Qset */ +#define NIC_MBOX_MSG_NICVF_PTR 0x13 /* Send nicvf ptr to PF */ +#define NIC_MBOX_MSG_PNICVF_PTR 0x14 /* Get primary qset nicvf ptr */ +#define NIC_MBOX_MSG_SNICVF_PTR 0x15 /* Send sqet nicvf ptr to PVF */ +#define NIC_MBOX_MSG_LOOPBACK 0x16 /* Set interface in loopback */ +#define NIC_MBOX_MSG_CFG_DONE 0xF0 /* VF configuration done */ +#define NIC_MBOX_MSG_SHUTDOWN 0xF1 /* VF is being shutdown */ + +struct nic_cfg_msg { + u8 msg; + u8 vf_id; + u8 node_id; + u8 tns_mode:1; + u8 sqs_mode:1; + u8 loopback_supported:1; + u8 mac_addr[ETH_ALEN]; +}; + +/* Qset configuration */ +struct qs_cfg_msg { + u8 msg; + u8 num; + u8 sqs_count; + u64 cfg; +}; + +/* Receive queue configuration */ +struct rq_cfg_msg { + u8 msg; + u8 qs_num; + u8 rq_num; + u64 cfg; +}; + +/* Send queue configuration */ +struct sq_cfg_msg { + u8 msg; + u8 qs_num; + u8 sq_num; + bool sqs_mode; + u64 cfg; +}; + +/* Set VF's MAC address */ +struct set_mac_msg { + u8 msg; + u8 vf_id; + u8 mac_addr[ETH_ALEN]; +}; + +/* Set Maximum frame size */ +struct set_frs_msg { + u8 msg; + u8 vf_id; + u16 max_frs; +}; + +/* Set CPI algorithm type */ +struct cpi_cfg_msg { + u8 msg; + u8 vf_id; + u8 rq_cnt; + u8 cpi_alg; +}; + +/* Get RSS table size */ +struct rss_sz_msg { + u8 msg; + u8 vf_id; + u16 ind_tbl_size; +}; + +/* Set RSS configuration */ +struct rss_cfg_msg { + u8 msg; + u8 vf_id; + u8 hash_bits; + u8 tbl_len; + u8 tbl_offset; +#define RSS_IND_TBL_LEN_PER_MBX_MSG 8 + u8 ind_tbl[RSS_IND_TBL_LEN_PER_MBX_MSG]; +}; + +struct bgx_stats_msg { + u8 msg; + u8 vf_id; + u8 rx; + u8 idx; + u64 stats; +}; + +/* Physical interface link status */ +struct bgx_link_status { + u8 msg; + u8 link_up; + u8 duplex; + u32 speed; +}; + +/* Get Extra Qset IDs */ +struct sqs_alloc { + u8 msg; + u8 vf_id; + u8 qs_count; +}; + +struct nicvf_ptr { + u8 msg; + u8 vf_id; + bool sqs_mode; + u8 sqs_id; + u64 nicvf; +}; + +/* Set interface in loopback mode */ +struct set_loopback { + u8 msg; + u8 vf_id; + bool enable; +}; + +/* 128 bit shared memory between PF and each VF */ +union nic_mbx { + struct { u8 msg; } msg; + struct nic_cfg_msg nic_cfg; + struct qs_cfg_msg qs; + struct rq_cfg_msg rq; + struct sq_cfg_msg sq; + struct set_mac_msg mac; + struct set_frs_msg frs; + struct cpi_cfg_msg cpi_cfg; + struct rss_sz_msg rss_size; + struct rss_cfg_msg rss_cfg; + struct bgx_stats_msg bgx_stats; + struct bgx_link_status link_status; + struct sqs_alloc sqs_alloc; + struct nicvf_ptr nicvf; + struct set_loopback lbk; +}; + +#define NIC_NODE_ID_MASK 0x03 +#define NIC_NODE_ID_SHIFT 44 + +static inline int nic_get_node_id(struct pci_dev *pdev) +{ + u64 addr = pci_resource_start(pdev, PCI_CFG_REG_BAR_NUM); + return ((addr >> NIC_NODE_ID_SHIFT) & NIC_NODE_ID_MASK); +} + +int nicvf_set_real_num_queues(struct net_device *netdev, + int tx_queues, int rx_queues); +int nicvf_open(struct net_device *netdev); +int nicvf_stop(struct net_device *netdev); +int nicvf_send_msg_to_pf(struct nicvf *vf, union nic_mbx *mbx); +void nicvf_config_rss(struct nicvf *nic); +void nicvf_set_rss_key(struct nicvf *nic); +void nicvf_set_ethtool_ops(struct net_device *netdev); +void nicvf_update_stats(struct nicvf *nic); +void nicvf_update_lmac_stats(struct nicvf *nic); + +#endif /* NIC_H */ diff --git a/kernel/drivers/net/ethernet/cavium/thunder/nic_main.c b/kernel/drivers/net/ethernet/cavium/thunder/nic_main.c new file mode 100644 index 000000000..5f24d11cb --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/thunder/nic_main.c @@ -0,0 +1,1122 @@ +/* + * Copyright (C) 2015 Cavium, Inc. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of version 2 of the GNU General Public License + * as published by the Free Software Foundation. + */ + +#include <linux/module.h> +#include <linux/interrupt.h> +#include <linux/pci.h> +#include <linux/etherdevice.h> +#include <linux/of.h> + +#include "nic_reg.h" +#include "nic.h" +#include "q_struct.h" +#include "thunder_bgx.h" + +#define DRV_NAME "thunder-nic" +#define DRV_VERSION "1.0" + +struct nicpf { + struct pci_dev *pdev; + u8 node; + unsigned int flags; + u8 num_vf_en; /* No of VF enabled */ + bool vf_enabled[MAX_NUM_VFS_SUPPORTED]; + void __iomem *reg_base; /* Register start address */ + u8 num_sqs_en; /* Secondary qsets enabled */ + u64 nicvf[MAX_NUM_VFS_SUPPORTED]; + u8 vf_sqs[MAX_NUM_VFS_SUPPORTED][MAX_SQS_PER_VF]; + u8 pqs_vf[MAX_NUM_VFS_SUPPORTED]; + bool sqs_used[MAX_NUM_VFS_SUPPORTED]; + struct pkind_cfg pkind; +#define NIC_SET_VF_LMAC_MAP(bgx, lmac) (((bgx & 0xF) << 4) | (lmac & 0xF)) +#define NIC_GET_BGX_FROM_VF_LMAC_MAP(map) ((map >> 4) & 0xF) +#define NIC_GET_LMAC_FROM_VF_LMAC_MAP(map) (map & 0xF) + u8 vf_lmac_map[MAX_LMAC]; + struct delayed_work dwork; + struct workqueue_struct *check_link; + u8 link[MAX_LMAC]; + u8 duplex[MAX_LMAC]; + u32 speed[MAX_LMAC]; + u16 cpi_base[MAX_NUM_VFS_SUPPORTED]; + u16 rssi_base[MAX_NUM_VFS_SUPPORTED]; + u16 rss_ind_tbl_size; + bool mbx_lock[MAX_NUM_VFS_SUPPORTED]; + + /* MSI-X */ + bool msix_enabled; + u8 num_vec; + struct msix_entry msix_entries[NIC_PF_MSIX_VECTORS]; + bool irq_allocated[NIC_PF_MSIX_VECTORS]; +}; + +static inline bool pass1_silicon(struct nicpf *nic) +{ + return nic->pdev->revision < 8; +} + +/* Supported devices */ +static const struct pci_device_id nic_id_table[] = { + { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_NIC_PF) }, + { 0, } /* end of table */ +}; + +MODULE_AUTHOR("Sunil Goutham"); +MODULE_DESCRIPTION("Cavium Thunder NIC Physical Function Driver"); +MODULE_LICENSE("GPL v2"); +MODULE_VERSION(DRV_VERSION); +MODULE_DEVICE_TABLE(pci, nic_id_table); + +/* The Cavium ThunderX network controller can *only* be found in SoCs + * containing the ThunderX ARM64 CPU implementation. All accesses to the device + * registers on this platform are implicitly strongly ordered with respect + * to memory accesses. So writeq_relaxed() and readq_relaxed() are safe to use + * with no memory barriers in this driver. The readq()/writeq() functions add + * explicit ordering operation which in this case are redundant, and only + * add overhead. + */ + +/* Register read/write APIs */ +static void nic_reg_write(struct nicpf *nic, u64 offset, u64 val) +{ + writeq_relaxed(val, nic->reg_base + offset); +} + +static u64 nic_reg_read(struct nicpf *nic, u64 offset) +{ + return readq_relaxed(nic->reg_base + offset); +} + +/* PF -> VF mailbox communication APIs */ +static void nic_enable_mbx_intr(struct nicpf *nic) +{ + /* Enable mailbox interrupt for all 128 VFs */ + nic_reg_write(nic, NIC_PF_MAILBOX_ENA_W1S, ~0ull); + nic_reg_write(nic, NIC_PF_MAILBOX_ENA_W1S + sizeof(u64), ~0ull); +} + +static void nic_clear_mbx_intr(struct nicpf *nic, int vf, int mbx_reg) +{ + nic_reg_write(nic, NIC_PF_MAILBOX_INT + (mbx_reg << 3), BIT_ULL(vf)); +} + +static u64 nic_get_mbx_addr(int vf) +{ + return NIC_PF_VF_0_127_MAILBOX_0_1 + (vf << NIC_VF_NUM_SHIFT); +} + +/* Send a mailbox message to VF + * @vf: vf to which this message to be sent + * @mbx: Message to be sent + */ +static void nic_send_msg_to_vf(struct nicpf *nic, int vf, union nic_mbx *mbx) +{ + void __iomem *mbx_addr = nic->reg_base + nic_get_mbx_addr(vf); + u64 *msg = (u64 *)mbx; + + /* In first revision HW, mbox interrupt is triggerred + * when PF writes to MBOX(1), in next revisions when + * PF writes to MBOX(0) + */ + if (pass1_silicon(nic)) { + /* see the comment for nic_reg_write()/nic_reg_read() + * functions above + */ + writeq_relaxed(msg[0], mbx_addr); + writeq_relaxed(msg[1], mbx_addr + 8); + } else { + writeq_relaxed(msg[1], mbx_addr + 8); + writeq_relaxed(msg[0], mbx_addr); + } +} + +/* Responds to VF's READY message with VF's + * ID, node, MAC address e.t.c + * @vf: VF which sent READY message + */ +static void nic_mbx_send_ready(struct nicpf *nic, int vf) +{ + union nic_mbx mbx = {}; + int bgx_idx, lmac; + const char *mac; + + mbx.nic_cfg.msg = NIC_MBOX_MSG_READY; + mbx.nic_cfg.vf_id = vf; + + mbx.nic_cfg.tns_mode = NIC_TNS_BYPASS_MODE; + + if (vf < MAX_LMAC) { + bgx_idx = NIC_GET_BGX_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vf]); + lmac = NIC_GET_LMAC_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vf]); + + mac = bgx_get_lmac_mac(nic->node, bgx_idx, lmac); + if (mac) + ether_addr_copy((u8 *)&mbx.nic_cfg.mac_addr, mac); + } + mbx.nic_cfg.sqs_mode = (vf >= nic->num_vf_en) ? true : false; + mbx.nic_cfg.node_id = nic->node; + + mbx.nic_cfg.loopback_supported = vf < MAX_LMAC; + + nic_send_msg_to_vf(nic, vf, &mbx); +} + +/* ACKs VF's mailbox message + * @vf: VF to which ACK to be sent + */ +static void nic_mbx_send_ack(struct nicpf *nic, int vf) +{ + union nic_mbx mbx = {}; + + mbx.msg.msg = NIC_MBOX_MSG_ACK; + nic_send_msg_to_vf(nic, vf, &mbx); +} + +/* NACKs VF's mailbox message that PF is not able to + * complete the action + * @vf: VF to which ACK to be sent + */ +static void nic_mbx_send_nack(struct nicpf *nic, int vf) +{ + union nic_mbx mbx = {}; + + mbx.msg.msg = NIC_MBOX_MSG_NACK; + nic_send_msg_to_vf(nic, vf, &mbx); +} + +/* Flush all in flight receive packets to memory and + * bring down an active RQ + */ +static int nic_rcv_queue_sw_sync(struct nicpf *nic) +{ + u16 timeout = ~0x00; + + nic_reg_write(nic, NIC_PF_SW_SYNC_RX, 0x01); + /* Wait till sync cycle is finished */ + while (timeout) { + if (nic_reg_read(nic, NIC_PF_SW_SYNC_RX_DONE) & 0x1) + break; + timeout--; + } + nic_reg_write(nic, NIC_PF_SW_SYNC_RX, 0x00); + if (!timeout) { + dev_err(&nic->pdev->dev, "Receive queue software sync failed"); + return 1; + } + return 0; +} + +/* Get BGX Rx/Tx stats and respond to VF's request */ +static void nic_get_bgx_stats(struct nicpf *nic, struct bgx_stats_msg *bgx) +{ + int bgx_idx, lmac; + union nic_mbx mbx = {}; + + bgx_idx = NIC_GET_BGX_FROM_VF_LMAC_MAP(nic->vf_lmac_map[bgx->vf_id]); + lmac = NIC_GET_LMAC_FROM_VF_LMAC_MAP(nic->vf_lmac_map[bgx->vf_id]); + + mbx.bgx_stats.msg = NIC_MBOX_MSG_BGX_STATS; + mbx.bgx_stats.vf_id = bgx->vf_id; + mbx.bgx_stats.rx = bgx->rx; + mbx.bgx_stats.idx = bgx->idx; + if (bgx->rx) + mbx.bgx_stats.stats = bgx_get_rx_stats(nic->node, bgx_idx, + lmac, bgx->idx); + else + mbx.bgx_stats.stats = bgx_get_tx_stats(nic->node, bgx_idx, + lmac, bgx->idx); + nic_send_msg_to_vf(nic, bgx->vf_id, &mbx); +} + +/* Update hardware min/max frame size */ +static int nic_update_hw_frs(struct nicpf *nic, int new_frs, int vf) +{ + if ((new_frs > NIC_HW_MAX_FRS) || (new_frs < NIC_HW_MIN_FRS)) { + dev_err(&nic->pdev->dev, + "Invalid MTU setting from VF%d rejected, should be between %d and %d\n", + vf, NIC_HW_MIN_FRS, NIC_HW_MAX_FRS); + return 1; + } + new_frs += ETH_HLEN; + if (new_frs <= nic->pkind.maxlen) + return 0; + + nic->pkind.maxlen = new_frs; + nic_reg_write(nic, NIC_PF_PKIND_0_15_CFG, *(u64 *)&nic->pkind); + return 0; +} + +/* Set minimum transmit packet size */ +static void nic_set_tx_pkt_pad(struct nicpf *nic, int size) +{ + int lmac; + u64 lmac_cfg; + + /* Max value that can be set is 60 */ + if (size > 60) + size = 60; + + for (lmac = 0; lmac < (MAX_BGX_PER_CN88XX * MAX_LMAC_PER_BGX); lmac++) { + lmac_cfg = nic_reg_read(nic, NIC_PF_LMAC_0_7_CFG | (lmac << 3)); + lmac_cfg &= ~(0xF << 2); + lmac_cfg |= ((size / 4) << 2); + nic_reg_write(nic, NIC_PF_LMAC_0_7_CFG | (lmac << 3), lmac_cfg); + } +} + +/* Function to check number of LMACs present and set VF::LMAC mapping. + * Mapping will be used while initializing channels. + */ +static void nic_set_lmac_vf_mapping(struct nicpf *nic) +{ + unsigned bgx_map = bgx_get_map(nic->node); + int bgx, next_bgx_lmac = 0; + int lmac, lmac_cnt = 0; + u64 lmac_credit; + + nic->num_vf_en = 0; + + for (bgx = 0; bgx < NIC_MAX_BGX; bgx++) { + if (!(bgx_map & (1 << bgx))) + continue; + lmac_cnt = bgx_get_lmac_count(nic->node, bgx); + for (lmac = 0; lmac < lmac_cnt; lmac++) + nic->vf_lmac_map[next_bgx_lmac++] = + NIC_SET_VF_LMAC_MAP(bgx, lmac); + nic->num_vf_en += lmac_cnt; + + /* Program LMAC credits */ + lmac_credit = (1ull << 1); /* channel credit enable */ + lmac_credit |= (0x1ff << 2); /* Max outstanding pkt count */ + /* 48KB BGX Tx buffer size, each unit is of size 16bytes */ + lmac_credit |= (((((48 * 1024) / lmac_cnt) - + NIC_HW_MAX_FRS) / 16) << 12); + lmac = bgx * MAX_LMAC_PER_BGX; + for (; lmac < lmac_cnt + (bgx * MAX_LMAC_PER_BGX); lmac++) + nic_reg_write(nic, + NIC_PF_LMAC_0_7_CREDIT + (lmac * 8), + lmac_credit); + } +} + +#define BGX0_BLOCK 8 +#define BGX1_BLOCK 9 + +static void nic_init_hw(struct nicpf *nic) +{ + int i; + + /* Enable NIC HW block */ + nic_reg_write(nic, NIC_PF_CFG, 0x3); + + /* Enable backpressure */ + nic_reg_write(nic, NIC_PF_BP_CFG, (1ULL << 6) | 0x03); + + /* Disable TNS mode on both interfaces */ + nic_reg_write(nic, NIC_PF_INTF_0_1_SEND_CFG, + (NIC_TNS_BYPASS_MODE << 7) | BGX0_BLOCK); + nic_reg_write(nic, NIC_PF_INTF_0_1_SEND_CFG | (1 << 8), + (NIC_TNS_BYPASS_MODE << 7) | BGX1_BLOCK); + nic_reg_write(nic, NIC_PF_INTF_0_1_BP_CFG, + (1ULL << 63) | BGX0_BLOCK); + nic_reg_write(nic, NIC_PF_INTF_0_1_BP_CFG + (1 << 8), + (1ULL << 63) | BGX1_BLOCK); + + /* PKIND configuration */ + nic->pkind.minlen = 0; + nic->pkind.maxlen = NIC_HW_MAX_FRS + ETH_HLEN; + nic->pkind.lenerr_en = 1; + nic->pkind.rx_hdr = 0; + nic->pkind.hdr_sl = 0; + + for (i = 0; i < NIC_MAX_PKIND; i++) + nic_reg_write(nic, NIC_PF_PKIND_0_15_CFG | (i << 3), + *(u64 *)&nic->pkind); + + nic_set_tx_pkt_pad(nic, NIC_HW_MIN_FRS); + + /* Timer config */ + nic_reg_write(nic, NIC_PF_INTR_TIMER_CFG, NICPF_CLK_PER_INT_TICK); + + /* Enable VLAN ethertype matching and stripping */ + nic_reg_write(nic, NIC_PF_RX_ETYPE_0_7, + (2 << 19) | (ETYPE_ALG_VLAN_STRIP << 16) | ETH_P_8021Q); +} + +/* Channel parse index configuration */ +static void nic_config_cpi(struct nicpf *nic, struct cpi_cfg_msg *cfg) +{ + u32 vnic, bgx, lmac, chan; + u32 padd, cpi_count = 0; + u64 cpi_base, cpi, rssi_base, rssi; + u8 qset, rq_idx = 0; + + vnic = cfg->vf_id; + bgx = NIC_GET_BGX_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vnic]); + lmac = NIC_GET_LMAC_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vnic]); + + chan = (lmac * MAX_BGX_CHANS_PER_LMAC) + (bgx * NIC_CHANS_PER_INF); + cpi_base = (lmac * NIC_MAX_CPI_PER_LMAC) + (bgx * NIC_CPI_PER_BGX); + rssi_base = (lmac * nic->rss_ind_tbl_size) + (bgx * NIC_RSSI_PER_BGX); + + /* Rx channel configuration */ + nic_reg_write(nic, NIC_PF_CHAN_0_255_RX_BP_CFG | (chan << 3), + (1ull << 63) | (vnic << 0)); + nic_reg_write(nic, NIC_PF_CHAN_0_255_RX_CFG | (chan << 3), + ((u64)cfg->cpi_alg << 62) | (cpi_base << 48)); + + if (cfg->cpi_alg == CPI_ALG_NONE) + cpi_count = 1; + else if (cfg->cpi_alg == CPI_ALG_VLAN) /* 3 bits of PCP */ + cpi_count = 8; + else if (cfg->cpi_alg == CPI_ALG_VLAN16) /* 3 bits PCP + DEI */ + cpi_count = 16; + else if (cfg->cpi_alg == CPI_ALG_DIFF) /* 6bits DSCP */ + cpi_count = NIC_MAX_CPI_PER_LMAC; + + /* RSS Qset, Qidx mapping */ + qset = cfg->vf_id; + rssi = rssi_base; + for (; rssi < (rssi_base + cfg->rq_cnt); rssi++) { + nic_reg_write(nic, NIC_PF_RSSI_0_4097_RQ | (rssi << 3), + (qset << 3) | rq_idx); + rq_idx++; + } + + rssi = 0; + cpi = cpi_base; + for (; cpi < (cpi_base + cpi_count); cpi++) { + /* Determine port to channel adder */ + if (cfg->cpi_alg != CPI_ALG_DIFF) + padd = cpi % cpi_count; + else + padd = cpi % 8; /* 3 bits CS out of 6bits DSCP */ + + /* Leave RSS_SIZE as '0' to disable RSS */ + if (pass1_silicon(nic)) { + nic_reg_write(nic, NIC_PF_CPI_0_2047_CFG | (cpi << 3), + (vnic << 24) | (padd << 16) | + (rssi_base + rssi)); + } else { + /* Set MPI_ALG to '0' to disable MCAM parsing */ + nic_reg_write(nic, NIC_PF_CPI_0_2047_CFG | (cpi << 3), + (padd << 16)); + /* MPI index is same as CPI if MPI_ALG is not enabled */ + nic_reg_write(nic, NIC_PF_MPI_0_2047_CFG | (cpi << 3), + (vnic << 24) | (rssi_base + rssi)); + } + + if ((rssi + 1) >= cfg->rq_cnt) + continue; + + if (cfg->cpi_alg == CPI_ALG_VLAN) + rssi++; + else if (cfg->cpi_alg == CPI_ALG_VLAN16) + rssi = ((cpi - cpi_base) & 0xe) >> 1; + else if (cfg->cpi_alg == CPI_ALG_DIFF) + rssi = ((cpi - cpi_base) & 0x38) >> 3; + } + nic->cpi_base[cfg->vf_id] = cpi_base; + nic->rssi_base[cfg->vf_id] = rssi_base; +} + +/* Responsds to VF with its RSS indirection table size */ +static void nic_send_rss_size(struct nicpf *nic, int vf) +{ + union nic_mbx mbx = {}; + u64 *msg; + + msg = (u64 *)&mbx; + + mbx.rss_size.msg = NIC_MBOX_MSG_RSS_SIZE; + mbx.rss_size.ind_tbl_size = nic->rss_ind_tbl_size; + nic_send_msg_to_vf(nic, vf, &mbx); +} + +/* Receive side scaling configuration + * configure: + * - RSS index + * - indir table i.e hash::RQ mapping + * - no of hash bits to consider + */ +static void nic_config_rss(struct nicpf *nic, struct rss_cfg_msg *cfg) +{ + u8 qset, idx = 0; + u64 cpi_cfg, cpi_base, rssi_base, rssi; + u64 idx_addr; + + rssi_base = nic->rssi_base[cfg->vf_id] + cfg->tbl_offset; + + rssi = rssi_base; + qset = cfg->vf_id; + + for (; rssi < (rssi_base + cfg->tbl_len); rssi++) { + u8 svf = cfg->ind_tbl[idx] >> 3; + + if (svf) + qset = nic->vf_sqs[cfg->vf_id][svf - 1]; + else + qset = cfg->vf_id; + nic_reg_write(nic, NIC_PF_RSSI_0_4097_RQ | (rssi << 3), + (qset << 3) | (cfg->ind_tbl[idx] & 0x7)); + idx++; + } + + cpi_base = nic->cpi_base[cfg->vf_id]; + if (pass1_silicon(nic)) + idx_addr = NIC_PF_CPI_0_2047_CFG; + else + idx_addr = NIC_PF_MPI_0_2047_CFG; + cpi_cfg = nic_reg_read(nic, idx_addr | (cpi_base << 3)); + cpi_cfg &= ~(0xFULL << 20); + cpi_cfg |= (cfg->hash_bits << 20); + nic_reg_write(nic, idx_addr | (cpi_base << 3), cpi_cfg); +} + +/* 4 level transmit side scheduler configutation + * for TNS bypass mode + * + * Sample configuration for SQ0 + * VNIC0-SQ0 -> TL4(0) -> TL3[0] -> TL2[0] -> TL1[0] -> BGX0 + * VNIC1-SQ0 -> TL4(8) -> TL3[2] -> TL2[0] -> TL1[0] -> BGX0 + * VNIC2-SQ0 -> TL4(16) -> TL3[4] -> TL2[1] -> TL1[0] -> BGX0 + * VNIC3-SQ0 -> TL4(24) -> TL3[6] -> TL2[1] -> TL1[0] -> BGX0 + * VNIC4-SQ0 -> TL4(512) -> TL3[128] -> TL2[32] -> TL1[1] -> BGX1 + * VNIC5-SQ0 -> TL4(520) -> TL3[130] -> TL2[32] -> TL1[1] -> BGX1 + * VNIC6-SQ0 -> TL4(528) -> TL3[132] -> TL2[33] -> TL1[1] -> BGX1 + * VNIC7-SQ0 -> TL4(536) -> TL3[134] -> TL2[33] -> TL1[1] -> BGX1 + */ +static void nic_tx_channel_cfg(struct nicpf *nic, u8 vnic, + struct sq_cfg_msg *sq) +{ + u32 bgx, lmac, chan; + u32 tl2, tl3, tl4; + u32 rr_quantum; + u8 sq_idx = sq->sq_num; + u8 pqs_vnic; + + if (sq->sqs_mode) + pqs_vnic = nic->pqs_vf[vnic]; + else + pqs_vnic = vnic; + + bgx = NIC_GET_BGX_FROM_VF_LMAC_MAP(nic->vf_lmac_map[pqs_vnic]); + lmac = NIC_GET_LMAC_FROM_VF_LMAC_MAP(nic->vf_lmac_map[pqs_vnic]); + + /* 24 bytes for FCS, IPG and preamble */ + rr_quantum = ((NIC_HW_MAX_FRS + 24) / 4); + + tl4 = (lmac * NIC_TL4_PER_LMAC) + (bgx * NIC_TL4_PER_BGX); + tl4 += sq_idx; + if (sq->sqs_mode) + tl4 += vnic * 8; + + tl3 = tl4 / (NIC_MAX_TL4 / NIC_MAX_TL3); + nic_reg_write(nic, NIC_PF_QSET_0_127_SQ_0_7_CFG2 | + ((u64)vnic << NIC_QS_ID_SHIFT) | + ((u32)sq_idx << NIC_Q_NUM_SHIFT), tl4); + nic_reg_write(nic, NIC_PF_TL4_0_1023_CFG | (tl4 << 3), + ((u64)vnic << 27) | ((u32)sq_idx << 24) | rr_quantum); + + nic_reg_write(nic, NIC_PF_TL3_0_255_CFG | (tl3 << 3), rr_quantum); + chan = (lmac * MAX_BGX_CHANS_PER_LMAC) + (bgx * NIC_CHANS_PER_INF); + nic_reg_write(nic, NIC_PF_TL3_0_255_CHAN | (tl3 << 3), chan); + /* Enable backpressure on the channel */ + nic_reg_write(nic, NIC_PF_CHAN_0_255_TX_CFG | (chan << 3), 1); + + tl2 = tl3 >> 2; + nic_reg_write(nic, NIC_PF_TL3A_0_63_CFG | (tl2 << 3), tl2); + nic_reg_write(nic, NIC_PF_TL2_0_63_CFG | (tl2 << 3), rr_quantum); + /* No priorities as of now */ + nic_reg_write(nic, NIC_PF_TL2_0_63_PRI | (tl2 << 3), 0x00); +} + +/* Send primary nicvf pointer to secondary QS's VF */ +static void nic_send_pnicvf(struct nicpf *nic, int sqs) +{ + union nic_mbx mbx = {}; + + mbx.nicvf.msg = NIC_MBOX_MSG_PNICVF_PTR; + mbx.nicvf.nicvf = nic->nicvf[nic->pqs_vf[sqs]]; + nic_send_msg_to_vf(nic, sqs, &mbx); +} + +/* Send SQS's nicvf pointer to primary QS's VF */ +static void nic_send_snicvf(struct nicpf *nic, struct nicvf_ptr *nicvf) +{ + union nic_mbx mbx = {}; + int sqs_id = nic->vf_sqs[nicvf->vf_id][nicvf->sqs_id]; + + mbx.nicvf.msg = NIC_MBOX_MSG_SNICVF_PTR; + mbx.nicvf.sqs_id = nicvf->sqs_id; + mbx.nicvf.nicvf = nic->nicvf[sqs_id]; + nic_send_msg_to_vf(nic, nicvf->vf_id, &mbx); +} + +/* Find next available Qset that can be assigned as a + * secondary Qset to a VF. + */ +static int nic_nxt_avail_sqs(struct nicpf *nic) +{ + int sqs; + + for (sqs = 0; sqs < nic->num_sqs_en; sqs++) { + if (!nic->sqs_used[sqs]) + nic->sqs_used[sqs] = true; + else + continue; + return sqs + nic->num_vf_en; + } + return -1; +} + +/* Allocate additional Qsets for requested VF */ +static void nic_alloc_sqs(struct nicpf *nic, struct sqs_alloc *sqs) +{ + union nic_mbx mbx = {}; + int idx, alloc_qs = 0; + int sqs_id; + + if (!nic->num_sqs_en) + goto send_mbox; + + for (idx = 0; idx < sqs->qs_count; idx++) { + sqs_id = nic_nxt_avail_sqs(nic); + if (sqs_id < 0) + break; + nic->vf_sqs[sqs->vf_id][idx] = sqs_id; + nic->pqs_vf[sqs_id] = sqs->vf_id; + alloc_qs++; + } + +send_mbox: + mbx.sqs_alloc.msg = NIC_MBOX_MSG_ALLOC_SQS; + mbx.sqs_alloc.vf_id = sqs->vf_id; + mbx.sqs_alloc.qs_count = alloc_qs; + nic_send_msg_to_vf(nic, sqs->vf_id, &mbx); +} + +static int nic_config_loopback(struct nicpf *nic, struct set_loopback *lbk) +{ + int bgx_idx, lmac_idx; + + if (lbk->vf_id > MAX_LMAC) + return -1; + + bgx_idx = NIC_GET_BGX_FROM_VF_LMAC_MAP(nic->vf_lmac_map[lbk->vf_id]); + lmac_idx = NIC_GET_LMAC_FROM_VF_LMAC_MAP(nic->vf_lmac_map[lbk->vf_id]); + + bgx_lmac_internal_loopback(nic->node, bgx_idx, lmac_idx, lbk->enable); + + return 0; +} + +static void nic_enable_vf(struct nicpf *nic, int vf, bool enable) +{ + int bgx, lmac; + + nic->vf_enabled[vf] = enable; + + if (vf >= nic->num_vf_en) + return; + + bgx = NIC_GET_BGX_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vf]); + lmac = NIC_GET_LMAC_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vf]); + + bgx_lmac_rx_tx_enable(nic->node, bgx, lmac, enable); +} + +/* Interrupt handler to handle mailbox messages from VFs */ +static void nic_handle_mbx_intr(struct nicpf *nic, int vf) +{ + union nic_mbx mbx = {}; + u64 *mbx_data; + u64 mbx_addr; + u64 reg_addr; + u64 cfg; + int bgx, lmac; + int i; + int ret = 0; + + nic->mbx_lock[vf] = true; + + mbx_addr = nic_get_mbx_addr(vf); + mbx_data = (u64 *)&mbx; + + for (i = 0; i < NIC_PF_VF_MAILBOX_SIZE; i++) { + *mbx_data = nic_reg_read(nic, mbx_addr); + mbx_data++; + mbx_addr += sizeof(u64); + } + + dev_dbg(&nic->pdev->dev, "%s: Mailbox msg %d from VF%d\n", + __func__, mbx.msg.msg, vf); + switch (mbx.msg.msg) { + case NIC_MBOX_MSG_READY: + nic_mbx_send_ready(nic, vf); + if (vf < MAX_LMAC) { + nic->link[vf] = 0; + nic->duplex[vf] = 0; + nic->speed[vf] = 0; + } + ret = 1; + break; + case NIC_MBOX_MSG_QS_CFG: + reg_addr = NIC_PF_QSET_0_127_CFG | + (mbx.qs.num << NIC_QS_ID_SHIFT); + cfg = mbx.qs.cfg; + /* Check if its a secondary Qset */ + if (vf >= nic->num_vf_en) { + cfg = cfg & (~0x7FULL); + /* Assign this Qset to primary Qset's VF */ + cfg |= nic->pqs_vf[vf]; + } + nic_reg_write(nic, reg_addr, cfg); + break; + case NIC_MBOX_MSG_RQ_CFG: + reg_addr = NIC_PF_QSET_0_127_RQ_0_7_CFG | + (mbx.rq.qs_num << NIC_QS_ID_SHIFT) | + (mbx.rq.rq_num << NIC_Q_NUM_SHIFT); + nic_reg_write(nic, reg_addr, mbx.rq.cfg); + break; + case NIC_MBOX_MSG_RQ_BP_CFG: + reg_addr = NIC_PF_QSET_0_127_RQ_0_7_BP_CFG | + (mbx.rq.qs_num << NIC_QS_ID_SHIFT) | + (mbx.rq.rq_num << NIC_Q_NUM_SHIFT); + nic_reg_write(nic, reg_addr, mbx.rq.cfg); + break; + case NIC_MBOX_MSG_RQ_SW_SYNC: + ret = nic_rcv_queue_sw_sync(nic); + break; + case NIC_MBOX_MSG_RQ_DROP_CFG: + reg_addr = NIC_PF_QSET_0_127_RQ_0_7_DROP_CFG | + (mbx.rq.qs_num << NIC_QS_ID_SHIFT) | + (mbx.rq.rq_num << NIC_Q_NUM_SHIFT); + nic_reg_write(nic, reg_addr, mbx.rq.cfg); + break; + case NIC_MBOX_MSG_SQ_CFG: + reg_addr = NIC_PF_QSET_0_127_SQ_0_7_CFG | + (mbx.sq.qs_num << NIC_QS_ID_SHIFT) | + (mbx.sq.sq_num << NIC_Q_NUM_SHIFT); + nic_reg_write(nic, reg_addr, mbx.sq.cfg); + nic_tx_channel_cfg(nic, mbx.qs.num, &mbx.sq); + break; + case NIC_MBOX_MSG_SET_MAC: + if (vf >= nic->num_vf_en) + break; + lmac = mbx.mac.vf_id; + bgx = NIC_GET_BGX_FROM_VF_LMAC_MAP(nic->vf_lmac_map[lmac]); + lmac = NIC_GET_LMAC_FROM_VF_LMAC_MAP(nic->vf_lmac_map[lmac]); + bgx_set_lmac_mac(nic->node, bgx, lmac, mbx.mac.mac_addr); + break; + case NIC_MBOX_MSG_SET_MAX_FRS: + ret = nic_update_hw_frs(nic, mbx.frs.max_frs, + mbx.frs.vf_id); + break; + case NIC_MBOX_MSG_CPI_CFG: + nic_config_cpi(nic, &mbx.cpi_cfg); + break; + case NIC_MBOX_MSG_RSS_SIZE: + nic_send_rss_size(nic, vf); + goto unlock; + case NIC_MBOX_MSG_RSS_CFG: + case NIC_MBOX_MSG_RSS_CFG_CONT: + nic_config_rss(nic, &mbx.rss_cfg); + break; + case NIC_MBOX_MSG_CFG_DONE: + /* Last message of VF config msg sequence */ + nic_enable_vf(nic, vf, true); + goto unlock; + case NIC_MBOX_MSG_SHUTDOWN: + /* First msg in VF teardown sequence */ + if (vf >= nic->num_vf_en) + nic->sqs_used[vf - nic->num_vf_en] = false; + nic->pqs_vf[vf] = 0; + nic_enable_vf(nic, vf, false); + break; + case NIC_MBOX_MSG_ALLOC_SQS: + nic_alloc_sqs(nic, &mbx.sqs_alloc); + goto unlock; + case NIC_MBOX_MSG_NICVF_PTR: + nic->nicvf[vf] = mbx.nicvf.nicvf; + break; + case NIC_MBOX_MSG_PNICVF_PTR: + nic_send_pnicvf(nic, vf); + goto unlock; + case NIC_MBOX_MSG_SNICVF_PTR: + nic_send_snicvf(nic, &mbx.nicvf); + goto unlock; + case NIC_MBOX_MSG_BGX_STATS: + nic_get_bgx_stats(nic, &mbx.bgx_stats); + goto unlock; + case NIC_MBOX_MSG_LOOPBACK: + ret = nic_config_loopback(nic, &mbx.lbk); + break; + default: + dev_err(&nic->pdev->dev, + "Invalid msg from VF%d, msg 0x%x\n", vf, mbx.msg.msg); + break; + } + + if (!ret) + nic_mbx_send_ack(nic, vf); + else if (mbx.msg.msg != NIC_MBOX_MSG_READY) + nic_mbx_send_nack(nic, vf); +unlock: + nic->mbx_lock[vf] = false; +} + +static void nic_mbx_intr_handler (struct nicpf *nic, int mbx) +{ + u64 intr; + u8 vf, vf_per_mbx_reg = 64; + + intr = nic_reg_read(nic, NIC_PF_MAILBOX_INT + (mbx << 3)); + dev_dbg(&nic->pdev->dev, "PF interrupt Mbox%d 0x%llx\n", mbx, intr); + for (vf = 0; vf < vf_per_mbx_reg; vf++) { + if (intr & (1ULL << vf)) { + dev_dbg(&nic->pdev->dev, "Intr from VF %d\n", + vf + (mbx * vf_per_mbx_reg)); + + nic_handle_mbx_intr(nic, vf + (mbx * vf_per_mbx_reg)); + nic_clear_mbx_intr(nic, vf, mbx); + } + } +} + +static irqreturn_t nic_mbx0_intr_handler (int irq, void *nic_irq) +{ + struct nicpf *nic = (struct nicpf *)nic_irq; + + nic_mbx_intr_handler(nic, 0); + + return IRQ_HANDLED; +} + +static irqreturn_t nic_mbx1_intr_handler (int irq, void *nic_irq) +{ + struct nicpf *nic = (struct nicpf *)nic_irq; + + nic_mbx_intr_handler(nic, 1); + + return IRQ_HANDLED; +} + +static int nic_enable_msix(struct nicpf *nic) +{ + int i, ret; + + nic->num_vec = NIC_PF_MSIX_VECTORS; + + for (i = 0; i < nic->num_vec; i++) + nic->msix_entries[i].entry = i; + + ret = pci_enable_msix(nic->pdev, nic->msix_entries, nic->num_vec); + if (ret) { + dev_err(&nic->pdev->dev, + "Request for #%d msix vectors failed\n", + nic->num_vec); + return ret; + } + + nic->msix_enabled = 1; + return 0; +} + +static void nic_disable_msix(struct nicpf *nic) +{ + if (nic->msix_enabled) { + pci_disable_msix(nic->pdev); + nic->msix_enabled = 0; + nic->num_vec = 0; + } +} + +static void nic_free_all_interrupts(struct nicpf *nic) +{ + int irq; + + for (irq = 0; irq < nic->num_vec; irq++) { + if (nic->irq_allocated[irq]) + free_irq(nic->msix_entries[irq].vector, nic); + nic->irq_allocated[irq] = false; + } +} + +static int nic_register_interrupts(struct nicpf *nic) +{ + int ret; + + /* Enable MSI-X */ + ret = nic_enable_msix(nic); + if (ret) + return ret; + + /* Register mailbox interrupt handlers */ + ret = request_irq(nic->msix_entries[NIC_PF_INTR_ID_MBOX0].vector, + nic_mbx0_intr_handler, 0, "NIC Mbox0", nic); + if (ret) + goto fail; + + nic->irq_allocated[NIC_PF_INTR_ID_MBOX0] = true; + + ret = request_irq(nic->msix_entries[NIC_PF_INTR_ID_MBOX1].vector, + nic_mbx1_intr_handler, 0, "NIC Mbox1", nic); + if (ret) + goto fail; + + nic->irq_allocated[NIC_PF_INTR_ID_MBOX1] = true; + + /* Enable mailbox interrupt */ + nic_enable_mbx_intr(nic); + return 0; + +fail: + dev_err(&nic->pdev->dev, "Request irq failed\n"); + nic_free_all_interrupts(nic); + return ret; +} + +static void nic_unregister_interrupts(struct nicpf *nic) +{ + nic_free_all_interrupts(nic); + nic_disable_msix(nic); +} + +static int nic_num_sqs_en(struct nicpf *nic, int vf_en) +{ + int pos, sqs_per_vf = MAX_SQS_PER_VF_SINGLE_NODE; + u16 total_vf; + + /* Check if its a multi-node environment */ + if (nr_node_ids > 1) + sqs_per_vf = MAX_SQS_PER_VF; + + pos = pci_find_ext_capability(nic->pdev, PCI_EXT_CAP_ID_SRIOV); + pci_read_config_word(nic->pdev, (pos + PCI_SRIOV_TOTAL_VF), &total_vf); + return min(total_vf - vf_en, vf_en * sqs_per_vf); +} + +static int nic_sriov_init(struct pci_dev *pdev, struct nicpf *nic) +{ + int pos = 0; + int vf_en; + int err; + u16 total_vf_cnt; + + pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV); + if (!pos) { + dev_err(&pdev->dev, "SRIOV capability is not found in PCIe config space\n"); + return -ENODEV; + } + + pci_read_config_word(pdev, (pos + PCI_SRIOV_TOTAL_VF), &total_vf_cnt); + if (total_vf_cnt < nic->num_vf_en) + nic->num_vf_en = total_vf_cnt; + + if (!total_vf_cnt) + return 0; + + vf_en = nic->num_vf_en; + nic->num_sqs_en = nic_num_sqs_en(nic, nic->num_vf_en); + vf_en += nic->num_sqs_en; + + err = pci_enable_sriov(pdev, vf_en); + if (err) { + dev_err(&pdev->dev, "SRIOV enable failed, num VF is %d\n", + vf_en); + nic->num_vf_en = 0; + return err; + } + + dev_info(&pdev->dev, "SRIOV enabled, number of VF available %d\n", + vf_en); + + nic->flags |= NIC_SRIOV_ENABLED; + return 0; +} + +/* Poll for BGX LMAC link status and update corresponding VF + * if there is a change, valid only if internal L2 switch + * is not present otherwise VF link is always treated as up + */ +static void nic_poll_for_link(struct work_struct *work) +{ + union nic_mbx mbx = {}; + struct nicpf *nic; + struct bgx_link_status link; + u8 vf, bgx, lmac; + + nic = container_of(work, struct nicpf, dwork.work); + + mbx.link_status.msg = NIC_MBOX_MSG_BGX_LINK_CHANGE; + + for (vf = 0; vf < nic->num_vf_en; vf++) { + /* Poll only if VF is UP */ + if (!nic->vf_enabled[vf]) + continue; + + /* Get BGX, LMAC indices for the VF */ + bgx = NIC_GET_BGX_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vf]); + lmac = NIC_GET_LMAC_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vf]); + /* Get interface link status */ + bgx_get_lmac_link_state(nic->node, bgx, lmac, &link); + + /* Inform VF only if link status changed */ + if (nic->link[vf] == link.link_up) + continue; + + if (!nic->mbx_lock[vf]) { + nic->link[vf] = link.link_up; + nic->duplex[vf] = link.duplex; + nic->speed[vf] = link.speed; + + /* Send a mbox message to VF with current link status */ + mbx.link_status.link_up = link.link_up; + mbx.link_status.duplex = link.duplex; + mbx.link_status.speed = link.speed; + nic_send_msg_to_vf(nic, vf, &mbx); + } + } + queue_delayed_work(nic->check_link, &nic->dwork, HZ * 2); +} + +static int nic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct device *dev = &pdev->dev; + struct nicpf *nic; + int err; + + BUILD_BUG_ON(sizeof(union nic_mbx) > 16); + + nic = devm_kzalloc(dev, sizeof(*nic), GFP_KERNEL); + if (!nic) + return -ENOMEM; + + pci_set_drvdata(pdev, nic); + + nic->pdev = pdev; + + err = pci_enable_device(pdev); + if (err) { + dev_err(dev, "Failed to enable PCI device\n"); + pci_set_drvdata(pdev, NULL); + return err; + } + + err = pci_request_regions(pdev, DRV_NAME); + if (err) { + dev_err(dev, "PCI request regions failed 0x%x\n", err); + goto err_disable_device; + } + + err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48)); + if (err) { + dev_err(dev, "Unable to get usable DMA configuration\n"); + goto err_release_regions; + } + + err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48)); + if (err) { + dev_err(dev, "Unable to get 48-bit DMA for consistent allocations\n"); + goto err_release_regions; + } + + /* MAP PF's configuration registers */ + nic->reg_base = pcim_iomap(pdev, PCI_CFG_REG_BAR_NUM, 0); + if (!nic->reg_base) { + dev_err(dev, "Cannot map config register space, aborting\n"); + err = -ENOMEM; + goto err_release_regions; + } + + nic->node = nic_get_node_id(pdev); + + nic_set_lmac_vf_mapping(nic); + + /* Initialize hardware */ + nic_init_hw(nic); + + /* Set RSS TBL size for each VF */ + nic->rss_ind_tbl_size = NIC_MAX_RSS_IDR_TBL_SIZE; + + /* Register interrupts */ + err = nic_register_interrupts(nic); + if (err) + goto err_release_regions; + + /* Configure SRIOV */ + err = nic_sriov_init(pdev, nic); + if (err) + goto err_unregister_interrupts; + + /* Register a physical link status poll fn() */ + nic->check_link = alloc_workqueue("check_link_status", + WQ_UNBOUND | WQ_MEM_RECLAIM, 1); + if (!nic->check_link) { + err = -ENOMEM; + goto err_disable_sriov; + } + + INIT_DELAYED_WORK(&nic->dwork, nic_poll_for_link); + queue_delayed_work(nic->check_link, &nic->dwork, 0); + + return 0; + +err_disable_sriov: + if (nic->flags & NIC_SRIOV_ENABLED) + pci_disable_sriov(pdev); +err_unregister_interrupts: + nic_unregister_interrupts(nic); +err_release_regions: + pci_release_regions(pdev); +err_disable_device: + pci_disable_device(pdev); + pci_set_drvdata(pdev, NULL); + return err; +} + +static void nic_remove(struct pci_dev *pdev) +{ + struct nicpf *nic = pci_get_drvdata(pdev); + + if (nic->flags & NIC_SRIOV_ENABLED) + pci_disable_sriov(pdev); + + if (nic->check_link) { + /* Destroy work Queue */ + cancel_delayed_work_sync(&nic->dwork); + destroy_workqueue(nic->check_link); + } + + nic_unregister_interrupts(nic); + pci_release_regions(pdev); + pci_disable_device(pdev); + pci_set_drvdata(pdev, NULL); +} + +static struct pci_driver nic_driver = { + .name = DRV_NAME, + .id_table = nic_id_table, + .probe = nic_probe, + .remove = nic_remove, +}; + +static int __init nic_init_module(void) +{ + pr_info("%s, ver %s\n", DRV_NAME, DRV_VERSION); + + return pci_register_driver(&nic_driver); +} + +static void __exit nic_cleanup_module(void) +{ + pci_unregister_driver(&nic_driver); +} + +module_init(nic_init_module); +module_exit(nic_cleanup_module); diff --git a/kernel/drivers/net/ethernet/cavium/thunder/nic_reg.h b/kernel/drivers/net/ethernet/cavium/thunder/nic_reg.h new file mode 100644 index 000000000..dd536be20 --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/thunder/nic_reg.h @@ -0,0 +1,217 @@ +/* + * Copyright (C) 2015 Cavium, Inc. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of version 2 of the GNU General Public License + * as published by the Free Software Foundation. + */ + +#ifndef NIC_REG_H +#define NIC_REG_H + +#define NIC_PF_REG_COUNT 29573 +#define NIC_VF_REG_COUNT 249 + +/* Physical function register offsets */ +#define NIC_PF_CFG (0x0000) +#define NIC_PF_STATUS (0x0010) +#define NIC_PF_INTR_TIMER_CFG (0x0030) +#define NIC_PF_BIST_STATUS (0x0040) +#define NIC_PF_SOFT_RESET (0x0050) +#define NIC_PF_TCP_TIMER (0x0060) +#define NIC_PF_BP_CFG (0x0080) +#define NIC_PF_RRM_CFG (0x0088) +#define NIC_PF_CQM_CF (0x00A0) +#define NIC_PF_CNM_CF (0x00A8) +#define NIC_PF_CNM_STATUS (0x00B0) +#define NIC_PF_CQ_AVG_CFG (0x00C0) +#define NIC_PF_RRM_AVG_CFG (0x00C8) +#define NIC_PF_INTF_0_1_SEND_CFG (0x0200) +#define NIC_PF_INTF_0_1_BP_CFG (0x0208) +#define NIC_PF_INTF_0_1_BP_DIS_0_1 (0x0210) +#define NIC_PF_INTF_0_1_BP_SW_0_1 (0x0220) +#define NIC_PF_RBDR_BP_STATE_0_3 (0x0240) +#define NIC_PF_MAILBOX_INT (0x0410) +#define NIC_PF_MAILBOX_INT_W1S (0x0430) +#define NIC_PF_MAILBOX_ENA_W1C (0x0450) +#define NIC_PF_MAILBOX_ENA_W1S (0x0470) +#define NIC_PF_RX_ETYPE_0_7 (0x0500) +#define NIC_PF_PKIND_0_15_CFG (0x0600) +#define NIC_PF_ECC0_FLIP0 (0x1000) +#define NIC_PF_ECC1_FLIP0 (0x1008) +#define NIC_PF_ECC2_FLIP0 (0x1010) +#define NIC_PF_ECC3_FLIP0 (0x1018) +#define NIC_PF_ECC0_FLIP1 (0x1080) +#define NIC_PF_ECC1_FLIP1 (0x1088) +#define NIC_PF_ECC2_FLIP1 (0x1090) +#define NIC_PF_ECC3_FLIP1 (0x1098) +#define NIC_PF_ECC0_CDIS (0x1100) +#define NIC_PF_ECC1_CDIS (0x1108) +#define NIC_PF_ECC2_CDIS (0x1110) +#define NIC_PF_ECC3_CDIS (0x1118) +#define NIC_PF_BIST0_STATUS (0x1280) +#define NIC_PF_BIST1_STATUS (0x1288) +#define NIC_PF_BIST2_STATUS (0x1290) +#define NIC_PF_BIST3_STATUS (0x1298) +#define NIC_PF_ECC0_SBE_INT (0x2000) +#define NIC_PF_ECC0_SBE_INT_W1S (0x2008) +#define NIC_PF_ECC0_SBE_ENA_W1C (0x2010) +#define NIC_PF_ECC0_SBE_ENA_W1S (0x2018) +#define NIC_PF_ECC0_DBE_INT (0x2100) +#define NIC_PF_ECC0_DBE_INT_W1S (0x2108) +#define NIC_PF_ECC0_DBE_ENA_W1C (0x2110) +#define NIC_PF_ECC0_DBE_ENA_W1S (0x2118) +#define NIC_PF_ECC1_SBE_INT (0x2200) +#define NIC_PF_ECC1_SBE_INT_W1S (0x2208) +#define NIC_PF_ECC1_SBE_ENA_W1C (0x2210) +#define NIC_PF_ECC1_SBE_ENA_W1S (0x2218) +#define NIC_PF_ECC1_DBE_INT (0x2300) +#define NIC_PF_ECC1_DBE_INT_W1S (0x2308) +#define NIC_PF_ECC1_DBE_ENA_W1C (0x2310) +#define NIC_PF_ECC1_DBE_ENA_W1S (0x2318) +#define NIC_PF_ECC2_SBE_INT (0x2400) +#define NIC_PF_ECC2_SBE_INT_W1S (0x2408) +#define NIC_PF_ECC2_SBE_ENA_W1C (0x2410) +#define NIC_PF_ECC2_SBE_ENA_W1S (0x2418) +#define NIC_PF_ECC2_DBE_INT (0x2500) +#define NIC_PF_ECC2_DBE_INT_W1S (0x2508) +#define NIC_PF_ECC2_DBE_ENA_W1C (0x2510) +#define NIC_PF_ECC2_DBE_ENA_W1S (0x2518) +#define NIC_PF_ECC3_SBE_INT (0x2600) +#define NIC_PF_ECC3_SBE_INT_W1S (0x2608) +#define NIC_PF_ECC3_SBE_ENA_W1C (0x2610) +#define NIC_PF_ECC3_SBE_ENA_W1S (0x2618) +#define NIC_PF_ECC3_DBE_INT (0x2700) +#define NIC_PF_ECC3_DBE_INT_W1S (0x2708) +#define NIC_PF_ECC3_DBE_ENA_W1C (0x2710) +#define NIC_PF_ECC3_DBE_ENA_W1S (0x2718) +#define NIC_PF_MCAM_0_191_ENA (0x100000) +#define NIC_PF_MCAM_0_191_M_0_5_DATA (0x110000) +#define NIC_PF_MCAM_CTRL (0x120000) +#define NIC_PF_CPI_0_2047_CFG (0x200000) +#define NIC_PF_MPI_0_2047_CFG (0x210000) +#define NIC_PF_RSSI_0_4097_RQ (0x220000) +#define NIC_PF_LMAC_0_7_CFG (0x240000) +#define NIC_PF_LMAC_0_7_SW_XOFF (0x242000) +#define NIC_PF_LMAC_0_7_CREDIT (0x244000) +#define NIC_PF_CHAN_0_255_TX_CFG (0x400000) +#define NIC_PF_CHAN_0_255_RX_CFG (0x420000) +#define NIC_PF_CHAN_0_255_SW_XOFF (0x440000) +#define NIC_PF_CHAN_0_255_CREDIT (0x460000) +#define NIC_PF_CHAN_0_255_RX_BP_CFG (0x480000) +#define NIC_PF_SW_SYNC_RX (0x490000) +#define NIC_PF_SW_SYNC_RX_DONE (0x490008) +#define NIC_PF_TL2_0_63_CFG (0x500000) +#define NIC_PF_TL2_0_63_PRI (0x520000) +#define NIC_PF_TL2_0_63_SH_STATUS (0x580000) +#define NIC_PF_TL3A_0_63_CFG (0x5F0000) +#define NIC_PF_TL3_0_255_CFG (0x600000) +#define NIC_PF_TL3_0_255_CHAN (0x620000) +#define NIC_PF_TL3_0_255_PIR (0x640000) +#define NIC_PF_TL3_0_255_SW_XOFF (0x660000) +#define NIC_PF_TL3_0_255_CNM_RATE (0x680000) +#define NIC_PF_TL3_0_255_SH_STATUS (0x6A0000) +#define NIC_PF_TL4A_0_255_CFG (0x6F0000) +#define NIC_PF_TL4_0_1023_CFG (0x800000) +#define NIC_PF_TL4_0_1023_SW_XOFF (0x820000) +#define NIC_PF_TL4_0_1023_SH_STATUS (0x840000) +#define NIC_PF_TL4A_0_1023_CNM_RATE (0x880000) +#define NIC_PF_TL4A_0_1023_CNM_STATUS (0x8A0000) +#define NIC_PF_VF_0_127_MAILBOX_0_1 (0x20002030) +#define NIC_PF_VNIC_0_127_TX_STAT_0_4 (0x20004000) +#define NIC_PF_VNIC_0_127_RX_STAT_0_13 (0x20004100) +#define NIC_PF_QSET_0_127_LOCK_0_15 (0x20006000) +#define NIC_PF_QSET_0_127_CFG (0x20010000) +#define NIC_PF_QSET_0_127_RQ_0_7_CFG (0x20010400) +#define NIC_PF_QSET_0_127_RQ_0_7_DROP_CFG (0x20010420) +#define NIC_PF_QSET_0_127_RQ_0_7_BP_CFG (0x20010500) +#define NIC_PF_QSET_0_127_RQ_0_7_STAT_0_1 (0x20010600) +#define NIC_PF_QSET_0_127_SQ_0_7_CFG (0x20010C00) +#define NIC_PF_QSET_0_127_SQ_0_7_CFG2 (0x20010C08) +#define NIC_PF_QSET_0_127_SQ_0_7_STAT_0_1 (0x20010D00) + +#define NIC_PF_MSIX_VEC_0_18_ADDR (0x000000) +#define NIC_PF_MSIX_VEC_0_CTL (0x000008) +#define NIC_PF_MSIX_PBA_0 (0x0F0000) + +/* Virtual function register offsets */ +#define NIC_VNIC_CFG (0x000020) +#define NIC_VF_PF_MAILBOX_0_1 (0x000130) +#define NIC_VF_INT (0x000200) +#define NIC_VF_INT_W1S (0x000220) +#define NIC_VF_ENA_W1C (0x000240) +#define NIC_VF_ENA_W1S (0x000260) + +#define NIC_VNIC_RSS_CFG (0x0020E0) +#define NIC_VNIC_RSS_KEY_0_4 (0x002200) +#define NIC_VNIC_TX_STAT_0_4 (0x004000) +#define NIC_VNIC_RX_STAT_0_13 (0x004100) +#define NIC_QSET_RQ_GEN_CFG (0x010010) + +#define NIC_QSET_CQ_0_7_CFG (0x010400) +#define NIC_QSET_CQ_0_7_CFG2 (0x010408) +#define NIC_QSET_CQ_0_7_THRESH (0x010410) +#define NIC_QSET_CQ_0_7_BASE (0x010420) +#define NIC_QSET_CQ_0_7_HEAD (0x010428) +#define NIC_QSET_CQ_0_7_TAIL (0x010430) +#define NIC_QSET_CQ_0_7_DOOR (0x010438) +#define NIC_QSET_CQ_0_7_STATUS (0x010440) +#define NIC_QSET_CQ_0_7_STATUS2 (0x010448) +#define NIC_QSET_CQ_0_7_DEBUG (0x010450) + +#define NIC_QSET_RQ_0_7_CFG (0x010600) +#define NIC_QSET_RQ_0_7_STAT_0_1 (0x010700) + +#define NIC_QSET_SQ_0_7_CFG (0x010800) +#define NIC_QSET_SQ_0_7_THRESH (0x010810) +#define NIC_QSET_SQ_0_7_BASE (0x010820) +#define NIC_QSET_SQ_0_7_HEAD (0x010828) +#define NIC_QSET_SQ_0_7_TAIL (0x010830) +#define NIC_QSET_SQ_0_7_DOOR (0x010838) +#define NIC_QSET_SQ_0_7_STATUS (0x010840) +#define NIC_QSET_SQ_0_7_DEBUG (0x010848) +#define NIC_QSET_SQ_0_7_CNM_CHG (0x010860) +#define NIC_QSET_SQ_0_7_STAT_0_1 (0x010900) + +#define NIC_QSET_RBDR_0_1_CFG (0x010C00) +#define NIC_QSET_RBDR_0_1_THRESH (0x010C10) +#define NIC_QSET_RBDR_0_1_BASE (0x010C20) +#define NIC_QSET_RBDR_0_1_HEAD (0x010C28) +#define NIC_QSET_RBDR_0_1_TAIL (0x010C30) +#define NIC_QSET_RBDR_0_1_DOOR (0x010C38) +#define NIC_QSET_RBDR_0_1_STATUS0 (0x010C40) +#define NIC_QSET_RBDR_0_1_STATUS1 (0x010C48) +#define NIC_QSET_RBDR_0_1_PREFETCH_STATUS (0x010C50) + +#define NIC_VF_MSIX_VECTOR_0_19_ADDR (0x000000) +#define NIC_VF_MSIX_VECTOR_0_19_CTL (0x000008) +#define NIC_VF_MSIX_PBA (0x0F0000) + +/* Offsets within registers */ +#define NIC_MSIX_VEC_SHIFT 4 +#define NIC_Q_NUM_SHIFT 18 +#define NIC_QS_ID_SHIFT 21 +#define NIC_VF_NUM_SHIFT 21 + +/* Port kind configuration register */ +struct pkind_cfg { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_42_63:22; + u64 hdr_sl:5; /* Header skip length */ + u64 rx_hdr:3; /* TNS Receive header present */ + u64 lenerr_en:1;/* L2 length error check enable */ + u64 reserved_32_32:1; + u64 maxlen:16; /* Max frame size */ + u64 minlen:16; /* Min frame size */ +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 minlen:16; + u64 maxlen:16; + u64 reserved_32_32:1; + u64 lenerr_en:1; + u64 rx_hdr:3; + u64 hdr_sl:5; + u64 reserved_42_63:22; +#endif +}; + +#endif /* NIC_REG_H */ diff --git a/kernel/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c b/kernel/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c new file mode 100644 index 000000000..a12b2e38c --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c @@ -0,0 +1,702 @@ +/* + * Copyright (C) 2015 Cavium, Inc. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of version 2 of the GNU General Public License + * as published by the Free Software Foundation. + */ + +/* ETHTOOL Support for VNIC_VF Device*/ + +#include <linux/pci.h> + +#include "nic_reg.h" +#include "nic.h" +#include "nicvf_queues.h" +#include "q_struct.h" +#include "thunder_bgx.h" + +#define DRV_NAME "thunder-nicvf" +#define DRV_VERSION "1.0" + +struct nicvf_stat { + char name[ETH_GSTRING_LEN]; + unsigned int index; +}; + +#define NICVF_HW_STAT(stat) { \ + .name = #stat, \ + .index = offsetof(struct nicvf_hw_stats, stat) / sizeof(u64), \ +} + +#define NICVF_DRV_STAT(stat) { \ + .name = #stat, \ + .index = offsetof(struct nicvf_drv_stats, stat) / sizeof(u64), \ +} + +static const struct nicvf_stat nicvf_hw_stats[] = { + NICVF_HW_STAT(rx_bytes), + NICVF_HW_STAT(rx_ucast_frames), + NICVF_HW_STAT(rx_bcast_frames), + NICVF_HW_STAT(rx_mcast_frames), + NICVF_HW_STAT(rx_fcs_errors), + NICVF_HW_STAT(rx_l2_errors), + NICVF_HW_STAT(rx_drop_red), + NICVF_HW_STAT(rx_drop_red_bytes), + NICVF_HW_STAT(rx_drop_overrun), + NICVF_HW_STAT(rx_drop_overrun_bytes), + NICVF_HW_STAT(rx_drop_bcast), + NICVF_HW_STAT(rx_drop_mcast), + NICVF_HW_STAT(rx_drop_l3_bcast), + NICVF_HW_STAT(rx_drop_l3_mcast), + NICVF_HW_STAT(rx_bgx_truncated_pkts), + NICVF_HW_STAT(rx_jabber_errs), + NICVF_HW_STAT(rx_fcs_errs), + NICVF_HW_STAT(rx_bgx_errs), + NICVF_HW_STAT(rx_prel2_errs), + NICVF_HW_STAT(rx_l2_hdr_malformed), + NICVF_HW_STAT(rx_oversize), + NICVF_HW_STAT(rx_undersize), + NICVF_HW_STAT(rx_l2_len_mismatch), + NICVF_HW_STAT(rx_l2_pclp), + NICVF_HW_STAT(rx_ip_ver_errs), + NICVF_HW_STAT(rx_ip_csum_errs), + NICVF_HW_STAT(rx_ip_hdr_malformed), + NICVF_HW_STAT(rx_ip_payload_malformed), + NICVF_HW_STAT(rx_ip_ttl_errs), + NICVF_HW_STAT(rx_l3_pclp), + NICVF_HW_STAT(rx_l4_malformed), + NICVF_HW_STAT(rx_l4_csum_errs), + NICVF_HW_STAT(rx_udp_len_errs), + NICVF_HW_STAT(rx_l4_port_errs), + NICVF_HW_STAT(rx_tcp_flag_errs), + NICVF_HW_STAT(rx_tcp_offset_errs), + NICVF_HW_STAT(rx_l4_pclp), + NICVF_HW_STAT(rx_truncated_pkts), + NICVF_HW_STAT(tx_bytes_ok), + NICVF_HW_STAT(tx_ucast_frames_ok), + NICVF_HW_STAT(tx_bcast_frames_ok), + NICVF_HW_STAT(tx_mcast_frames_ok), +}; + +static const struct nicvf_stat nicvf_drv_stats[] = { + NICVF_DRV_STAT(rx_frames_ok), + NICVF_DRV_STAT(rx_frames_64), + NICVF_DRV_STAT(rx_frames_127), + NICVF_DRV_STAT(rx_frames_255), + NICVF_DRV_STAT(rx_frames_511), + NICVF_DRV_STAT(rx_frames_1023), + NICVF_DRV_STAT(rx_frames_1518), + NICVF_DRV_STAT(rx_frames_jumbo), + NICVF_DRV_STAT(rx_drops), + NICVF_DRV_STAT(tx_frames_ok), + NICVF_DRV_STAT(tx_tso), + NICVF_DRV_STAT(tx_drops), + NICVF_DRV_STAT(txq_stop), + NICVF_DRV_STAT(txq_wake), +}; + +static const struct nicvf_stat nicvf_queue_stats[] = { + { "bytes", 0 }, + { "frames", 1 }, +}; + +static const unsigned int nicvf_n_hw_stats = ARRAY_SIZE(nicvf_hw_stats); +static const unsigned int nicvf_n_drv_stats = ARRAY_SIZE(nicvf_drv_stats); +static const unsigned int nicvf_n_queue_stats = ARRAY_SIZE(nicvf_queue_stats); + +static int nicvf_get_settings(struct net_device *netdev, + struct ethtool_cmd *cmd) +{ + struct nicvf *nic = netdev_priv(netdev); + + cmd->supported = 0; + cmd->transceiver = XCVR_EXTERNAL; + + if (!nic->link_up) { + cmd->duplex = DUPLEX_UNKNOWN; + ethtool_cmd_speed_set(cmd, SPEED_UNKNOWN); + return 0; + } + + if (nic->speed <= 1000) { + cmd->port = PORT_MII; + cmd->autoneg = AUTONEG_ENABLE; + } else { + cmd->port = PORT_FIBRE; + cmd->autoneg = AUTONEG_DISABLE; + } + cmd->duplex = nic->duplex; + ethtool_cmd_speed_set(cmd, nic->speed); + + return 0; +} + +static u32 nicvf_get_link(struct net_device *netdev) +{ + struct nicvf *nic = netdev_priv(netdev); + + return nic->link_up; +} + +static void nicvf_get_drvinfo(struct net_device *netdev, + struct ethtool_drvinfo *info) +{ + struct nicvf *nic = netdev_priv(netdev); + + strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); + strlcpy(info->version, DRV_VERSION, sizeof(info->version)); + strlcpy(info->bus_info, pci_name(nic->pdev), sizeof(info->bus_info)); +} + +static u32 nicvf_get_msglevel(struct net_device *netdev) +{ + struct nicvf *nic = netdev_priv(netdev); + + return nic->msg_enable; +} + +static void nicvf_set_msglevel(struct net_device *netdev, u32 lvl) +{ + struct nicvf *nic = netdev_priv(netdev); + + nic->msg_enable = lvl; +} + +static void nicvf_get_qset_strings(struct nicvf *nic, u8 **data, int qset) +{ + int stats, qidx; + int start_qidx = qset * MAX_RCV_QUEUES_PER_QS; + + for (qidx = 0; qidx < nic->qs->rq_cnt; qidx++) { + for (stats = 0; stats < nicvf_n_queue_stats; stats++) { + sprintf(*data, "rxq%d: %s", qidx + start_qidx, + nicvf_queue_stats[stats].name); + *data += ETH_GSTRING_LEN; + } + } + + for (qidx = 0; qidx < nic->qs->sq_cnt; qidx++) { + for (stats = 0; stats < nicvf_n_queue_stats; stats++) { + sprintf(*data, "txq%d: %s", qidx + start_qidx, + nicvf_queue_stats[stats].name); + *data += ETH_GSTRING_LEN; + } + } +} + +static void nicvf_get_strings(struct net_device *netdev, u32 sset, u8 *data) +{ + struct nicvf *nic = netdev_priv(netdev); + int stats; + int sqs; + + if (sset != ETH_SS_STATS) + return; + + for (stats = 0; stats < nicvf_n_hw_stats; stats++) { + memcpy(data, nicvf_hw_stats[stats].name, ETH_GSTRING_LEN); + data += ETH_GSTRING_LEN; + } + + for (stats = 0; stats < nicvf_n_drv_stats; stats++) { + memcpy(data, nicvf_drv_stats[stats].name, ETH_GSTRING_LEN); + data += ETH_GSTRING_LEN; + } + + nicvf_get_qset_strings(nic, &data, 0); + + for (sqs = 0; sqs < nic->sqs_count; sqs++) { + if (!nic->snicvf[sqs]) + continue; + nicvf_get_qset_strings(nic->snicvf[sqs], &data, sqs + 1); + } + + for (stats = 0; stats < BGX_RX_STATS_COUNT; stats++) { + sprintf(data, "bgx_rxstat%d: ", stats); + data += ETH_GSTRING_LEN; + } + + for (stats = 0; stats < BGX_TX_STATS_COUNT; stats++) { + sprintf(data, "bgx_txstat%d: ", stats); + data += ETH_GSTRING_LEN; + } +} + +static int nicvf_get_sset_count(struct net_device *netdev, int sset) +{ + struct nicvf *nic = netdev_priv(netdev); + int qstats_count; + int sqs; + + if (sset != ETH_SS_STATS) + return -EINVAL; + + qstats_count = nicvf_n_queue_stats * + (nic->qs->rq_cnt + nic->qs->sq_cnt); + for (sqs = 0; sqs < nic->sqs_count; sqs++) { + struct nicvf *snic; + + snic = nic->snicvf[sqs]; + if (!snic) + continue; + qstats_count += nicvf_n_queue_stats * + (snic->qs->rq_cnt + snic->qs->sq_cnt); + } + + return nicvf_n_hw_stats + nicvf_n_drv_stats + + qstats_count + + BGX_RX_STATS_COUNT + BGX_TX_STATS_COUNT; +} + +static void nicvf_get_qset_stats(struct nicvf *nic, + struct ethtool_stats *stats, u64 **data) +{ + int stat, qidx; + + if (!nic) + return; + + for (qidx = 0; qidx < nic->qs->rq_cnt; qidx++) { + nicvf_update_rq_stats(nic, qidx); + for (stat = 0; stat < nicvf_n_queue_stats; stat++) + *((*data)++) = ((u64 *)&nic->qs->rq[qidx].stats) + [nicvf_queue_stats[stat].index]; + } + + for (qidx = 0; qidx < nic->qs->sq_cnt; qidx++) { + nicvf_update_sq_stats(nic, qidx); + for (stat = 0; stat < nicvf_n_queue_stats; stat++) + *((*data)++) = ((u64 *)&nic->qs->sq[qidx].stats) + [nicvf_queue_stats[stat].index]; + } +} + +static void nicvf_get_ethtool_stats(struct net_device *netdev, + struct ethtool_stats *stats, u64 *data) +{ + struct nicvf *nic = netdev_priv(netdev); + int stat; + int sqs; + + nicvf_update_stats(nic); + + /* Update LMAC stats */ + nicvf_update_lmac_stats(nic); + + for (stat = 0; stat < nicvf_n_hw_stats; stat++) + *(data++) = ((u64 *)&nic->hw_stats) + [nicvf_hw_stats[stat].index]; + for (stat = 0; stat < nicvf_n_drv_stats; stat++) + *(data++) = ((u64 *)&nic->drv_stats) + [nicvf_drv_stats[stat].index]; + + nicvf_get_qset_stats(nic, stats, &data); + + for (sqs = 0; sqs < nic->sqs_count; sqs++) { + if (!nic->snicvf[sqs]) + continue; + nicvf_get_qset_stats(nic->snicvf[sqs], stats, &data); + } + + for (stat = 0; stat < BGX_RX_STATS_COUNT; stat++) + *(data++) = nic->bgx_stats.rx_stats[stat]; + for (stat = 0; stat < BGX_TX_STATS_COUNT; stat++) + *(data++) = nic->bgx_stats.tx_stats[stat]; +} + +static int nicvf_get_regs_len(struct net_device *dev) +{ + return sizeof(u64) * NIC_VF_REG_COUNT; +} + +static void nicvf_get_regs(struct net_device *dev, + struct ethtool_regs *regs, void *reg) +{ + struct nicvf *nic = netdev_priv(dev); + u64 *p = (u64 *)reg; + u64 reg_offset; + int mbox, key, stat, q; + int i = 0; + + regs->version = 0; + memset(p, 0, NIC_VF_REG_COUNT); + + p[i++] = nicvf_reg_read(nic, NIC_VNIC_CFG); + /* Mailbox registers */ + for (mbox = 0; mbox < NIC_PF_VF_MAILBOX_SIZE; mbox++) + p[i++] = nicvf_reg_read(nic, + NIC_VF_PF_MAILBOX_0_1 | (mbox << 3)); + + p[i++] = nicvf_reg_read(nic, NIC_VF_INT); + p[i++] = nicvf_reg_read(nic, NIC_VF_INT_W1S); + p[i++] = nicvf_reg_read(nic, NIC_VF_ENA_W1C); + p[i++] = nicvf_reg_read(nic, NIC_VF_ENA_W1S); + p[i++] = nicvf_reg_read(nic, NIC_VNIC_RSS_CFG); + + for (key = 0; key < RSS_HASH_KEY_SIZE; key++) + p[i++] = nicvf_reg_read(nic, NIC_VNIC_RSS_KEY_0_4 | (key << 3)); + + /* Tx/Rx statistics */ + for (stat = 0; stat < TX_STATS_ENUM_LAST; stat++) + p[i++] = nicvf_reg_read(nic, + NIC_VNIC_TX_STAT_0_4 | (stat << 3)); + + for (i = 0; i < RX_STATS_ENUM_LAST; i++) + p[i++] = nicvf_reg_read(nic, + NIC_VNIC_RX_STAT_0_13 | (stat << 3)); + + p[i++] = nicvf_reg_read(nic, NIC_QSET_RQ_GEN_CFG); + + /* All completion queue's registers */ + for (q = 0; q < MAX_CMP_QUEUES_PER_QS; q++) { + p[i++] = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_CFG, q); + p[i++] = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_CFG2, q); + p[i++] = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_THRESH, q); + p[i++] = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_BASE, q); + p[i++] = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_HEAD, q); + p[i++] = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_TAIL, q); + p[i++] = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_DOOR, q); + p[i++] = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_STATUS, q); + p[i++] = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_STATUS2, q); + p[i++] = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_DEBUG, q); + } + + /* All receive queue's registers */ + for (q = 0; q < MAX_RCV_QUEUES_PER_QS; q++) { + p[i++] = nicvf_queue_reg_read(nic, NIC_QSET_RQ_0_7_CFG, q); + p[i++] = nicvf_queue_reg_read(nic, + NIC_QSET_RQ_0_7_STAT_0_1, q); + reg_offset = NIC_QSET_RQ_0_7_STAT_0_1 | (1 << 3); + p[i++] = nicvf_queue_reg_read(nic, reg_offset, q); + } + + for (q = 0; q < MAX_SND_QUEUES_PER_QS; q++) { + p[i++] = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_CFG, q); + p[i++] = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_THRESH, q); + p[i++] = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_BASE, q); + p[i++] = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_HEAD, q); + p[i++] = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_TAIL, q); + p[i++] = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_DOOR, q); + p[i++] = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_STATUS, q); + p[i++] = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_DEBUG, q); + p[i++] = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_CNM_CHG, q); + p[i++] = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_STAT_0_1, q); + reg_offset = NIC_QSET_SQ_0_7_STAT_0_1 | (1 << 3); + p[i++] = nicvf_queue_reg_read(nic, reg_offset, q); + } + + for (q = 0; q < MAX_RCV_BUF_DESC_RINGS_PER_QS; q++) { + p[i++] = nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_CFG, q); + p[i++] = nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_THRESH, q); + p[i++] = nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_BASE, q); + p[i++] = nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_HEAD, q); + p[i++] = nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_TAIL, q); + p[i++] = nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_DOOR, q); + p[i++] = nicvf_queue_reg_read(nic, + NIC_QSET_RBDR_0_1_STATUS0, q); + p[i++] = nicvf_queue_reg_read(nic, + NIC_QSET_RBDR_0_1_STATUS1, q); + reg_offset = NIC_QSET_RBDR_0_1_PREFETCH_STATUS; + p[i++] = nicvf_queue_reg_read(nic, reg_offset, q); + } +} + +static int nicvf_get_coalesce(struct net_device *netdev, + struct ethtool_coalesce *cmd) +{ + struct nicvf *nic = netdev_priv(netdev); + + cmd->rx_coalesce_usecs = nic->cq_coalesce_usecs; + return 0; +} + +static void nicvf_get_ringparam(struct net_device *netdev, + struct ethtool_ringparam *ring) +{ + struct nicvf *nic = netdev_priv(netdev); + struct queue_set *qs = nic->qs; + + ring->rx_max_pending = MAX_RCV_BUF_COUNT; + ring->rx_pending = qs->rbdr_len; + ring->tx_max_pending = MAX_SND_QUEUE_LEN; + ring->tx_pending = qs->sq_len; +} + +static int nicvf_get_rss_hash_opts(struct nicvf *nic, + struct ethtool_rxnfc *info) +{ + info->data = 0; + + switch (info->flow_type) { + case TCP_V4_FLOW: + case TCP_V6_FLOW: + case UDP_V4_FLOW: + case UDP_V6_FLOW: + case SCTP_V4_FLOW: + case SCTP_V6_FLOW: + info->data |= RXH_L4_B_0_1 | RXH_L4_B_2_3; + case IPV4_FLOW: + case IPV6_FLOW: + info->data |= RXH_IP_SRC | RXH_IP_DST; + break; + default: + return -EINVAL; + } + + return 0; +} + +static int nicvf_get_rxnfc(struct net_device *dev, + struct ethtool_rxnfc *info, u32 *rules) +{ + struct nicvf *nic = netdev_priv(dev); + int ret = -EOPNOTSUPP; + + switch (info->cmd) { + case ETHTOOL_GRXRINGS: + info->data = nic->rx_queues; + ret = 0; + break; + case ETHTOOL_GRXFH: + return nicvf_get_rss_hash_opts(nic, info); + default: + break; + } + return ret; +} + +static int nicvf_set_rss_hash_opts(struct nicvf *nic, + struct ethtool_rxnfc *info) +{ + struct nicvf_rss_info *rss = &nic->rss_info; + u64 rss_cfg = nicvf_reg_read(nic, NIC_VNIC_RSS_CFG); + + if (!rss->enable) + netdev_err(nic->netdev, + "RSS is disabled, hash cannot be set\n"); + + netdev_info(nic->netdev, "Set RSS flow type = %d, data = %lld\n", + info->flow_type, info->data); + + if (!(info->data & RXH_IP_SRC) || !(info->data & RXH_IP_DST)) + return -EINVAL; + + switch (info->flow_type) { + case TCP_V4_FLOW: + case TCP_V6_FLOW: + switch (info->data & (RXH_L4_B_0_1 | RXH_L4_B_2_3)) { + case 0: + rss_cfg &= ~(1ULL << RSS_HASH_TCP); + break; + case (RXH_L4_B_0_1 | RXH_L4_B_2_3): + rss_cfg |= (1ULL << RSS_HASH_TCP); + break; + default: + return -EINVAL; + } + break; + case UDP_V4_FLOW: + case UDP_V6_FLOW: + switch (info->data & (RXH_L4_B_0_1 | RXH_L4_B_2_3)) { + case 0: + rss_cfg &= ~(1ULL << RSS_HASH_UDP); + break; + case (RXH_L4_B_0_1 | RXH_L4_B_2_3): + rss_cfg |= (1ULL << RSS_HASH_UDP); + break; + default: + return -EINVAL; + } + break; + case SCTP_V4_FLOW: + case SCTP_V6_FLOW: + switch (info->data & (RXH_L4_B_0_1 | RXH_L4_B_2_3)) { + case 0: + rss_cfg &= ~(1ULL << RSS_HASH_L4ETC); + break; + case (RXH_L4_B_0_1 | RXH_L4_B_2_3): + rss_cfg |= (1ULL << RSS_HASH_L4ETC); + break; + default: + return -EINVAL; + } + break; + case IPV4_FLOW: + case IPV6_FLOW: + rss_cfg = RSS_HASH_IP; + break; + default: + return -EINVAL; + } + + nicvf_reg_write(nic, NIC_VNIC_RSS_CFG, rss_cfg); + return 0; +} + +static int nicvf_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info) +{ + struct nicvf *nic = netdev_priv(dev); + + switch (info->cmd) { + case ETHTOOL_SRXFH: + return nicvf_set_rss_hash_opts(nic, info); + default: + break; + } + return -EOPNOTSUPP; +} + +static u32 nicvf_get_rxfh_key_size(struct net_device *netdev) +{ + return RSS_HASH_KEY_SIZE * sizeof(u64); +} + +static u32 nicvf_get_rxfh_indir_size(struct net_device *dev) +{ + struct nicvf *nic = netdev_priv(dev); + + return nic->rss_info.rss_size; +} + +static int nicvf_get_rxfh(struct net_device *dev, u32 *indir, u8 *hkey, + u8 *hfunc) +{ + struct nicvf *nic = netdev_priv(dev); + struct nicvf_rss_info *rss = &nic->rss_info; + int idx; + + if (indir) { + for (idx = 0; idx < rss->rss_size; idx++) + indir[idx] = rss->ind_tbl[idx]; + } + + if (hkey) + memcpy(hkey, rss->key, RSS_HASH_KEY_SIZE * sizeof(u64)); + + if (hfunc) + *hfunc = ETH_RSS_HASH_TOP; + + return 0; +} + +static int nicvf_set_rxfh(struct net_device *dev, const u32 *indir, + const u8 *hkey, u8 hfunc) +{ + struct nicvf *nic = netdev_priv(dev); + struct nicvf_rss_info *rss = &nic->rss_info; + int idx; + + if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP) + return -EOPNOTSUPP; + + if (!rss->enable) { + netdev_err(nic->netdev, + "RSS is disabled, cannot change settings\n"); + return -EIO; + } + + if (indir) { + for (idx = 0; idx < rss->rss_size; idx++) + rss->ind_tbl[idx] = indir[idx]; + } + + if (hkey) { + memcpy(rss->key, hkey, RSS_HASH_KEY_SIZE * sizeof(u64)); + nicvf_set_rss_key(nic); + } + + nicvf_config_rss(nic); + return 0; +} + +/* Get no of queues device supports and current queue count */ +static void nicvf_get_channels(struct net_device *dev, + struct ethtool_channels *channel) +{ + struct nicvf *nic = netdev_priv(dev); + + memset(channel, 0, sizeof(*channel)); + + channel->max_rx = nic->max_queues; + channel->max_tx = nic->max_queues; + + channel->rx_count = nic->rx_queues; + channel->tx_count = nic->tx_queues; +} + +/* Set no of Tx, Rx queues to be used */ +static int nicvf_set_channels(struct net_device *dev, + struct ethtool_channels *channel) +{ + struct nicvf *nic = netdev_priv(dev); + int err = 0; + bool if_up = netif_running(dev); + int cqcount; + + if (!channel->rx_count || !channel->tx_count) + return -EINVAL; + if (channel->rx_count > nic->max_queues) + return -EINVAL; + if (channel->tx_count > nic->max_queues) + return -EINVAL; + + if (if_up) + nicvf_stop(dev); + + cqcount = max(channel->rx_count, channel->tx_count); + + if (cqcount > MAX_CMP_QUEUES_PER_QS) { + nic->sqs_count = roundup(cqcount, MAX_CMP_QUEUES_PER_QS); + nic->sqs_count = (nic->sqs_count / MAX_CMP_QUEUES_PER_QS) - 1; + } else { + nic->sqs_count = 0; + } + + nic->qs->rq_cnt = min_t(u32, channel->rx_count, MAX_RCV_QUEUES_PER_QS); + nic->qs->sq_cnt = min_t(u32, channel->tx_count, MAX_SND_QUEUES_PER_QS); + nic->qs->cq_cnt = max(nic->qs->rq_cnt, nic->qs->sq_cnt); + + nic->rx_queues = channel->rx_count; + nic->tx_queues = channel->tx_count; + err = nicvf_set_real_num_queues(dev, nic->tx_queues, nic->rx_queues); + if (err) + return err; + + if (if_up) + nicvf_open(dev); + + netdev_info(dev, "Setting num Tx rings to %d, Rx rings to %d success\n", + nic->tx_queues, nic->rx_queues); + + return err; +} + +static const struct ethtool_ops nicvf_ethtool_ops = { + .get_settings = nicvf_get_settings, + .get_link = nicvf_get_link, + .get_drvinfo = nicvf_get_drvinfo, + .get_msglevel = nicvf_get_msglevel, + .set_msglevel = nicvf_set_msglevel, + .get_strings = nicvf_get_strings, + .get_sset_count = nicvf_get_sset_count, + .get_ethtool_stats = nicvf_get_ethtool_stats, + .get_regs_len = nicvf_get_regs_len, + .get_regs = nicvf_get_regs, + .get_coalesce = nicvf_get_coalesce, + .get_ringparam = nicvf_get_ringparam, + .get_rxnfc = nicvf_get_rxnfc, + .set_rxnfc = nicvf_set_rxnfc, + .get_rxfh_key_size = nicvf_get_rxfh_key_size, + .get_rxfh_indir_size = nicvf_get_rxfh_indir_size, + .get_rxfh = nicvf_get_rxfh, + .set_rxfh = nicvf_set_rxfh, + .get_channels = nicvf_get_channels, + .set_channels = nicvf_set_channels, + .get_ts_info = ethtool_op_get_ts_info, +}; + +void nicvf_set_ethtool_ops(struct net_device *netdev) +{ + netdev->ethtool_ops = &nicvf_ethtool_ops; +} diff --git a/kernel/drivers/net/ethernet/cavium/thunder/nicvf_main.c b/kernel/drivers/net/ethernet/cavium/thunder/nicvf_main.c new file mode 100644 index 000000000..dde8dc720 --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/thunder/nicvf_main.c @@ -0,0 +1,1631 @@ +/* + * Copyright (C) 2015 Cavium, Inc. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of version 2 of the GNU General Public License + * as published by the Free Software Foundation. + */ + +#include <linux/module.h> +#include <linux/interrupt.h> +#include <linux/pci.h> +#include <linux/netdevice.h> +#include <linux/if_vlan.h> +#include <linux/etherdevice.h> +#include <linux/ethtool.h> +#include <linux/log2.h> +#include <linux/prefetch.h> +#include <linux/irq.h> + +#include "nic_reg.h" +#include "nic.h" +#include "nicvf_queues.h" +#include "thunder_bgx.h" + +#define DRV_NAME "thunder-nicvf" +#define DRV_VERSION "1.0" + +/* Supported devices */ +static const struct pci_device_id nicvf_id_table[] = { + { PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM, + PCI_DEVICE_ID_THUNDER_NIC_VF, + PCI_VENDOR_ID_CAVIUM, 0xA134) }, + { PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM, + PCI_DEVICE_ID_THUNDER_PASS1_NIC_VF, + PCI_VENDOR_ID_CAVIUM, 0xA11E) }, + { 0, } /* end of table */ +}; + +MODULE_AUTHOR("Sunil Goutham"); +MODULE_DESCRIPTION("Cavium Thunder NIC Virtual Function Driver"); +MODULE_LICENSE("GPL v2"); +MODULE_VERSION(DRV_VERSION); +MODULE_DEVICE_TABLE(pci, nicvf_id_table); + +static int debug = 0x00; +module_param(debug, int, 0644); +MODULE_PARM_DESC(debug, "Debug message level bitmap"); + +static int cpi_alg = CPI_ALG_NONE; +module_param(cpi_alg, int, S_IRUGO); +MODULE_PARM_DESC(cpi_alg, + "PFC algorithm (0=none, 1=VLAN, 2=VLAN16, 3=IP Diffserv)"); + +static inline u8 nicvf_netdev_qidx(struct nicvf *nic, u8 qidx) +{ + if (nic->sqs_mode) + return qidx + ((nic->sqs_id + 1) * MAX_CMP_QUEUES_PER_QS); + else + return qidx; +} + +static inline void nicvf_set_rx_frame_cnt(struct nicvf *nic, + struct sk_buff *skb) +{ + if (skb->len <= 64) + nic->drv_stats.rx_frames_64++; + else if (skb->len <= 127) + nic->drv_stats.rx_frames_127++; + else if (skb->len <= 255) + nic->drv_stats.rx_frames_255++; + else if (skb->len <= 511) + nic->drv_stats.rx_frames_511++; + else if (skb->len <= 1023) + nic->drv_stats.rx_frames_1023++; + else if (skb->len <= 1518) + nic->drv_stats.rx_frames_1518++; + else + nic->drv_stats.rx_frames_jumbo++; +} + +/* The Cavium ThunderX network controller can *only* be found in SoCs + * containing the ThunderX ARM64 CPU implementation. All accesses to the device + * registers on this platform are implicitly strongly ordered with respect + * to memory accesses. So writeq_relaxed() and readq_relaxed() are safe to use + * with no memory barriers in this driver. The readq()/writeq() functions add + * explicit ordering operation which in this case are redundant, and only + * add overhead. + */ + +/* Register read/write APIs */ +void nicvf_reg_write(struct nicvf *nic, u64 offset, u64 val) +{ + writeq_relaxed(val, nic->reg_base + offset); +} + +u64 nicvf_reg_read(struct nicvf *nic, u64 offset) +{ + return readq_relaxed(nic->reg_base + offset); +} + +void nicvf_queue_reg_write(struct nicvf *nic, u64 offset, + u64 qidx, u64 val) +{ + void __iomem *addr = nic->reg_base + offset; + + writeq_relaxed(val, addr + (qidx << NIC_Q_NUM_SHIFT)); +} + +u64 nicvf_queue_reg_read(struct nicvf *nic, u64 offset, u64 qidx) +{ + void __iomem *addr = nic->reg_base + offset; + + return readq_relaxed(addr + (qidx << NIC_Q_NUM_SHIFT)); +} + +/* VF -> PF mailbox communication */ +static void nicvf_write_to_mbx(struct nicvf *nic, union nic_mbx *mbx) +{ + u64 *msg = (u64 *)mbx; + + nicvf_reg_write(nic, NIC_VF_PF_MAILBOX_0_1 + 0, msg[0]); + nicvf_reg_write(nic, NIC_VF_PF_MAILBOX_0_1 + 8, msg[1]); +} + +int nicvf_send_msg_to_pf(struct nicvf *nic, union nic_mbx *mbx) +{ + int timeout = NIC_MBOX_MSG_TIMEOUT; + int sleep = 10; + + nic->pf_acked = false; + nic->pf_nacked = false; + + nicvf_write_to_mbx(nic, mbx); + + /* Wait for previous message to be acked, timeout 2sec */ + while (!nic->pf_acked) { + if (nic->pf_nacked) + return -EINVAL; + msleep(sleep); + if (nic->pf_acked) + break; + timeout -= sleep; + if (!timeout) { + netdev_err(nic->netdev, + "PF didn't ack to mbox msg %d from VF%d\n", + (mbx->msg.msg & 0xFF), nic->vf_id); + return -EBUSY; + } + } + return 0; +} + +/* Checks if VF is able to comminicate with PF +* and also gets the VNIC number this VF is associated to. +*/ +static int nicvf_check_pf_ready(struct nicvf *nic) +{ + union nic_mbx mbx = {}; + + mbx.msg.msg = NIC_MBOX_MSG_READY; + if (nicvf_send_msg_to_pf(nic, &mbx)) { + netdev_err(nic->netdev, + "PF didn't respond to READY msg\n"); + return 0; + } + + return 1; +} + +static void nicvf_read_bgx_stats(struct nicvf *nic, struct bgx_stats_msg *bgx) +{ + if (bgx->rx) + nic->bgx_stats.rx_stats[bgx->idx] = bgx->stats; + else + nic->bgx_stats.tx_stats[bgx->idx] = bgx->stats; +} + +static void nicvf_handle_mbx_intr(struct nicvf *nic) +{ + union nic_mbx mbx = {}; + u64 *mbx_data; + u64 mbx_addr; + int i; + + mbx_addr = NIC_VF_PF_MAILBOX_0_1; + mbx_data = (u64 *)&mbx; + + for (i = 0; i < NIC_PF_VF_MAILBOX_SIZE; i++) { + *mbx_data = nicvf_reg_read(nic, mbx_addr); + mbx_data++; + mbx_addr += sizeof(u64); + } + + netdev_dbg(nic->netdev, "Mbox message: msg: 0x%x\n", mbx.msg.msg); + switch (mbx.msg.msg) { + case NIC_MBOX_MSG_READY: + nic->pf_acked = true; + nic->vf_id = mbx.nic_cfg.vf_id & 0x7F; + nic->tns_mode = mbx.nic_cfg.tns_mode & 0x7F; + nic->node = mbx.nic_cfg.node_id; + if (!nic->set_mac_pending) + ether_addr_copy(nic->netdev->dev_addr, + mbx.nic_cfg.mac_addr); + nic->sqs_mode = mbx.nic_cfg.sqs_mode; + nic->loopback_supported = mbx.nic_cfg.loopback_supported; + nic->link_up = false; + nic->duplex = 0; + nic->speed = 0; + break; + case NIC_MBOX_MSG_ACK: + nic->pf_acked = true; + break; + case NIC_MBOX_MSG_NACK: + nic->pf_nacked = true; + break; + case NIC_MBOX_MSG_RSS_SIZE: + nic->rss_info.rss_size = mbx.rss_size.ind_tbl_size; + nic->pf_acked = true; + break; + case NIC_MBOX_MSG_BGX_STATS: + nicvf_read_bgx_stats(nic, &mbx.bgx_stats); + nic->pf_acked = true; + break; + case NIC_MBOX_MSG_BGX_LINK_CHANGE: + nic->pf_acked = true; + nic->link_up = mbx.link_status.link_up; + nic->duplex = mbx.link_status.duplex; + nic->speed = mbx.link_status.speed; + if (nic->link_up) { + netdev_info(nic->netdev, "%s: Link is Up %d Mbps %s\n", + nic->netdev->name, nic->speed, + nic->duplex == DUPLEX_FULL ? + "Full duplex" : "Half duplex"); + netif_carrier_on(nic->netdev); + netif_tx_start_all_queues(nic->netdev); + } else { + netdev_info(nic->netdev, "%s: Link is Down\n", + nic->netdev->name); + netif_carrier_off(nic->netdev); + netif_tx_stop_all_queues(nic->netdev); + } + break; + case NIC_MBOX_MSG_ALLOC_SQS: + nic->sqs_count = mbx.sqs_alloc.qs_count; + nic->pf_acked = true; + break; + case NIC_MBOX_MSG_SNICVF_PTR: + /* Primary VF: make note of secondary VF's pointer + * to be used while packet transmission. + */ + nic->snicvf[mbx.nicvf.sqs_id] = + (struct nicvf *)mbx.nicvf.nicvf; + nic->pf_acked = true; + break; + case NIC_MBOX_MSG_PNICVF_PTR: + /* Secondary VF/Qset: make note of primary VF's pointer + * to be used while packet reception, to handover packet + * to primary VF's netdev. + */ + nic->pnicvf = (struct nicvf *)mbx.nicvf.nicvf; + nic->pf_acked = true; + break; + default: + netdev_err(nic->netdev, + "Invalid message from PF, msg 0x%x\n", mbx.msg.msg); + break; + } + nicvf_clear_intr(nic, NICVF_INTR_MBOX, 0); +} + +static int nicvf_hw_set_mac_addr(struct nicvf *nic, struct net_device *netdev) +{ + union nic_mbx mbx = {}; + + mbx.mac.msg = NIC_MBOX_MSG_SET_MAC; + mbx.mac.vf_id = nic->vf_id; + ether_addr_copy(mbx.mac.mac_addr, netdev->dev_addr); + + return nicvf_send_msg_to_pf(nic, &mbx); +} + +static void nicvf_config_cpi(struct nicvf *nic) +{ + union nic_mbx mbx = {}; + + mbx.cpi_cfg.msg = NIC_MBOX_MSG_CPI_CFG; + mbx.cpi_cfg.vf_id = nic->vf_id; + mbx.cpi_cfg.cpi_alg = nic->cpi_alg; + mbx.cpi_cfg.rq_cnt = nic->qs->rq_cnt; + + nicvf_send_msg_to_pf(nic, &mbx); +} + +static void nicvf_get_rss_size(struct nicvf *nic) +{ + union nic_mbx mbx = {}; + + mbx.rss_size.msg = NIC_MBOX_MSG_RSS_SIZE; + mbx.rss_size.vf_id = nic->vf_id; + nicvf_send_msg_to_pf(nic, &mbx); +} + +void nicvf_config_rss(struct nicvf *nic) +{ + union nic_mbx mbx = {}; + struct nicvf_rss_info *rss = &nic->rss_info; + int ind_tbl_len = rss->rss_size; + int i, nextq = 0; + + mbx.rss_cfg.vf_id = nic->vf_id; + mbx.rss_cfg.hash_bits = rss->hash_bits; + while (ind_tbl_len) { + mbx.rss_cfg.tbl_offset = nextq; + mbx.rss_cfg.tbl_len = min(ind_tbl_len, + RSS_IND_TBL_LEN_PER_MBX_MSG); + mbx.rss_cfg.msg = mbx.rss_cfg.tbl_offset ? + NIC_MBOX_MSG_RSS_CFG_CONT : NIC_MBOX_MSG_RSS_CFG; + + for (i = 0; i < mbx.rss_cfg.tbl_len; i++) + mbx.rss_cfg.ind_tbl[i] = rss->ind_tbl[nextq++]; + + nicvf_send_msg_to_pf(nic, &mbx); + + ind_tbl_len -= mbx.rss_cfg.tbl_len; + } +} + +void nicvf_set_rss_key(struct nicvf *nic) +{ + struct nicvf_rss_info *rss = &nic->rss_info; + u64 key_addr = NIC_VNIC_RSS_KEY_0_4; + int idx; + + for (idx = 0; idx < RSS_HASH_KEY_SIZE; idx++) { + nicvf_reg_write(nic, key_addr, rss->key[idx]); + key_addr += sizeof(u64); + } +} + +static int nicvf_rss_init(struct nicvf *nic) +{ + struct nicvf_rss_info *rss = &nic->rss_info; + int idx; + + nicvf_get_rss_size(nic); + + if (cpi_alg != CPI_ALG_NONE) { + rss->enable = false; + rss->hash_bits = 0; + return 0; + } + + rss->enable = true; + + /* Using the HW reset value for now */ + rss->key[0] = 0xFEED0BADFEED0BADULL; + rss->key[1] = 0xFEED0BADFEED0BADULL; + rss->key[2] = 0xFEED0BADFEED0BADULL; + rss->key[3] = 0xFEED0BADFEED0BADULL; + rss->key[4] = 0xFEED0BADFEED0BADULL; + + nicvf_set_rss_key(nic); + + rss->cfg = RSS_IP_HASH_ENA | RSS_TCP_HASH_ENA | RSS_UDP_HASH_ENA; + nicvf_reg_write(nic, NIC_VNIC_RSS_CFG, rss->cfg); + + rss->hash_bits = ilog2(rounddown_pow_of_two(rss->rss_size)); + + for (idx = 0; idx < rss->rss_size; idx++) + rss->ind_tbl[idx] = ethtool_rxfh_indir_default(idx, + nic->rx_queues); + nicvf_config_rss(nic); + return 1; +} + +/* Request PF to allocate additional Qsets */ +static void nicvf_request_sqs(struct nicvf *nic) +{ + union nic_mbx mbx = {}; + int sqs; + int sqs_count = nic->sqs_count; + int rx_queues = 0, tx_queues = 0; + + /* Only primary VF should request */ + if (nic->sqs_mode || !nic->sqs_count) + return; + + mbx.sqs_alloc.msg = NIC_MBOX_MSG_ALLOC_SQS; + mbx.sqs_alloc.vf_id = nic->vf_id; + mbx.sqs_alloc.qs_count = nic->sqs_count; + if (nicvf_send_msg_to_pf(nic, &mbx)) { + /* No response from PF */ + nic->sqs_count = 0; + return; + } + + /* Return if no Secondary Qsets available */ + if (!nic->sqs_count) + return; + + if (nic->rx_queues > MAX_RCV_QUEUES_PER_QS) + rx_queues = nic->rx_queues - MAX_RCV_QUEUES_PER_QS; + if (nic->tx_queues > MAX_SND_QUEUES_PER_QS) + tx_queues = nic->tx_queues - MAX_SND_QUEUES_PER_QS; + + /* Set no of Rx/Tx queues in each of the SQsets */ + for (sqs = 0; sqs < nic->sqs_count; sqs++) { + mbx.nicvf.msg = NIC_MBOX_MSG_SNICVF_PTR; + mbx.nicvf.vf_id = nic->vf_id; + mbx.nicvf.sqs_id = sqs; + nicvf_send_msg_to_pf(nic, &mbx); + + nic->snicvf[sqs]->sqs_id = sqs; + if (rx_queues > MAX_RCV_QUEUES_PER_QS) { + nic->snicvf[sqs]->qs->rq_cnt = MAX_RCV_QUEUES_PER_QS; + rx_queues -= MAX_RCV_QUEUES_PER_QS; + } else { + nic->snicvf[sqs]->qs->rq_cnt = rx_queues; + rx_queues = 0; + } + + if (tx_queues > MAX_SND_QUEUES_PER_QS) { + nic->snicvf[sqs]->qs->sq_cnt = MAX_SND_QUEUES_PER_QS; + tx_queues -= MAX_SND_QUEUES_PER_QS; + } else { + nic->snicvf[sqs]->qs->sq_cnt = tx_queues; + tx_queues = 0; + } + + nic->snicvf[sqs]->qs->cq_cnt = + max(nic->snicvf[sqs]->qs->rq_cnt, nic->snicvf[sqs]->qs->sq_cnt); + + /* Initialize secondary Qset's queues and its interrupts */ + nicvf_open(nic->snicvf[sqs]->netdev); + } + + /* Update stack with actual Rx/Tx queue count allocated */ + if (sqs_count != nic->sqs_count) + nicvf_set_real_num_queues(nic->netdev, + nic->tx_queues, nic->rx_queues); +} + +/* Send this Qset's nicvf pointer to PF. + * PF inturn sends primary VF's nicvf struct to secondary Qsets/VFs + * so that packets received by these Qsets can use primary VF's netdev + */ +static void nicvf_send_vf_struct(struct nicvf *nic) +{ + union nic_mbx mbx = {}; + + mbx.nicvf.msg = NIC_MBOX_MSG_NICVF_PTR; + mbx.nicvf.sqs_mode = nic->sqs_mode; + mbx.nicvf.nicvf = (u64)nic; + nicvf_send_msg_to_pf(nic, &mbx); +} + +static void nicvf_get_primary_vf_struct(struct nicvf *nic) +{ + union nic_mbx mbx = {}; + + mbx.nicvf.msg = NIC_MBOX_MSG_PNICVF_PTR; + nicvf_send_msg_to_pf(nic, &mbx); +} + +int nicvf_set_real_num_queues(struct net_device *netdev, + int tx_queues, int rx_queues) +{ + int err = 0; + + err = netif_set_real_num_tx_queues(netdev, tx_queues); + if (err) { + netdev_err(netdev, + "Failed to set no of Tx queues: %d\n", tx_queues); + return err; + } + + err = netif_set_real_num_rx_queues(netdev, rx_queues); + if (err) + netdev_err(netdev, + "Failed to set no of Rx queues: %d\n", rx_queues); + return err; +} + +static int nicvf_init_resources(struct nicvf *nic) +{ + int err; + union nic_mbx mbx = {}; + + mbx.msg.msg = NIC_MBOX_MSG_CFG_DONE; + + /* Enable Qset */ + nicvf_qset_config(nic, true); + + /* Initialize queues and HW for data transfer */ + err = nicvf_config_data_transfer(nic, true); + if (err) { + netdev_err(nic->netdev, + "Failed to alloc/config VF's QSet resources\n"); + return err; + } + + /* Send VF config done msg to PF */ + nicvf_write_to_mbx(nic, &mbx); + + return 0; +} + +static void nicvf_snd_pkt_handler(struct net_device *netdev, + struct cmp_queue *cq, + struct cqe_send_t *cqe_tx, int cqe_type) +{ + struct sk_buff *skb = NULL; + struct nicvf *nic = netdev_priv(netdev); + struct snd_queue *sq; + struct sq_hdr_subdesc *hdr; + + sq = &nic->qs->sq[cqe_tx->sq_idx]; + + hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, cqe_tx->sqe_ptr); + if (hdr->subdesc_type != SQ_DESC_TYPE_HEADER) + return; + + netdev_dbg(nic->netdev, + "%s Qset #%d SQ #%d SQ ptr #%d subdesc count %d\n", + __func__, cqe_tx->sq_qs, cqe_tx->sq_idx, + cqe_tx->sqe_ptr, hdr->subdesc_cnt); + + nicvf_put_sq_desc(sq, hdr->subdesc_cnt + 1); + nicvf_check_cqe_tx_errs(nic, cq, cqe_tx); + skb = (struct sk_buff *)sq->skbuff[cqe_tx->sqe_ptr]; + /* For TSO offloaded packets only one head SKB needs to be freed */ + if (skb) { + prefetch(skb); + dev_consume_skb_any(skb); + sq->skbuff[cqe_tx->sqe_ptr] = (u64)NULL; + } +} + +static inline void nicvf_set_rxhash(struct net_device *netdev, + struct cqe_rx_t *cqe_rx, + struct sk_buff *skb) +{ + u8 hash_type; + u32 hash; + + if (!(netdev->features & NETIF_F_RXHASH)) + return; + + switch (cqe_rx->rss_alg) { + case RSS_ALG_TCP_IP: + case RSS_ALG_UDP_IP: + hash_type = PKT_HASH_TYPE_L4; + hash = cqe_rx->rss_tag; + break; + case RSS_ALG_IP: + hash_type = PKT_HASH_TYPE_L3; + hash = cqe_rx->rss_tag; + break; + default: + hash_type = PKT_HASH_TYPE_NONE; + hash = 0; + } + + skb_set_hash(skb, hash, hash_type); +} + +static void nicvf_rcv_pkt_handler(struct net_device *netdev, + struct napi_struct *napi, + struct cmp_queue *cq, + struct cqe_rx_t *cqe_rx, int cqe_type) +{ + struct sk_buff *skb; + struct nicvf *nic = netdev_priv(netdev); + int err = 0; + int rq_idx; + + rq_idx = nicvf_netdev_qidx(nic, cqe_rx->rq_idx); + + if (nic->sqs_mode) { + /* Use primary VF's 'nicvf' struct */ + nic = nic->pnicvf; + netdev = nic->netdev; + } + + /* Check for errors */ + err = nicvf_check_cqe_rx_errs(nic, cq, cqe_rx); + if (err && !cqe_rx->rb_cnt) + return; + + skb = nicvf_get_rcv_skb(nic, cqe_rx); + if (!skb) { + netdev_dbg(nic->netdev, "Packet not received\n"); + return; + } + + if (netif_msg_pktdata(nic)) { + netdev_info(nic->netdev, "%s: skb 0x%p, len=%d\n", netdev->name, + skb, skb->len); + print_hex_dump(KERN_INFO, "", DUMP_PREFIX_OFFSET, 16, 1, + skb->data, skb->len, true); + } + + /* If error packet, drop it here */ + if (err) { + dev_kfree_skb_any(skb); + return; + } + + nicvf_set_rx_frame_cnt(nic, skb); + + nicvf_set_rxhash(netdev, cqe_rx, skb); + + skb_record_rx_queue(skb, rq_idx); + if (netdev->hw_features & NETIF_F_RXCSUM) { + /* HW by default verifies TCP/UDP/SCTP checksums */ + skb->ip_summed = CHECKSUM_UNNECESSARY; + } else { + skb_checksum_none_assert(skb); + } + + skb->protocol = eth_type_trans(skb, netdev); + + /* Check for stripped VLAN */ + if (cqe_rx->vlan_found && cqe_rx->vlan_stripped) + __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), + ntohs((__force __be16)cqe_rx->vlan_tci)); + + if (napi && (netdev->features & NETIF_F_GRO)) + napi_gro_receive(napi, skb); + else + netif_receive_skb(skb); +} + +static int nicvf_cq_intr_handler(struct net_device *netdev, u8 cq_idx, + struct napi_struct *napi, int budget) +{ + int processed_cqe, work_done = 0, tx_done = 0; + int cqe_count, cqe_head; + struct nicvf *nic = netdev_priv(netdev); + struct queue_set *qs = nic->qs; + struct cmp_queue *cq = &qs->cq[cq_idx]; + struct cqe_rx_t *cq_desc; + struct netdev_queue *txq; + + spin_lock_bh(&cq->lock); +loop: + processed_cqe = 0; + /* Get no of valid CQ entries to process */ + cqe_count = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_STATUS, cq_idx); + cqe_count &= CQ_CQE_COUNT; + if (!cqe_count) + goto done; + + /* Get head of the valid CQ entries */ + cqe_head = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_HEAD, cq_idx) >> 9; + cqe_head &= 0xFFFF; + + netdev_dbg(nic->netdev, "%s CQ%d cqe_count %d cqe_head %d\n", + __func__, cq_idx, cqe_count, cqe_head); + while (processed_cqe < cqe_count) { + /* Get the CQ descriptor */ + cq_desc = (struct cqe_rx_t *)GET_CQ_DESC(cq, cqe_head); + cqe_head++; + cqe_head &= (cq->dmem.q_len - 1); + /* Initiate prefetch for next descriptor */ + prefetch((struct cqe_rx_t *)GET_CQ_DESC(cq, cqe_head)); + + if ((work_done >= budget) && napi && + (cq_desc->cqe_type != CQE_TYPE_SEND)) { + break; + } + + netdev_dbg(nic->netdev, "CQ%d cq_desc->cqe_type %d\n", + cq_idx, cq_desc->cqe_type); + switch (cq_desc->cqe_type) { + case CQE_TYPE_RX: + nicvf_rcv_pkt_handler(netdev, napi, cq, + cq_desc, CQE_TYPE_RX); + work_done++; + break; + case CQE_TYPE_SEND: + nicvf_snd_pkt_handler(netdev, cq, + (void *)cq_desc, CQE_TYPE_SEND); + tx_done++; + break; + case CQE_TYPE_INVALID: + case CQE_TYPE_RX_SPLIT: + case CQE_TYPE_RX_TCP: + case CQE_TYPE_SEND_PTP: + /* Ignore for now */ + break; + } + processed_cqe++; + } + netdev_dbg(nic->netdev, + "%s CQ%d processed_cqe %d work_done %d budget %d\n", + __func__, cq_idx, processed_cqe, work_done, budget); + + /* Ring doorbell to inform H/W to reuse processed CQEs */ + nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_DOOR, + cq_idx, processed_cqe); + + if ((work_done < budget) && napi) + goto loop; + +done: + /* Wakeup TXQ if its stopped earlier due to SQ full */ + if (tx_done) { + netdev = nic->pnicvf->netdev; + txq = netdev_get_tx_queue(netdev, + nicvf_netdev_qidx(nic, cq_idx)); + nic = nic->pnicvf; + if (netif_tx_queue_stopped(txq) && netif_carrier_ok(netdev)) { + netif_tx_start_queue(txq); + nic->drv_stats.txq_wake++; + if (netif_msg_tx_err(nic)) + netdev_warn(netdev, + "%s: Transmit queue wakeup SQ%d\n", + netdev->name, cq_idx); + } + } + + spin_unlock_bh(&cq->lock); + return work_done; +} + +static int nicvf_poll(struct napi_struct *napi, int budget) +{ + u64 cq_head; + int work_done = 0; + struct net_device *netdev = napi->dev; + struct nicvf *nic = netdev_priv(netdev); + struct nicvf_cq_poll *cq; + + cq = container_of(napi, struct nicvf_cq_poll, napi); + work_done = nicvf_cq_intr_handler(netdev, cq->cq_idx, napi, budget); + + if (work_done < budget) { + /* Slow packet rate, exit polling */ + napi_complete(napi); + /* Re-enable interrupts */ + cq_head = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_HEAD, + cq->cq_idx); + nicvf_clear_intr(nic, NICVF_INTR_CQ, cq->cq_idx); + nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_HEAD, + cq->cq_idx, cq_head); + nicvf_enable_intr(nic, NICVF_INTR_CQ, cq->cq_idx); + } + return work_done; +} + +/* Qset error interrupt handler + * + * As of now only CQ errors are handled + */ +static void nicvf_handle_qs_err(unsigned long data) +{ + struct nicvf *nic = (struct nicvf *)data; + struct queue_set *qs = nic->qs; + int qidx; + u64 status; + + netif_tx_disable(nic->netdev); + + /* Check if it is CQ err */ + for (qidx = 0; qidx < qs->cq_cnt; qidx++) { + status = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_STATUS, + qidx); + if (!(status & CQ_ERR_MASK)) + continue; + /* Process already queued CQEs and reconfig CQ */ + nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx); + nicvf_sq_disable(nic, qidx); + nicvf_cq_intr_handler(nic->netdev, qidx, NULL, 0); + nicvf_cmp_queue_config(nic, qs, qidx, true); + nicvf_sq_free_used_descs(nic->netdev, &qs->sq[qidx], qidx); + nicvf_sq_enable(nic, &qs->sq[qidx], qidx); + + nicvf_enable_intr(nic, NICVF_INTR_CQ, qidx); + } + + netif_tx_start_all_queues(nic->netdev); + /* Re-enable Qset error interrupt */ + nicvf_enable_intr(nic, NICVF_INTR_QS_ERR, 0); +} + +static void nicvf_dump_intr_status(struct nicvf *nic) +{ + if (netif_msg_intr(nic)) + netdev_info(nic->netdev, "%s: interrupt status 0x%llx\n", + nic->netdev->name, nicvf_reg_read(nic, NIC_VF_INT)); +} + +static irqreturn_t nicvf_misc_intr_handler(int irq, void *nicvf_irq) +{ + struct nicvf *nic = (struct nicvf *)nicvf_irq; + u64 intr; + + nicvf_dump_intr_status(nic); + + intr = nicvf_reg_read(nic, NIC_VF_INT); + /* Check for spurious interrupt */ + if (!(intr & NICVF_INTR_MBOX_MASK)) + return IRQ_HANDLED; + + nicvf_handle_mbx_intr(nic); + + return IRQ_HANDLED; +} + +static irqreturn_t nicvf_intr_handler(int irq, void *cq_irq) +{ + struct nicvf_cq_poll *cq_poll = (struct nicvf_cq_poll *)cq_irq; + struct nicvf *nic = cq_poll->nicvf; + int qidx = cq_poll->cq_idx; + + nicvf_dump_intr_status(nic); + + /* Disable interrupts */ + nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx); + + /* Schedule NAPI */ + napi_schedule(&cq_poll->napi); + + /* Clear interrupt */ + nicvf_clear_intr(nic, NICVF_INTR_CQ, qidx); + + return IRQ_HANDLED; +} + +static irqreturn_t nicvf_rbdr_intr_handler(int irq, void *nicvf_irq) +{ + struct nicvf *nic = (struct nicvf *)nicvf_irq; + u8 qidx; + + + nicvf_dump_intr_status(nic); + + /* Disable RBDR interrupt and schedule softirq */ + for (qidx = 0; qidx < nic->qs->rbdr_cnt; qidx++) { + if (!nicvf_is_intr_enabled(nic, NICVF_INTR_RBDR, qidx)) + continue; + nicvf_disable_intr(nic, NICVF_INTR_RBDR, qidx); + tasklet_hi_schedule(&nic->rbdr_task); + /* Clear interrupt */ + nicvf_clear_intr(nic, NICVF_INTR_RBDR, qidx); + } + + return IRQ_HANDLED; +} + +static irqreturn_t nicvf_qs_err_intr_handler(int irq, void *nicvf_irq) +{ + struct nicvf *nic = (struct nicvf *)nicvf_irq; + + nicvf_dump_intr_status(nic); + + /* Disable Qset err interrupt and schedule softirq */ + nicvf_disable_intr(nic, NICVF_INTR_QS_ERR, 0); + tasklet_hi_schedule(&nic->qs_err_task); + nicvf_clear_intr(nic, NICVF_INTR_QS_ERR, 0); + + return IRQ_HANDLED; +} + +static int nicvf_enable_msix(struct nicvf *nic) +{ + int ret, vec; + + nic->num_vec = NIC_VF_MSIX_VECTORS; + + for (vec = 0; vec < nic->num_vec; vec++) + nic->msix_entries[vec].entry = vec; + + ret = pci_enable_msix(nic->pdev, nic->msix_entries, nic->num_vec); + if (ret) { + netdev_err(nic->netdev, + "Req for #%d msix vectors failed\n", nic->num_vec); + return 0; + } + nic->msix_enabled = 1; + return 1; +} + +static void nicvf_disable_msix(struct nicvf *nic) +{ + if (nic->msix_enabled) { + pci_disable_msix(nic->pdev); + nic->msix_enabled = 0; + nic->num_vec = 0; + } +} + +static int nicvf_register_interrupts(struct nicvf *nic) +{ + int irq, ret = 0; + int vector; + + for_each_cq_irq(irq) + sprintf(nic->irq_name[irq], "NICVF%d CQ%d", + nic->vf_id, irq); + + for_each_sq_irq(irq) + sprintf(nic->irq_name[irq], "NICVF%d SQ%d", + nic->vf_id, irq - NICVF_INTR_ID_SQ); + + for_each_rbdr_irq(irq) + sprintf(nic->irq_name[irq], "NICVF%d RBDR%d", + nic->vf_id, irq - NICVF_INTR_ID_RBDR); + + /* Register CQ interrupts */ + for (irq = 0; irq < nic->qs->cq_cnt; irq++) { + vector = nic->msix_entries[irq].vector; + ret = request_irq(vector, nicvf_intr_handler, + 0, nic->irq_name[irq], nic->napi[irq]); + if (ret) + goto err; + nic->irq_allocated[irq] = true; + } + + /* Register RBDR interrupt */ + for (irq = NICVF_INTR_ID_RBDR; + irq < (NICVF_INTR_ID_RBDR + nic->qs->rbdr_cnt); irq++) { + vector = nic->msix_entries[irq].vector; + ret = request_irq(vector, nicvf_rbdr_intr_handler, + 0, nic->irq_name[irq], nic); + if (ret) + goto err; + nic->irq_allocated[irq] = true; + } + + /* Register QS error interrupt */ + sprintf(nic->irq_name[NICVF_INTR_ID_QS_ERR], + "NICVF%d Qset error", nic->vf_id); + irq = NICVF_INTR_ID_QS_ERR; + ret = request_irq(nic->msix_entries[irq].vector, + nicvf_qs_err_intr_handler, + 0, nic->irq_name[irq], nic); + if (!ret) + nic->irq_allocated[irq] = true; + +err: + if (ret) + netdev_err(nic->netdev, "request_irq failed, vector %d\n", irq); + + return ret; +} + +static void nicvf_unregister_interrupts(struct nicvf *nic) +{ + int irq; + + /* Free registered interrupts */ + for (irq = 0; irq < nic->num_vec; irq++) { + if (!nic->irq_allocated[irq]) + continue; + + if (irq < NICVF_INTR_ID_SQ) + free_irq(nic->msix_entries[irq].vector, nic->napi[irq]); + else + free_irq(nic->msix_entries[irq].vector, nic); + + nic->irq_allocated[irq] = false; + } + + /* Disable MSI-X */ + nicvf_disable_msix(nic); +} + +/* Initialize MSIX vectors and register MISC interrupt. + * Send READY message to PF to check if its alive + */ +static int nicvf_register_misc_interrupt(struct nicvf *nic) +{ + int ret = 0; + int irq = NICVF_INTR_ID_MISC; + + /* Return if mailbox interrupt is already registered */ + if (nic->msix_enabled) + return 0; + + /* Enable MSI-X */ + if (!nicvf_enable_msix(nic)) + return 1; + + sprintf(nic->irq_name[irq], "%s Mbox", "NICVF"); + /* Register Misc interrupt */ + ret = request_irq(nic->msix_entries[irq].vector, + nicvf_misc_intr_handler, 0, nic->irq_name[irq], nic); + + if (ret) + return ret; + nic->irq_allocated[irq] = true; + + /* Enable mailbox interrupt */ + nicvf_enable_intr(nic, NICVF_INTR_MBOX, 0); + + /* Check if VF is able to communicate with PF */ + if (!nicvf_check_pf_ready(nic)) { + nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0); + nicvf_unregister_interrupts(nic); + return 1; + } + + return 0; +} + +static netdev_tx_t nicvf_xmit(struct sk_buff *skb, struct net_device *netdev) +{ + struct nicvf *nic = netdev_priv(netdev); + int qid = skb_get_queue_mapping(skb); + struct netdev_queue *txq = netdev_get_tx_queue(netdev, qid); + + /* Check for minimum packet length */ + if (skb->len <= ETH_HLEN) { + dev_kfree_skb(skb); + return NETDEV_TX_OK; + } + + if (!netif_tx_queue_stopped(txq) && !nicvf_sq_append_skb(nic, skb)) { + netif_tx_stop_queue(txq); + nic->drv_stats.txq_stop++; + if (netif_msg_tx_err(nic)) + netdev_warn(netdev, + "%s: Transmit ring full, stopping SQ%d\n", + netdev->name, qid); + return NETDEV_TX_BUSY; + } + + return NETDEV_TX_OK; +} + +static inline void nicvf_free_cq_poll(struct nicvf *nic) +{ + struct nicvf_cq_poll *cq_poll; + int qidx; + + for (qidx = 0; qidx < nic->qs->cq_cnt; qidx++) { + cq_poll = nic->napi[qidx]; + if (!cq_poll) + continue; + nic->napi[qidx] = NULL; + kfree(cq_poll); + } +} + +int nicvf_stop(struct net_device *netdev) +{ + int irq, qidx; + struct nicvf *nic = netdev_priv(netdev); + struct queue_set *qs = nic->qs; + struct nicvf_cq_poll *cq_poll = NULL; + union nic_mbx mbx = {}; + + mbx.msg.msg = NIC_MBOX_MSG_SHUTDOWN; + nicvf_send_msg_to_pf(nic, &mbx); + + netif_carrier_off(netdev); + netif_tx_stop_all_queues(nic->netdev); + nic->link_up = false; + + /* Teardown secondary qsets first */ + if (!nic->sqs_mode) { + for (qidx = 0; qidx < nic->sqs_count; qidx++) { + if (!nic->snicvf[qidx]) + continue; + nicvf_stop(nic->snicvf[qidx]->netdev); + nic->snicvf[qidx] = NULL; + } + } + + /* Disable RBDR & QS error interrupts */ + for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) { + nicvf_disable_intr(nic, NICVF_INTR_RBDR, qidx); + nicvf_clear_intr(nic, NICVF_INTR_RBDR, qidx); + } + nicvf_disable_intr(nic, NICVF_INTR_QS_ERR, 0); + nicvf_clear_intr(nic, NICVF_INTR_QS_ERR, 0); + + /* Wait for pending IRQ handlers to finish */ + for (irq = 0; irq < nic->num_vec; irq++) + synchronize_irq(nic->msix_entries[irq].vector); + + tasklet_kill(&nic->rbdr_task); + tasklet_kill(&nic->qs_err_task); + if (nic->rb_work_scheduled) + cancel_delayed_work_sync(&nic->rbdr_work); + + for (qidx = 0; qidx < nic->qs->cq_cnt; qidx++) { + cq_poll = nic->napi[qidx]; + if (!cq_poll) + continue; + napi_synchronize(&cq_poll->napi); + /* CQ intr is enabled while napi_complete, + * so disable it now + */ + nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx); + nicvf_clear_intr(nic, NICVF_INTR_CQ, qidx); + napi_disable(&cq_poll->napi); + netif_napi_del(&cq_poll->napi); + } + + netif_tx_disable(netdev); + + /* Free resources */ + nicvf_config_data_transfer(nic, false); + + /* Disable HW Qset */ + nicvf_qset_config(nic, false); + + /* disable mailbox interrupt */ + nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0); + + nicvf_unregister_interrupts(nic); + + nicvf_free_cq_poll(nic); + + /* Clear multiqset info */ + nic->pnicvf = nic; + nic->sqs_count = 0; + + return 0; +} + +int nicvf_open(struct net_device *netdev) +{ + int err, qidx; + struct nicvf *nic = netdev_priv(netdev); + struct queue_set *qs = nic->qs; + struct nicvf_cq_poll *cq_poll = NULL; + + nic->mtu = netdev->mtu; + + netif_carrier_off(netdev); + + err = nicvf_register_misc_interrupt(nic); + if (err) + return err; + + /* Register NAPI handler for processing CQEs */ + for (qidx = 0; qidx < qs->cq_cnt; qidx++) { + cq_poll = kzalloc(sizeof(*cq_poll), GFP_KERNEL); + if (!cq_poll) { + err = -ENOMEM; + goto napi_del; + } + cq_poll->cq_idx = qidx; + cq_poll->nicvf = nic; + netif_napi_add(netdev, &cq_poll->napi, nicvf_poll, + NAPI_POLL_WEIGHT); + napi_enable(&cq_poll->napi); + nic->napi[qidx] = cq_poll; + } + + /* Check if we got MAC address from PF or else generate a radom MAC */ + if (is_zero_ether_addr(netdev->dev_addr)) { + eth_hw_addr_random(netdev); + nicvf_hw_set_mac_addr(nic, netdev); + } + + if (nic->set_mac_pending) { + nic->set_mac_pending = false; + nicvf_hw_set_mac_addr(nic, netdev); + } + + /* Init tasklet for handling Qset err interrupt */ + tasklet_init(&nic->qs_err_task, nicvf_handle_qs_err, + (unsigned long)nic); + + /* Init RBDR tasklet which will refill RBDR */ + tasklet_init(&nic->rbdr_task, nicvf_rbdr_task, + (unsigned long)nic); + INIT_DELAYED_WORK(&nic->rbdr_work, nicvf_rbdr_work); + + /* Configure CPI alorithm */ + nic->cpi_alg = cpi_alg; + if (!nic->sqs_mode) + nicvf_config_cpi(nic); + + nicvf_request_sqs(nic); + if (nic->sqs_mode) + nicvf_get_primary_vf_struct(nic); + + /* Configure receive side scaling */ + if (!nic->sqs_mode) + nicvf_rss_init(nic); + + err = nicvf_register_interrupts(nic); + if (err) + goto cleanup; + + /* Initialize the queues */ + err = nicvf_init_resources(nic); + if (err) + goto cleanup; + + /* Make sure queue initialization is written */ + wmb(); + + nicvf_reg_write(nic, NIC_VF_INT, -1); + /* Enable Qset err interrupt */ + nicvf_enable_intr(nic, NICVF_INTR_QS_ERR, 0); + + /* Enable completion queue interrupt */ + for (qidx = 0; qidx < qs->cq_cnt; qidx++) + nicvf_enable_intr(nic, NICVF_INTR_CQ, qidx); + + /* Enable RBDR threshold interrupt */ + for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) + nicvf_enable_intr(nic, NICVF_INTR_RBDR, qidx); + + nic->drv_stats.txq_stop = 0; + nic->drv_stats.txq_wake = 0; + + return 0; +cleanup: + nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0); + nicvf_unregister_interrupts(nic); + tasklet_kill(&nic->qs_err_task); + tasklet_kill(&nic->rbdr_task); +napi_del: + for (qidx = 0; qidx < qs->cq_cnt; qidx++) { + cq_poll = nic->napi[qidx]; + if (!cq_poll) + continue; + napi_disable(&cq_poll->napi); + netif_napi_del(&cq_poll->napi); + } + nicvf_free_cq_poll(nic); + return err; +} + +static int nicvf_update_hw_max_frs(struct nicvf *nic, int mtu) +{ + union nic_mbx mbx = {}; + + mbx.frs.msg = NIC_MBOX_MSG_SET_MAX_FRS; + mbx.frs.max_frs = mtu; + mbx.frs.vf_id = nic->vf_id; + + return nicvf_send_msg_to_pf(nic, &mbx); +} + +static int nicvf_change_mtu(struct net_device *netdev, int new_mtu) +{ + struct nicvf *nic = netdev_priv(netdev); + + if (new_mtu > NIC_HW_MAX_FRS) + return -EINVAL; + + if (new_mtu < NIC_HW_MIN_FRS) + return -EINVAL; + + if (nicvf_update_hw_max_frs(nic, new_mtu)) + return -EINVAL; + netdev->mtu = new_mtu; + nic->mtu = new_mtu; + + return 0; +} + +static int nicvf_set_mac_address(struct net_device *netdev, void *p) +{ + struct sockaddr *addr = p; + struct nicvf *nic = netdev_priv(netdev); + + if (!is_valid_ether_addr(addr->sa_data)) + return -EADDRNOTAVAIL; + + memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); + + if (nic->msix_enabled) { + if (nicvf_hw_set_mac_addr(nic, netdev)) + return -EBUSY; + } else { + nic->set_mac_pending = true; + } + + return 0; +} + +void nicvf_update_lmac_stats(struct nicvf *nic) +{ + int stat = 0; + union nic_mbx mbx = {}; + + if (!netif_running(nic->netdev)) + return; + + mbx.bgx_stats.msg = NIC_MBOX_MSG_BGX_STATS; + mbx.bgx_stats.vf_id = nic->vf_id; + /* Rx stats */ + mbx.bgx_stats.rx = 1; + while (stat < BGX_RX_STATS_COUNT) { + mbx.bgx_stats.idx = stat; + if (nicvf_send_msg_to_pf(nic, &mbx)) + return; + stat++; + } + + stat = 0; + + /* Tx stats */ + mbx.bgx_stats.rx = 0; + while (stat < BGX_TX_STATS_COUNT) { + mbx.bgx_stats.idx = stat; + if (nicvf_send_msg_to_pf(nic, &mbx)) + return; + stat++; + } +} + +void nicvf_update_stats(struct nicvf *nic) +{ + int qidx; + struct nicvf_hw_stats *stats = &nic->hw_stats; + struct nicvf_drv_stats *drv_stats = &nic->drv_stats; + struct queue_set *qs = nic->qs; + +#define GET_RX_STATS(reg) \ + nicvf_reg_read(nic, NIC_VNIC_RX_STAT_0_13 | (reg << 3)) +#define GET_TX_STATS(reg) \ + nicvf_reg_read(nic, NIC_VNIC_TX_STAT_0_4 | (reg << 3)) + + stats->rx_bytes = GET_RX_STATS(RX_OCTS); + stats->rx_ucast_frames = GET_RX_STATS(RX_UCAST); + stats->rx_bcast_frames = GET_RX_STATS(RX_BCAST); + stats->rx_mcast_frames = GET_RX_STATS(RX_MCAST); + stats->rx_fcs_errors = GET_RX_STATS(RX_FCS); + stats->rx_l2_errors = GET_RX_STATS(RX_L2ERR); + stats->rx_drop_red = GET_RX_STATS(RX_RED); + stats->rx_drop_red_bytes = GET_RX_STATS(RX_RED_OCTS); + stats->rx_drop_overrun = GET_RX_STATS(RX_ORUN); + stats->rx_drop_overrun_bytes = GET_RX_STATS(RX_ORUN_OCTS); + stats->rx_drop_bcast = GET_RX_STATS(RX_DRP_BCAST); + stats->rx_drop_mcast = GET_RX_STATS(RX_DRP_MCAST); + stats->rx_drop_l3_bcast = GET_RX_STATS(RX_DRP_L3BCAST); + stats->rx_drop_l3_mcast = GET_RX_STATS(RX_DRP_L3MCAST); + + stats->tx_bytes_ok = GET_TX_STATS(TX_OCTS); + stats->tx_ucast_frames_ok = GET_TX_STATS(TX_UCAST); + stats->tx_bcast_frames_ok = GET_TX_STATS(TX_BCAST); + stats->tx_mcast_frames_ok = GET_TX_STATS(TX_MCAST); + stats->tx_drops = GET_TX_STATS(TX_DROP); + + drv_stats->tx_frames_ok = stats->tx_ucast_frames_ok + + stats->tx_bcast_frames_ok + + stats->tx_mcast_frames_ok; + drv_stats->rx_drops = stats->rx_drop_red + + stats->rx_drop_overrun; + drv_stats->tx_drops = stats->tx_drops; + + /* Update RQ and SQ stats */ + for (qidx = 0; qidx < qs->rq_cnt; qidx++) + nicvf_update_rq_stats(nic, qidx); + for (qidx = 0; qidx < qs->sq_cnt; qidx++) + nicvf_update_sq_stats(nic, qidx); +} + +static struct rtnl_link_stats64 *nicvf_get_stats64(struct net_device *netdev, + struct rtnl_link_stats64 *stats) +{ + struct nicvf *nic = netdev_priv(netdev); + struct nicvf_hw_stats *hw_stats = &nic->hw_stats; + struct nicvf_drv_stats *drv_stats = &nic->drv_stats; + + nicvf_update_stats(nic); + + stats->rx_bytes = hw_stats->rx_bytes; + stats->rx_packets = drv_stats->rx_frames_ok; + stats->rx_dropped = drv_stats->rx_drops; + stats->multicast = hw_stats->rx_mcast_frames; + + stats->tx_bytes = hw_stats->tx_bytes_ok; + stats->tx_packets = drv_stats->tx_frames_ok; + stats->tx_dropped = drv_stats->tx_drops; + + return stats; +} + +static void nicvf_tx_timeout(struct net_device *dev) +{ + struct nicvf *nic = netdev_priv(dev); + + if (netif_msg_tx_err(nic)) + netdev_warn(dev, "%s: Transmit timed out, resetting\n", + dev->name); + + schedule_work(&nic->reset_task); +} + +static void nicvf_reset_task(struct work_struct *work) +{ + struct nicvf *nic; + + nic = container_of(work, struct nicvf, reset_task); + + if (!netif_running(nic->netdev)) + return; + + nicvf_stop(nic->netdev); + nicvf_open(nic->netdev); + nic->netdev->trans_start = jiffies; +} + +static int nicvf_config_loopback(struct nicvf *nic, + netdev_features_t features) +{ + union nic_mbx mbx = {}; + + mbx.lbk.msg = NIC_MBOX_MSG_LOOPBACK; + mbx.lbk.vf_id = nic->vf_id; + mbx.lbk.enable = (features & NETIF_F_LOOPBACK) != 0; + + return nicvf_send_msg_to_pf(nic, &mbx); +} + +static netdev_features_t nicvf_fix_features(struct net_device *netdev, + netdev_features_t features) +{ + struct nicvf *nic = netdev_priv(netdev); + + if ((features & NETIF_F_LOOPBACK) && + netif_running(netdev) && !nic->loopback_supported) + features &= ~NETIF_F_LOOPBACK; + + return features; +} + +static int nicvf_set_features(struct net_device *netdev, + netdev_features_t features) +{ + struct nicvf *nic = netdev_priv(netdev); + netdev_features_t changed = features ^ netdev->features; + + if (changed & NETIF_F_HW_VLAN_CTAG_RX) + nicvf_config_vlan_stripping(nic, features); + + if ((changed & NETIF_F_LOOPBACK) && netif_running(netdev)) + return nicvf_config_loopback(nic, features); + + return 0; +} + +static const struct net_device_ops nicvf_netdev_ops = { + .ndo_open = nicvf_open, + .ndo_stop = nicvf_stop, + .ndo_start_xmit = nicvf_xmit, + .ndo_change_mtu = nicvf_change_mtu, + .ndo_set_mac_address = nicvf_set_mac_address, + .ndo_get_stats64 = nicvf_get_stats64, + .ndo_tx_timeout = nicvf_tx_timeout, + .ndo_fix_features = nicvf_fix_features, + .ndo_set_features = nicvf_set_features, +}; + +static int nicvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct device *dev = &pdev->dev; + struct net_device *netdev; + struct nicvf *nic; + int err, qcount; + + err = pci_enable_device(pdev); + if (err) { + dev_err(dev, "Failed to enable PCI device\n"); + return err; + } + + err = pci_request_regions(pdev, DRV_NAME); + if (err) { + dev_err(dev, "PCI request regions failed 0x%x\n", err); + goto err_disable_device; + } + + err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48)); + if (err) { + dev_err(dev, "Unable to get usable DMA configuration\n"); + goto err_release_regions; + } + + err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48)); + if (err) { + dev_err(dev, "unable to get 48-bit DMA for consistent allocations\n"); + goto err_release_regions; + } + + qcount = MAX_CMP_QUEUES_PER_QS; + + /* Restrict multiqset support only for host bound VFs */ + if (pdev->is_virtfn) { + /* Set max number of queues per VF */ + qcount = roundup(num_online_cpus(), MAX_CMP_QUEUES_PER_QS); + qcount = min(qcount, + (MAX_SQS_PER_VF + 1) * MAX_CMP_QUEUES_PER_QS); + } + + netdev = alloc_etherdev_mqs(sizeof(struct nicvf), qcount, qcount); + if (!netdev) { + err = -ENOMEM; + goto err_release_regions; + } + + pci_set_drvdata(pdev, netdev); + + SET_NETDEV_DEV(netdev, &pdev->dev); + + nic = netdev_priv(netdev); + nic->netdev = netdev; + nic->pdev = pdev; + nic->pnicvf = nic; + nic->max_queues = qcount; + + /* MAP VF's configuration registers */ + nic->reg_base = pcim_iomap(pdev, PCI_CFG_REG_BAR_NUM, 0); + if (!nic->reg_base) { + dev_err(dev, "Cannot map config register space, aborting\n"); + err = -ENOMEM; + goto err_free_netdev; + } + + err = nicvf_set_qset_resources(nic); + if (err) + goto err_free_netdev; + + /* Check if PF is alive and get MAC address for this VF */ + err = nicvf_register_misc_interrupt(nic); + if (err) + goto err_free_netdev; + + nicvf_send_vf_struct(nic); + + /* Check if this VF is in QS only mode */ + if (nic->sqs_mode) + return 0; + + err = nicvf_set_real_num_queues(netdev, nic->tx_queues, nic->rx_queues); + if (err) + goto err_unregister_interrupts; + + netdev->hw_features = (NETIF_F_RXCSUM | NETIF_F_IP_CSUM | NETIF_F_SG | + NETIF_F_TSO | NETIF_F_GRO | + NETIF_F_HW_VLAN_CTAG_RX); + + netdev->hw_features |= NETIF_F_RXHASH; + + netdev->features |= netdev->hw_features; + netdev->hw_features |= NETIF_F_LOOPBACK; + + netdev->vlan_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO; + + netdev->netdev_ops = &nicvf_netdev_ops; + netdev->watchdog_timeo = NICVF_TX_TIMEOUT; + + INIT_WORK(&nic->reset_task, nicvf_reset_task); + + err = register_netdev(netdev); + if (err) { + dev_err(dev, "Failed to register netdevice\n"); + goto err_unregister_interrupts; + } + + nic->msg_enable = debug; + + nicvf_set_ethtool_ops(netdev); + + return 0; + +err_unregister_interrupts: + nicvf_unregister_interrupts(nic); +err_free_netdev: + pci_set_drvdata(pdev, NULL); + free_netdev(netdev); +err_release_regions: + pci_release_regions(pdev); +err_disable_device: + pci_disable_device(pdev); + return err; +} + +static void nicvf_remove(struct pci_dev *pdev) +{ + struct net_device *netdev = pci_get_drvdata(pdev); + struct nicvf *nic; + struct net_device *pnetdev; + + if (!netdev) + return; + + nic = netdev_priv(netdev); + pnetdev = nic->pnicvf->netdev; + + /* Check if this Qset is assigned to different VF. + * If yes, clean primary and all secondary Qsets. + */ + if (pnetdev && (pnetdev->reg_state == NETREG_REGISTERED)) + unregister_netdev(pnetdev); + nicvf_unregister_interrupts(nic); + pci_set_drvdata(pdev, NULL); + free_netdev(netdev); + pci_release_regions(pdev); + pci_disable_device(pdev); +} + +static void nicvf_shutdown(struct pci_dev *pdev) +{ + nicvf_remove(pdev); +} + +static struct pci_driver nicvf_driver = { + .name = DRV_NAME, + .id_table = nicvf_id_table, + .probe = nicvf_probe, + .remove = nicvf_remove, + .shutdown = nicvf_shutdown, +}; + +static int __init nicvf_init_module(void) +{ + pr_info("%s, ver %s\n", DRV_NAME, DRV_VERSION); + + return pci_register_driver(&nicvf_driver); +} + +static void __exit nicvf_cleanup_module(void) +{ + pci_unregister_driver(&nicvf_driver); +} + +module_init(nicvf_init_module); +module_exit(nicvf_cleanup_module); diff --git a/kernel/drivers/net/ethernet/cavium/thunder/nicvf_queues.c b/kernel/drivers/net/ethernet/cavium/thunder/nicvf_queues.c new file mode 100644 index 000000000..206b6a71a --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/thunder/nicvf_queues.c @@ -0,0 +1,1560 @@ +/* + * Copyright (C) 2015 Cavium, Inc. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of version 2 of the GNU General Public License + * as published by the Free Software Foundation. + */ + +#include <linux/pci.h> +#include <linux/netdevice.h> +#include <linux/ip.h> +#include <linux/etherdevice.h> +#include <net/ip.h> +#include <net/tso.h> + +#include "nic_reg.h" +#include "nic.h" +#include "q_struct.h" +#include "nicvf_queues.h" + +struct rbuf_info { + struct page *page; + void *data; + u64 offset; +}; + +#define GET_RBUF_INFO(x) ((struct rbuf_info *)(x - NICVF_RCV_BUF_ALIGN_BYTES)) + +/* Poll a register for a specific value */ +static int nicvf_poll_reg(struct nicvf *nic, int qidx, + u64 reg, int bit_pos, int bits, int val) +{ + u64 bit_mask; + u64 reg_val; + int timeout = 10; + + bit_mask = (1ULL << bits) - 1; + bit_mask = (bit_mask << bit_pos); + + while (timeout) { + reg_val = nicvf_queue_reg_read(nic, reg, qidx); + if (((reg_val & bit_mask) >> bit_pos) == val) + return 0; + usleep_range(1000, 2000); + timeout--; + } + netdev_err(nic->netdev, "Poll on reg 0x%llx failed\n", reg); + return 1; +} + +/* Allocate memory for a queue's descriptors */ +static int nicvf_alloc_q_desc_mem(struct nicvf *nic, struct q_desc_mem *dmem, + int q_len, int desc_size, int align_bytes) +{ + dmem->q_len = q_len; + dmem->size = (desc_size * q_len) + align_bytes; + /* Save address, need it while freeing */ + dmem->unalign_base = dma_zalloc_coherent(&nic->pdev->dev, dmem->size, + &dmem->dma, GFP_KERNEL); + if (!dmem->unalign_base) + return -ENOMEM; + + /* Align memory address for 'align_bytes' */ + dmem->phys_base = NICVF_ALIGNED_ADDR((u64)dmem->dma, align_bytes); + dmem->base = dmem->unalign_base + (dmem->phys_base - dmem->dma); + return 0; +} + +/* Free queue's descriptor memory */ +static void nicvf_free_q_desc_mem(struct nicvf *nic, struct q_desc_mem *dmem) +{ + if (!dmem) + return; + + dma_free_coherent(&nic->pdev->dev, dmem->size, + dmem->unalign_base, dmem->dma); + dmem->unalign_base = NULL; + dmem->base = NULL; +} + +/* Allocate buffer for packet reception + * HW returns memory address where packet is DMA'ed but not a pointer + * into RBDR ring, so save buffer address at the start of fragment and + * align the start address to a cache aligned address + */ +static inline int nicvf_alloc_rcv_buffer(struct nicvf *nic, gfp_t gfp, + u32 buf_len, u64 **rbuf) +{ + u64 data; + struct rbuf_info *rinfo; + int order = get_order(buf_len); + + /* Check if request can be accomodated in previous allocated page */ + if (nic->rb_page) { + if ((nic->rb_page_offset + buf_len + buf_len) > + (PAGE_SIZE << order)) { + nic->rb_page = NULL; + } else { + nic->rb_page_offset += buf_len; + get_page(nic->rb_page); + } + } + + /* Allocate a new page */ + if (!nic->rb_page) { + nic->rb_page = alloc_pages(gfp | __GFP_COMP | __GFP_NOWARN, + order); + if (!nic->rb_page) { + netdev_err(nic->netdev, + "Failed to allocate new rcv buffer\n"); + return -ENOMEM; + } + nic->rb_page_offset = 0; + } + + data = (u64)page_address(nic->rb_page) + nic->rb_page_offset; + + /* Align buffer addr to cache line i.e 128 bytes */ + rinfo = (struct rbuf_info *)(data + NICVF_RCV_BUF_ALIGN_LEN(data)); + /* Save page address for reference updation */ + rinfo->page = nic->rb_page; + /* Store start address for later retrieval */ + rinfo->data = (void *)data; + /* Store alignment offset */ + rinfo->offset = NICVF_RCV_BUF_ALIGN_LEN(data); + + data += rinfo->offset; + + /* Give next aligned address to hw for DMA */ + *rbuf = (u64 *)(data + NICVF_RCV_BUF_ALIGN_BYTES); + return 0; +} + +/* Retrieve actual buffer start address and build skb for received packet */ +static struct sk_buff *nicvf_rb_ptr_to_skb(struct nicvf *nic, + u64 rb_ptr, int len) +{ + struct sk_buff *skb; + struct rbuf_info *rinfo; + + rb_ptr = (u64)phys_to_virt(rb_ptr); + /* Get buffer start address and alignment offset */ + rinfo = GET_RBUF_INFO(rb_ptr); + + /* Now build an skb to give to stack */ + skb = build_skb(rinfo->data, RCV_FRAG_LEN); + if (!skb) { + put_page(rinfo->page); + return NULL; + } + + /* Set correct skb->data */ + skb_reserve(skb, rinfo->offset + NICVF_RCV_BUF_ALIGN_BYTES); + + prefetch((void *)rb_ptr); + return skb; +} + +/* Allocate RBDR ring and populate receive buffers */ +static int nicvf_init_rbdr(struct nicvf *nic, struct rbdr *rbdr, + int ring_len, int buf_size) +{ + int idx; + u64 *rbuf; + struct rbdr_entry_t *desc; + int err; + + err = nicvf_alloc_q_desc_mem(nic, &rbdr->dmem, ring_len, + sizeof(struct rbdr_entry_t), + NICVF_RCV_BUF_ALIGN_BYTES); + if (err) + return err; + + rbdr->desc = rbdr->dmem.base; + /* Buffer size has to be in multiples of 128 bytes */ + rbdr->dma_size = buf_size; + rbdr->enable = true; + rbdr->thresh = RBDR_THRESH; + + nic->rb_page = NULL; + for (idx = 0; idx < ring_len; idx++) { + err = nicvf_alloc_rcv_buffer(nic, GFP_KERNEL, RCV_FRAG_LEN, + &rbuf); + if (err) + return err; + + desc = GET_RBDR_DESC(rbdr, idx); + desc->buf_addr = virt_to_phys(rbuf) >> NICVF_RCV_BUF_ALIGN; + } + return 0; +} + +/* Free RBDR ring and its receive buffers */ +static void nicvf_free_rbdr(struct nicvf *nic, struct rbdr *rbdr) +{ + int head, tail; + u64 buf_addr; + struct rbdr_entry_t *desc; + struct rbuf_info *rinfo; + + if (!rbdr) + return; + + rbdr->enable = false; + if (!rbdr->dmem.base) + return; + + head = rbdr->head; + tail = rbdr->tail; + + /* Free SKBs */ + while (head != tail) { + desc = GET_RBDR_DESC(rbdr, head); + buf_addr = desc->buf_addr << NICVF_RCV_BUF_ALIGN; + rinfo = GET_RBUF_INFO((u64)phys_to_virt(buf_addr)); + put_page(rinfo->page); + head++; + head &= (rbdr->dmem.q_len - 1); + } + /* Free SKB of tail desc */ + desc = GET_RBDR_DESC(rbdr, tail); + buf_addr = desc->buf_addr << NICVF_RCV_BUF_ALIGN; + rinfo = GET_RBUF_INFO((u64)phys_to_virt(buf_addr)); + put_page(rinfo->page); + + /* Free RBDR ring */ + nicvf_free_q_desc_mem(nic, &rbdr->dmem); +} + +/* Refill receive buffer descriptors with new buffers. + */ +static void nicvf_refill_rbdr(struct nicvf *nic, gfp_t gfp) +{ + struct queue_set *qs = nic->qs; + int rbdr_idx = qs->rbdr_cnt; + int tail, qcount; + int refill_rb_cnt; + struct rbdr *rbdr; + struct rbdr_entry_t *desc; + u64 *rbuf; + int new_rb = 0; + +refill: + if (!rbdr_idx) + return; + rbdr_idx--; + rbdr = &qs->rbdr[rbdr_idx]; + /* Check if it's enabled */ + if (!rbdr->enable) + goto next_rbdr; + + /* Get no of desc's to be refilled */ + qcount = nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_STATUS0, rbdr_idx); + qcount &= 0x7FFFF; + /* Doorbell can be ringed with a max of ring size minus 1 */ + if (qcount >= (qs->rbdr_len - 1)) + goto next_rbdr; + else + refill_rb_cnt = qs->rbdr_len - qcount - 1; + + /* Start filling descs from tail */ + tail = nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_TAIL, rbdr_idx) >> 3; + while (refill_rb_cnt) { + tail++; + tail &= (rbdr->dmem.q_len - 1); + + if (nicvf_alloc_rcv_buffer(nic, gfp, RCV_FRAG_LEN, &rbuf)) + break; + + desc = GET_RBDR_DESC(rbdr, tail); + desc->buf_addr = virt_to_phys(rbuf) >> NICVF_RCV_BUF_ALIGN; + refill_rb_cnt--; + new_rb++; + } + + /* make sure all memory stores are done before ringing doorbell */ + smp_wmb(); + + /* Check if buffer allocation failed */ + if (refill_rb_cnt) + nic->rb_alloc_fail = true; + else + nic->rb_alloc_fail = false; + + /* Notify HW */ + nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_DOOR, + rbdr_idx, new_rb); +next_rbdr: + /* Re-enable RBDR interrupts only if buffer allocation is success */ + if (!nic->rb_alloc_fail && rbdr->enable) + nicvf_enable_intr(nic, NICVF_INTR_RBDR, rbdr_idx); + + if (rbdr_idx) + goto refill; +} + +/* Alloc rcv buffers in non-atomic mode for better success */ +void nicvf_rbdr_work(struct work_struct *work) +{ + struct nicvf *nic = container_of(work, struct nicvf, rbdr_work.work); + + nicvf_refill_rbdr(nic, GFP_KERNEL); + if (nic->rb_alloc_fail) + schedule_delayed_work(&nic->rbdr_work, msecs_to_jiffies(10)); + else + nic->rb_work_scheduled = false; +} + +/* In Softirq context, alloc rcv buffers in atomic mode */ +void nicvf_rbdr_task(unsigned long data) +{ + struct nicvf *nic = (struct nicvf *)data; + + nicvf_refill_rbdr(nic, GFP_ATOMIC); + if (nic->rb_alloc_fail) { + nic->rb_work_scheduled = true; + schedule_delayed_work(&nic->rbdr_work, msecs_to_jiffies(10)); + } +} + +/* Initialize completion queue */ +static int nicvf_init_cmp_queue(struct nicvf *nic, + struct cmp_queue *cq, int q_len) +{ + int err; + + err = nicvf_alloc_q_desc_mem(nic, &cq->dmem, q_len, CMP_QUEUE_DESC_SIZE, + NICVF_CQ_BASE_ALIGN_BYTES); + if (err) + return err; + + cq->desc = cq->dmem.base; + cq->thresh = CMP_QUEUE_CQE_THRESH; + nic->cq_coalesce_usecs = (CMP_QUEUE_TIMER_THRESH * 0.05) - 1; + + return 0; +} + +static void nicvf_free_cmp_queue(struct nicvf *nic, struct cmp_queue *cq) +{ + if (!cq) + return; + if (!cq->dmem.base) + return; + + nicvf_free_q_desc_mem(nic, &cq->dmem); +} + +/* Initialize transmit queue */ +static int nicvf_init_snd_queue(struct nicvf *nic, + struct snd_queue *sq, int q_len) +{ + int err; + + err = nicvf_alloc_q_desc_mem(nic, &sq->dmem, q_len, SND_QUEUE_DESC_SIZE, + NICVF_SQ_BASE_ALIGN_BYTES); + if (err) + return err; + + sq->desc = sq->dmem.base; + sq->skbuff = kcalloc(q_len, sizeof(u64), GFP_KERNEL); + if (!sq->skbuff) + return -ENOMEM; + sq->head = 0; + sq->tail = 0; + atomic_set(&sq->free_cnt, q_len - 1); + sq->thresh = SND_QUEUE_THRESH; + + /* Preallocate memory for TSO segment's header */ + sq->tso_hdrs = dma_alloc_coherent(&nic->pdev->dev, + q_len * TSO_HEADER_SIZE, + &sq->tso_hdrs_phys, GFP_KERNEL); + if (!sq->tso_hdrs) + return -ENOMEM; + + return 0; +} + +static void nicvf_free_snd_queue(struct nicvf *nic, struct snd_queue *sq) +{ + if (!sq) + return; + if (!sq->dmem.base) + return; + + if (sq->tso_hdrs) + dma_free_coherent(&nic->pdev->dev, + sq->dmem.q_len * TSO_HEADER_SIZE, + sq->tso_hdrs, sq->tso_hdrs_phys); + + kfree(sq->skbuff); + nicvf_free_q_desc_mem(nic, &sq->dmem); +} + +static void nicvf_reclaim_snd_queue(struct nicvf *nic, + struct queue_set *qs, int qidx) +{ + /* Disable send queue */ + nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, 0); + /* Check if SQ is stopped */ + if (nicvf_poll_reg(nic, qidx, NIC_QSET_SQ_0_7_STATUS, 21, 1, 0x01)) + return; + /* Reset send queue */ + nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, NICVF_SQ_RESET); +} + +static void nicvf_reclaim_rcv_queue(struct nicvf *nic, + struct queue_set *qs, int qidx) +{ + union nic_mbx mbx = {}; + + /* Make sure all packets in the pipeline are written back into mem */ + mbx.msg.msg = NIC_MBOX_MSG_RQ_SW_SYNC; + nicvf_send_msg_to_pf(nic, &mbx); +} + +static void nicvf_reclaim_cmp_queue(struct nicvf *nic, + struct queue_set *qs, int qidx) +{ + /* Disable timer threshold (doesn't get reset upon CQ reset */ + nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG2, qidx, 0); + /* Disable completion queue */ + nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG, qidx, 0); + /* Reset completion queue */ + nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG, qidx, NICVF_CQ_RESET); +} + +static void nicvf_reclaim_rbdr(struct nicvf *nic, + struct rbdr *rbdr, int qidx) +{ + u64 tmp, fifo_state; + int timeout = 10; + + /* Save head and tail pointers for feeing up buffers */ + rbdr->head = nicvf_queue_reg_read(nic, + NIC_QSET_RBDR_0_1_HEAD, + qidx) >> 3; + rbdr->tail = nicvf_queue_reg_read(nic, + NIC_QSET_RBDR_0_1_TAIL, + qidx) >> 3; + + /* If RBDR FIFO is in 'FAIL' state then do a reset first + * before relaiming. + */ + fifo_state = nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_STATUS0, qidx); + if (((fifo_state >> 62) & 0x03) == 0x3) + nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG, + qidx, NICVF_RBDR_RESET); + + /* Disable RBDR */ + nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG, qidx, 0); + if (nicvf_poll_reg(nic, qidx, NIC_QSET_RBDR_0_1_STATUS0, 62, 2, 0x00)) + return; + while (1) { + tmp = nicvf_queue_reg_read(nic, + NIC_QSET_RBDR_0_1_PREFETCH_STATUS, + qidx); + if ((tmp & 0xFFFFFFFF) == ((tmp >> 32) & 0xFFFFFFFF)) + break; + usleep_range(1000, 2000); + timeout--; + if (!timeout) { + netdev_err(nic->netdev, + "Failed polling on prefetch status\n"); + return; + } + } + nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG, + qidx, NICVF_RBDR_RESET); + + if (nicvf_poll_reg(nic, qidx, NIC_QSET_RBDR_0_1_STATUS0, 62, 2, 0x02)) + return; + nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG, qidx, 0x00); + if (nicvf_poll_reg(nic, qidx, NIC_QSET_RBDR_0_1_STATUS0, 62, 2, 0x00)) + return; +} + +void nicvf_config_vlan_stripping(struct nicvf *nic, netdev_features_t features) +{ + u64 rq_cfg; + int sqs; + + rq_cfg = nicvf_queue_reg_read(nic, NIC_QSET_RQ_GEN_CFG, 0); + + /* Enable first VLAN stripping */ + if (features & NETIF_F_HW_VLAN_CTAG_RX) + rq_cfg |= (1ULL << 25); + else + rq_cfg &= ~(1ULL << 25); + nicvf_queue_reg_write(nic, NIC_QSET_RQ_GEN_CFG, 0, rq_cfg); + + /* Configure Secondary Qsets, if any */ + for (sqs = 0; sqs < nic->sqs_count; sqs++) + if (nic->snicvf[sqs]) + nicvf_queue_reg_write(nic->snicvf[sqs], + NIC_QSET_RQ_GEN_CFG, 0, rq_cfg); +} + +/* Configures receive queue */ +static void nicvf_rcv_queue_config(struct nicvf *nic, struct queue_set *qs, + int qidx, bool enable) +{ + union nic_mbx mbx = {}; + struct rcv_queue *rq; + struct rq_cfg rq_cfg; + + rq = &qs->rq[qidx]; + rq->enable = enable; + + /* Disable receive queue */ + nicvf_queue_reg_write(nic, NIC_QSET_RQ_0_7_CFG, qidx, 0); + + if (!rq->enable) { + nicvf_reclaim_rcv_queue(nic, qs, qidx); + return; + } + + rq->cq_qs = qs->vnic_id; + rq->cq_idx = qidx; + rq->start_rbdr_qs = qs->vnic_id; + rq->start_qs_rbdr_idx = qs->rbdr_cnt - 1; + rq->cont_rbdr_qs = qs->vnic_id; + rq->cont_qs_rbdr_idx = qs->rbdr_cnt - 1; + /* all writes of RBDR data to be loaded into L2 Cache as well*/ + rq->caching = 1; + + /* Send a mailbox msg to PF to config RQ */ + mbx.rq.msg = NIC_MBOX_MSG_RQ_CFG; + mbx.rq.qs_num = qs->vnic_id; + mbx.rq.rq_num = qidx; + mbx.rq.cfg = (rq->caching << 26) | (rq->cq_qs << 19) | + (rq->cq_idx << 16) | (rq->cont_rbdr_qs << 9) | + (rq->cont_qs_rbdr_idx << 8) | + (rq->start_rbdr_qs << 1) | (rq->start_qs_rbdr_idx); + nicvf_send_msg_to_pf(nic, &mbx); + + mbx.rq.msg = NIC_MBOX_MSG_RQ_BP_CFG; + mbx.rq.cfg = (1ULL << 63) | (1ULL << 62) | (qs->vnic_id << 0); + nicvf_send_msg_to_pf(nic, &mbx); + + /* RQ drop config + * Enable CQ drop to reserve sufficient CQEs for all tx packets + */ + mbx.rq.msg = NIC_MBOX_MSG_RQ_DROP_CFG; + mbx.rq.cfg = (1ULL << 62) | (RQ_CQ_DROP << 8); + nicvf_send_msg_to_pf(nic, &mbx); + + nicvf_queue_reg_write(nic, NIC_QSET_RQ_GEN_CFG, 0, 0x00); + if (!nic->sqs_mode) + nicvf_config_vlan_stripping(nic, nic->netdev->features); + + /* Enable Receive queue */ + rq_cfg.ena = 1; + rq_cfg.tcp_ena = 0; + nicvf_queue_reg_write(nic, NIC_QSET_RQ_0_7_CFG, qidx, *(u64 *)&rq_cfg); +} + +/* Configures completion queue */ +void nicvf_cmp_queue_config(struct nicvf *nic, struct queue_set *qs, + int qidx, bool enable) +{ + struct cmp_queue *cq; + struct cq_cfg cq_cfg; + + cq = &qs->cq[qidx]; + cq->enable = enable; + + if (!cq->enable) { + nicvf_reclaim_cmp_queue(nic, qs, qidx); + return; + } + + /* Reset completion queue */ + nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG, qidx, NICVF_CQ_RESET); + + if (!cq->enable) + return; + + spin_lock_init(&cq->lock); + /* Set completion queue base address */ + nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_BASE, + qidx, (u64)(cq->dmem.phys_base)); + + /* Enable Completion queue */ + cq_cfg.ena = 1; + cq_cfg.reset = 0; + cq_cfg.caching = 0; + cq_cfg.qsize = CMP_QSIZE; + cq_cfg.avg_con = 0; + nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG, qidx, *(u64 *)&cq_cfg); + + /* Set threshold value for interrupt generation */ + nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_THRESH, qidx, cq->thresh); + nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG2, + qidx, CMP_QUEUE_TIMER_THRESH); +} + +/* Configures transmit queue */ +static void nicvf_snd_queue_config(struct nicvf *nic, struct queue_set *qs, + int qidx, bool enable) +{ + union nic_mbx mbx = {}; + struct snd_queue *sq; + struct sq_cfg sq_cfg; + + sq = &qs->sq[qidx]; + sq->enable = enable; + + if (!sq->enable) { + nicvf_reclaim_snd_queue(nic, qs, qidx); + return; + } + + /* Reset send queue */ + nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, NICVF_SQ_RESET); + + sq->cq_qs = qs->vnic_id; + sq->cq_idx = qidx; + + /* Send a mailbox msg to PF to config SQ */ + mbx.sq.msg = NIC_MBOX_MSG_SQ_CFG; + mbx.sq.qs_num = qs->vnic_id; + mbx.sq.sq_num = qidx; + mbx.sq.sqs_mode = nic->sqs_mode; + mbx.sq.cfg = (sq->cq_qs << 3) | sq->cq_idx; + nicvf_send_msg_to_pf(nic, &mbx); + + /* Set queue base address */ + nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_BASE, + qidx, (u64)(sq->dmem.phys_base)); + + /* Enable send queue & set queue size */ + sq_cfg.ena = 1; + sq_cfg.reset = 0; + sq_cfg.ldwb = 0; + sq_cfg.qsize = SND_QSIZE; + sq_cfg.tstmp_bgx_intf = 0; + nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, *(u64 *)&sq_cfg); + + /* Set threshold value for interrupt generation */ + nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_THRESH, qidx, sq->thresh); + + /* Set queue:cpu affinity for better load distribution */ + if (cpu_online(qidx)) { + cpumask_set_cpu(qidx, &sq->affinity_mask); + netif_set_xps_queue(nic->netdev, + &sq->affinity_mask, qidx); + } +} + +/* Configures receive buffer descriptor ring */ +static void nicvf_rbdr_config(struct nicvf *nic, struct queue_set *qs, + int qidx, bool enable) +{ + struct rbdr *rbdr; + struct rbdr_cfg rbdr_cfg; + + rbdr = &qs->rbdr[qidx]; + nicvf_reclaim_rbdr(nic, rbdr, qidx); + if (!enable) + return; + + /* Set descriptor base address */ + nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_BASE, + qidx, (u64)(rbdr->dmem.phys_base)); + + /* Enable RBDR & set queue size */ + /* Buffer size should be in multiples of 128 bytes */ + rbdr_cfg.ena = 1; + rbdr_cfg.reset = 0; + rbdr_cfg.ldwb = 0; + rbdr_cfg.qsize = RBDR_SIZE; + rbdr_cfg.avg_con = 0; + rbdr_cfg.lines = rbdr->dma_size / 128; + nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG, + qidx, *(u64 *)&rbdr_cfg); + + /* Notify HW */ + nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_DOOR, + qidx, qs->rbdr_len - 1); + + /* Set threshold value for interrupt generation */ + nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_THRESH, + qidx, rbdr->thresh - 1); +} + +/* Requests PF to assign and enable Qset */ +void nicvf_qset_config(struct nicvf *nic, bool enable) +{ + union nic_mbx mbx = {}; + struct queue_set *qs = nic->qs; + struct qs_cfg *qs_cfg; + + if (!qs) { + netdev_warn(nic->netdev, + "Qset is still not allocated, don't init queues\n"); + return; + } + + qs->enable = enable; + qs->vnic_id = nic->vf_id; + + /* Send a mailbox msg to PF to config Qset */ + mbx.qs.msg = NIC_MBOX_MSG_QS_CFG; + mbx.qs.num = qs->vnic_id; + mbx.qs.sqs_count = nic->sqs_count; + + mbx.qs.cfg = 0; + qs_cfg = (struct qs_cfg *)&mbx.qs.cfg; + if (qs->enable) { + qs_cfg->ena = 1; +#ifdef __BIG_ENDIAN + qs_cfg->be = 1; +#endif + qs_cfg->vnic = qs->vnic_id; + } + nicvf_send_msg_to_pf(nic, &mbx); +} + +static void nicvf_free_resources(struct nicvf *nic) +{ + int qidx; + struct queue_set *qs = nic->qs; + + /* Free receive buffer descriptor ring */ + for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) + nicvf_free_rbdr(nic, &qs->rbdr[qidx]); + + /* Free completion queue */ + for (qidx = 0; qidx < qs->cq_cnt; qidx++) + nicvf_free_cmp_queue(nic, &qs->cq[qidx]); + + /* Free send queue */ + for (qidx = 0; qidx < qs->sq_cnt; qidx++) + nicvf_free_snd_queue(nic, &qs->sq[qidx]); +} + +static int nicvf_alloc_resources(struct nicvf *nic) +{ + int qidx; + struct queue_set *qs = nic->qs; + + /* Alloc receive buffer descriptor ring */ + for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) { + if (nicvf_init_rbdr(nic, &qs->rbdr[qidx], qs->rbdr_len, + DMA_BUFFER_LEN)) + goto alloc_fail; + } + + /* Alloc send queue */ + for (qidx = 0; qidx < qs->sq_cnt; qidx++) { + if (nicvf_init_snd_queue(nic, &qs->sq[qidx], qs->sq_len)) + goto alloc_fail; + } + + /* Alloc completion queue */ + for (qidx = 0; qidx < qs->cq_cnt; qidx++) { + if (nicvf_init_cmp_queue(nic, &qs->cq[qidx], qs->cq_len)) + goto alloc_fail; + } + + return 0; +alloc_fail: + nicvf_free_resources(nic); + return -ENOMEM; +} + +int nicvf_set_qset_resources(struct nicvf *nic) +{ + struct queue_set *qs; + + qs = devm_kzalloc(&nic->pdev->dev, sizeof(*qs), GFP_KERNEL); + if (!qs) + return -ENOMEM; + nic->qs = qs; + + /* Set count of each queue */ + qs->rbdr_cnt = RBDR_CNT; + qs->rq_cnt = RCV_QUEUE_CNT; + qs->sq_cnt = SND_QUEUE_CNT; + qs->cq_cnt = CMP_QUEUE_CNT; + + /* Set queue lengths */ + qs->rbdr_len = RCV_BUF_COUNT; + qs->sq_len = SND_QUEUE_LEN; + qs->cq_len = CMP_QUEUE_LEN; + + nic->rx_queues = qs->rq_cnt; + nic->tx_queues = qs->sq_cnt; + + return 0; +} + +int nicvf_config_data_transfer(struct nicvf *nic, bool enable) +{ + bool disable = false; + struct queue_set *qs = nic->qs; + int qidx; + + if (!qs) + return 0; + + if (enable) { + if (nicvf_alloc_resources(nic)) + return -ENOMEM; + + for (qidx = 0; qidx < qs->sq_cnt; qidx++) + nicvf_snd_queue_config(nic, qs, qidx, enable); + for (qidx = 0; qidx < qs->cq_cnt; qidx++) + nicvf_cmp_queue_config(nic, qs, qidx, enable); + for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) + nicvf_rbdr_config(nic, qs, qidx, enable); + for (qidx = 0; qidx < qs->rq_cnt; qidx++) + nicvf_rcv_queue_config(nic, qs, qidx, enable); + } else { + for (qidx = 0; qidx < qs->rq_cnt; qidx++) + nicvf_rcv_queue_config(nic, qs, qidx, disable); + for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) + nicvf_rbdr_config(nic, qs, qidx, disable); + for (qidx = 0; qidx < qs->sq_cnt; qidx++) + nicvf_snd_queue_config(nic, qs, qidx, disable); + for (qidx = 0; qidx < qs->cq_cnt; qidx++) + nicvf_cmp_queue_config(nic, qs, qidx, disable); + + nicvf_free_resources(nic); + } + + return 0; +} + +/* Get a free desc from SQ + * returns descriptor ponter & descriptor number + */ +static inline int nicvf_get_sq_desc(struct snd_queue *sq, int desc_cnt) +{ + int qentry; + + qentry = sq->tail; + atomic_sub(desc_cnt, &sq->free_cnt); + sq->tail += desc_cnt; + sq->tail &= (sq->dmem.q_len - 1); + + return qentry; +} + +/* Free descriptor back to SQ for future use */ +void nicvf_put_sq_desc(struct snd_queue *sq, int desc_cnt) +{ + atomic_add(desc_cnt, &sq->free_cnt); + sq->head += desc_cnt; + sq->head &= (sq->dmem.q_len - 1); +} + +static inline int nicvf_get_nxt_sqentry(struct snd_queue *sq, int qentry) +{ + qentry++; + qentry &= (sq->dmem.q_len - 1); + return qentry; +} + +void nicvf_sq_enable(struct nicvf *nic, struct snd_queue *sq, int qidx) +{ + u64 sq_cfg; + + sq_cfg = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_CFG, qidx); + sq_cfg |= NICVF_SQ_EN; + nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, sq_cfg); + /* Ring doorbell so that H/W restarts processing SQEs */ + nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_DOOR, qidx, 0); +} + +void nicvf_sq_disable(struct nicvf *nic, int qidx) +{ + u64 sq_cfg; + + sq_cfg = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_CFG, qidx); + sq_cfg &= ~NICVF_SQ_EN; + nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, sq_cfg); +} + +void nicvf_sq_free_used_descs(struct net_device *netdev, struct snd_queue *sq, + int qidx) +{ + u64 head, tail; + struct sk_buff *skb; + struct nicvf *nic = netdev_priv(netdev); + struct sq_hdr_subdesc *hdr; + + head = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_HEAD, qidx) >> 4; + tail = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_TAIL, qidx) >> 4; + while (sq->head != head) { + hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, sq->head); + if (hdr->subdesc_type != SQ_DESC_TYPE_HEADER) { + nicvf_put_sq_desc(sq, 1); + continue; + } + skb = (struct sk_buff *)sq->skbuff[sq->head]; + if (skb) + dev_kfree_skb_any(skb); + atomic64_add(1, (atomic64_t *)&netdev->stats.tx_packets); + atomic64_add(hdr->tot_len, + (atomic64_t *)&netdev->stats.tx_bytes); + nicvf_put_sq_desc(sq, hdr->subdesc_cnt + 1); + } +} + +/* Calculate no of SQ subdescriptors needed to transmit all + * segments of this TSO packet. + * Taken from 'Tilera network driver' with a minor modification. + */ +static int nicvf_tso_count_subdescs(struct sk_buff *skb) +{ + struct skb_shared_info *sh = skb_shinfo(skb); + unsigned int sh_len = skb_transport_offset(skb) + tcp_hdrlen(skb); + unsigned int data_len = skb->len - sh_len; + unsigned int p_len = sh->gso_size; + long f_id = -1; /* id of the current fragment */ + long f_size = skb_headlen(skb) - sh_len; /* current fragment size */ + long f_used = 0; /* bytes used from the current fragment */ + long n; /* size of the current piece of payload */ + int num_edescs = 0; + int segment; + + for (segment = 0; segment < sh->gso_segs; segment++) { + unsigned int p_used = 0; + + /* One edesc for header and for each piece of the payload. */ + for (num_edescs++; p_used < p_len; num_edescs++) { + /* Advance as needed. */ + while (f_used >= f_size) { + f_id++; + f_size = skb_frag_size(&sh->frags[f_id]); + f_used = 0; + } + + /* Use bytes from the current fragment. */ + n = p_len - p_used; + if (n > f_size - f_used) + n = f_size - f_used; + f_used += n; + p_used += n; + } + + /* The last segment may be less than gso_size. */ + data_len -= p_len; + if (data_len < p_len) + p_len = data_len; + } + + /* '+ gso_segs' for SQ_HDR_SUDESCs for each segment */ + return num_edescs + sh->gso_segs; +} + +/* Get the number of SQ descriptors needed to xmit this skb */ +static int nicvf_sq_subdesc_required(struct nicvf *nic, struct sk_buff *skb) +{ + int subdesc_cnt = MIN_SQ_DESC_PER_PKT_XMIT; + + if (skb_shinfo(skb)->gso_size) { + subdesc_cnt = nicvf_tso_count_subdescs(skb); + return subdesc_cnt; + } + + if (skb_shinfo(skb)->nr_frags) + subdesc_cnt += skb_shinfo(skb)->nr_frags; + + return subdesc_cnt; +} + +/* Add SQ HEADER subdescriptor. + * First subdescriptor for every send descriptor. + */ +static inline void +nicvf_sq_add_hdr_subdesc(struct snd_queue *sq, int qentry, + int subdesc_cnt, struct sk_buff *skb, int len) +{ + int proto; + struct sq_hdr_subdesc *hdr; + + hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, qentry); + sq->skbuff[qentry] = (u64)skb; + + memset(hdr, 0, SND_QUEUE_DESC_SIZE); + hdr->subdesc_type = SQ_DESC_TYPE_HEADER; + /* Enable notification via CQE after processing SQE */ + hdr->post_cqe = 1; + /* No of subdescriptors following this */ + hdr->subdesc_cnt = subdesc_cnt; + hdr->tot_len = len; + + /* Offload checksum calculation to HW */ + if (skb->ip_summed == CHECKSUM_PARTIAL) { + hdr->csum_l3 = 1; /* Enable IP csum calculation */ + hdr->l3_offset = skb_network_offset(skb); + hdr->l4_offset = skb_transport_offset(skb); + + proto = ip_hdr(skb)->protocol; + switch (proto) { + case IPPROTO_TCP: + hdr->csum_l4 = SEND_L4_CSUM_TCP; + break; + case IPPROTO_UDP: + hdr->csum_l4 = SEND_L4_CSUM_UDP; + break; + case IPPROTO_SCTP: + hdr->csum_l4 = SEND_L4_CSUM_SCTP; + break; + } + } +} + +/* SQ GATHER subdescriptor + * Must follow HDR descriptor + */ +static inline void nicvf_sq_add_gather_subdesc(struct snd_queue *sq, int qentry, + int size, u64 data) +{ + struct sq_gather_subdesc *gather; + + qentry &= (sq->dmem.q_len - 1); + gather = (struct sq_gather_subdesc *)GET_SQ_DESC(sq, qentry); + + memset(gather, 0, SND_QUEUE_DESC_SIZE); + gather->subdesc_type = SQ_DESC_TYPE_GATHER; + gather->ld_type = NIC_SEND_LD_TYPE_E_LDD; + gather->size = size; + gather->addr = data; +} + +/* Segment a TSO packet into 'gso_size' segments and append + * them to SQ for transfer + */ +static int nicvf_sq_append_tso(struct nicvf *nic, struct snd_queue *sq, + int sq_num, int qentry, struct sk_buff *skb) +{ + struct tso_t tso; + int seg_subdescs = 0, desc_cnt = 0; + int seg_len, total_len, data_left; + int hdr_qentry = qentry; + int hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); + + tso_start(skb, &tso); + total_len = skb->len - hdr_len; + while (total_len > 0) { + char *hdr; + + /* Save Qentry for adding HDR_SUBDESC at the end */ + hdr_qentry = qentry; + + data_left = min_t(int, skb_shinfo(skb)->gso_size, total_len); + total_len -= data_left; + + /* Add segment's header */ + qentry = nicvf_get_nxt_sqentry(sq, qentry); + hdr = sq->tso_hdrs + qentry * TSO_HEADER_SIZE; + tso_build_hdr(skb, hdr, &tso, data_left, total_len == 0); + nicvf_sq_add_gather_subdesc(sq, qentry, hdr_len, + sq->tso_hdrs_phys + + qentry * TSO_HEADER_SIZE); + /* HDR_SUDESC + GATHER */ + seg_subdescs = 2; + seg_len = hdr_len; + + /* Add segment's payload fragments */ + while (data_left > 0) { + int size; + + size = min_t(int, tso.size, data_left); + + qentry = nicvf_get_nxt_sqentry(sq, qentry); + nicvf_sq_add_gather_subdesc(sq, qentry, size, + virt_to_phys(tso.data)); + seg_subdescs++; + seg_len += size; + + data_left -= size; + tso_build_data(skb, &tso, size); + } + nicvf_sq_add_hdr_subdesc(sq, hdr_qentry, + seg_subdescs - 1, skb, seg_len); + sq->skbuff[hdr_qentry] = (u64)NULL; + qentry = nicvf_get_nxt_sqentry(sq, qentry); + + desc_cnt += seg_subdescs; + } + /* Save SKB in the last segment for freeing */ + sq->skbuff[hdr_qentry] = (u64)skb; + + /* make sure all memory stores are done before ringing doorbell */ + smp_wmb(); + + /* Inform HW to xmit all TSO segments */ + nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_DOOR, + sq_num, desc_cnt); + nic->drv_stats.tx_tso++; + return 1; +} + +/* Append an skb to a SQ for packet transfer. */ +int nicvf_sq_append_skb(struct nicvf *nic, struct sk_buff *skb) +{ + int i, size; + int subdesc_cnt; + int sq_num, qentry; + struct queue_set *qs; + struct snd_queue *sq; + + sq_num = skb_get_queue_mapping(skb); + if (sq_num >= MAX_SND_QUEUES_PER_QS) { + /* Get secondary Qset's SQ structure */ + i = sq_num / MAX_SND_QUEUES_PER_QS; + if (!nic->snicvf[i - 1]) { + netdev_warn(nic->netdev, + "Secondary Qset#%d's ptr not initialized\n", + i - 1); + return 1; + } + nic = (struct nicvf *)nic->snicvf[i - 1]; + sq_num = sq_num % MAX_SND_QUEUES_PER_QS; + } + + qs = nic->qs; + sq = &qs->sq[sq_num]; + + subdesc_cnt = nicvf_sq_subdesc_required(nic, skb); + if (subdesc_cnt > atomic_read(&sq->free_cnt)) + goto append_fail; + + qentry = nicvf_get_sq_desc(sq, subdesc_cnt); + + /* Check if its a TSO packet */ + if (skb_shinfo(skb)->gso_size) + return nicvf_sq_append_tso(nic, sq, sq_num, qentry, skb); + + /* Add SQ header subdesc */ + nicvf_sq_add_hdr_subdesc(sq, qentry, subdesc_cnt - 1, skb, skb->len); + + /* Add SQ gather subdescs */ + qentry = nicvf_get_nxt_sqentry(sq, qentry); + size = skb_is_nonlinear(skb) ? skb_headlen(skb) : skb->len; + nicvf_sq_add_gather_subdesc(sq, qentry, size, virt_to_phys(skb->data)); + + /* Check for scattered buffer */ + if (!skb_is_nonlinear(skb)) + goto doorbell; + + for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { + const struct skb_frag_struct *frag; + + frag = &skb_shinfo(skb)->frags[i]; + + qentry = nicvf_get_nxt_sqentry(sq, qentry); + size = skb_frag_size(frag); + nicvf_sq_add_gather_subdesc(sq, qentry, size, + virt_to_phys( + skb_frag_address(frag))); + } + +doorbell: + /* make sure all memory stores are done before ringing doorbell */ + smp_wmb(); + + /* Inform HW to xmit new packet */ + nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_DOOR, + sq_num, subdesc_cnt); + return 1; + +append_fail: + /* Use original PCI dev for debug log */ + nic = nic->pnicvf; + netdev_dbg(nic->netdev, "Not enough SQ descriptors to xmit pkt\n"); + return 0; +} + +static inline unsigned frag_num(unsigned i) +{ +#ifdef __BIG_ENDIAN + return (i & ~3) + 3 - (i & 3); +#else + return i; +#endif +} + +/* Returns SKB for a received packet */ +struct sk_buff *nicvf_get_rcv_skb(struct nicvf *nic, struct cqe_rx_t *cqe_rx) +{ + int frag; + int payload_len = 0; + struct sk_buff *skb = NULL; + struct sk_buff *skb_frag = NULL; + struct sk_buff *prev_frag = NULL; + u16 *rb_lens = NULL; + u64 *rb_ptrs = NULL; + + rb_lens = (void *)cqe_rx + (3 * sizeof(u64)); + rb_ptrs = (void *)cqe_rx + (6 * sizeof(u64)); + + netdev_dbg(nic->netdev, "%s rb_cnt %d rb0_ptr %llx rb0_sz %d\n", + __func__, cqe_rx->rb_cnt, cqe_rx->rb0_ptr, cqe_rx->rb0_sz); + + for (frag = 0; frag < cqe_rx->rb_cnt; frag++) { + payload_len = rb_lens[frag_num(frag)]; + if (!frag) { + /* First fragment */ + skb = nicvf_rb_ptr_to_skb(nic, + *rb_ptrs - cqe_rx->align_pad, + payload_len); + if (!skb) + return NULL; + skb_reserve(skb, cqe_rx->align_pad); + skb_put(skb, payload_len); + } else { + /* Add fragments */ + skb_frag = nicvf_rb_ptr_to_skb(nic, *rb_ptrs, + payload_len); + if (!skb_frag) { + dev_kfree_skb(skb); + return NULL; + } + + if (!skb_shinfo(skb)->frag_list) + skb_shinfo(skb)->frag_list = skb_frag; + else + prev_frag->next = skb_frag; + + prev_frag = skb_frag; + skb->len += payload_len; + skb->data_len += payload_len; + skb_frag->len = payload_len; + } + /* Next buffer pointer */ + rb_ptrs++; + } + return skb; +} + +/* Enable interrupt */ +void nicvf_enable_intr(struct nicvf *nic, int int_type, int q_idx) +{ + u64 reg_val; + + reg_val = nicvf_reg_read(nic, NIC_VF_ENA_W1S); + + switch (int_type) { + case NICVF_INTR_CQ: + reg_val |= ((1ULL << q_idx) << NICVF_INTR_CQ_SHIFT); + break; + case NICVF_INTR_SQ: + reg_val |= ((1ULL << q_idx) << NICVF_INTR_SQ_SHIFT); + break; + case NICVF_INTR_RBDR: + reg_val |= ((1ULL << q_idx) << NICVF_INTR_RBDR_SHIFT); + break; + case NICVF_INTR_PKT_DROP: + reg_val |= (1ULL << NICVF_INTR_PKT_DROP_SHIFT); + break; + case NICVF_INTR_TCP_TIMER: + reg_val |= (1ULL << NICVF_INTR_TCP_TIMER_SHIFT); + break; + case NICVF_INTR_MBOX: + reg_val |= (1ULL << NICVF_INTR_MBOX_SHIFT); + break; + case NICVF_INTR_QS_ERR: + reg_val |= (1ULL << NICVF_INTR_QS_ERR_SHIFT); + break; + default: + netdev_err(nic->netdev, + "Failed to enable interrupt: unknown type\n"); + break; + } + + nicvf_reg_write(nic, NIC_VF_ENA_W1S, reg_val); +} + +/* Disable interrupt */ +void nicvf_disable_intr(struct nicvf *nic, int int_type, int q_idx) +{ + u64 reg_val = 0; + + switch (int_type) { + case NICVF_INTR_CQ: + reg_val |= ((1ULL << q_idx) << NICVF_INTR_CQ_SHIFT); + break; + case NICVF_INTR_SQ: + reg_val |= ((1ULL << q_idx) << NICVF_INTR_SQ_SHIFT); + break; + case NICVF_INTR_RBDR: + reg_val |= ((1ULL << q_idx) << NICVF_INTR_RBDR_SHIFT); + break; + case NICVF_INTR_PKT_DROP: + reg_val |= (1ULL << NICVF_INTR_PKT_DROP_SHIFT); + break; + case NICVF_INTR_TCP_TIMER: + reg_val |= (1ULL << NICVF_INTR_TCP_TIMER_SHIFT); + break; + case NICVF_INTR_MBOX: + reg_val |= (1ULL << NICVF_INTR_MBOX_SHIFT); + break; + case NICVF_INTR_QS_ERR: + reg_val |= (1ULL << NICVF_INTR_QS_ERR_SHIFT); + break; + default: + netdev_err(nic->netdev, + "Failed to disable interrupt: unknown type\n"); + break; + } + + nicvf_reg_write(nic, NIC_VF_ENA_W1C, reg_val); +} + +/* Clear interrupt */ +void nicvf_clear_intr(struct nicvf *nic, int int_type, int q_idx) +{ + u64 reg_val = 0; + + switch (int_type) { + case NICVF_INTR_CQ: + reg_val = ((1ULL << q_idx) << NICVF_INTR_CQ_SHIFT); + break; + case NICVF_INTR_SQ: + reg_val = ((1ULL << q_idx) << NICVF_INTR_SQ_SHIFT); + break; + case NICVF_INTR_RBDR: + reg_val = ((1ULL << q_idx) << NICVF_INTR_RBDR_SHIFT); + break; + case NICVF_INTR_PKT_DROP: + reg_val = (1ULL << NICVF_INTR_PKT_DROP_SHIFT); + break; + case NICVF_INTR_TCP_TIMER: + reg_val = (1ULL << NICVF_INTR_TCP_TIMER_SHIFT); + break; + case NICVF_INTR_MBOX: + reg_val = (1ULL << NICVF_INTR_MBOX_SHIFT); + break; + case NICVF_INTR_QS_ERR: + reg_val |= (1ULL << NICVF_INTR_QS_ERR_SHIFT); + break; + default: + netdev_err(nic->netdev, + "Failed to clear interrupt: unknown type\n"); + break; + } + + nicvf_reg_write(nic, NIC_VF_INT, reg_val); +} + +/* Check if interrupt is enabled */ +int nicvf_is_intr_enabled(struct nicvf *nic, int int_type, int q_idx) +{ + u64 reg_val; + u64 mask = 0xff; + + reg_val = nicvf_reg_read(nic, NIC_VF_ENA_W1S); + + switch (int_type) { + case NICVF_INTR_CQ: + mask = ((1ULL << q_idx) << NICVF_INTR_CQ_SHIFT); + break; + case NICVF_INTR_SQ: + mask = ((1ULL << q_idx) << NICVF_INTR_SQ_SHIFT); + break; + case NICVF_INTR_RBDR: + mask = ((1ULL << q_idx) << NICVF_INTR_RBDR_SHIFT); + break; + case NICVF_INTR_PKT_DROP: + mask = NICVF_INTR_PKT_DROP_MASK; + break; + case NICVF_INTR_TCP_TIMER: + mask = NICVF_INTR_TCP_TIMER_MASK; + break; + case NICVF_INTR_MBOX: + mask = NICVF_INTR_MBOX_MASK; + break; + case NICVF_INTR_QS_ERR: + mask = NICVF_INTR_QS_ERR_MASK; + break; + default: + netdev_err(nic->netdev, + "Failed to check interrupt enable: unknown type\n"); + break; + } + + return (reg_val & mask); +} + +void nicvf_update_rq_stats(struct nicvf *nic, int rq_idx) +{ + struct rcv_queue *rq; + +#define GET_RQ_STATS(reg) \ + nicvf_reg_read(nic, NIC_QSET_RQ_0_7_STAT_0_1 |\ + (rq_idx << NIC_Q_NUM_SHIFT) | (reg << 3)) + + rq = &nic->qs->rq[rq_idx]; + rq->stats.bytes = GET_RQ_STATS(RQ_SQ_STATS_OCTS); + rq->stats.pkts = GET_RQ_STATS(RQ_SQ_STATS_PKTS); +} + +void nicvf_update_sq_stats(struct nicvf *nic, int sq_idx) +{ + struct snd_queue *sq; + +#define GET_SQ_STATS(reg) \ + nicvf_reg_read(nic, NIC_QSET_SQ_0_7_STAT_0_1 |\ + (sq_idx << NIC_Q_NUM_SHIFT) | (reg << 3)) + + sq = &nic->qs->sq[sq_idx]; + sq->stats.bytes = GET_SQ_STATS(RQ_SQ_STATS_OCTS); + sq->stats.pkts = GET_SQ_STATS(RQ_SQ_STATS_PKTS); +} + +/* Check for errors in the receive cmp.queue entry */ +int nicvf_check_cqe_rx_errs(struct nicvf *nic, + struct cmp_queue *cq, struct cqe_rx_t *cqe_rx) +{ + struct nicvf_hw_stats *stats = &nic->hw_stats; + struct nicvf_drv_stats *drv_stats = &nic->drv_stats; + + if (!cqe_rx->err_level && !cqe_rx->err_opcode) { + drv_stats->rx_frames_ok++; + return 0; + } + + if (netif_msg_rx_err(nic)) + netdev_err(nic->netdev, + "%s: RX error CQE err_level 0x%x err_opcode 0x%x\n", + nic->netdev->name, + cqe_rx->err_level, cqe_rx->err_opcode); + + switch (cqe_rx->err_opcode) { + case CQ_RX_ERROP_RE_PARTIAL: + stats->rx_bgx_truncated_pkts++; + break; + case CQ_RX_ERROP_RE_JABBER: + stats->rx_jabber_errs++; + break; + case CQ_RX_ERROP_RE_FCS: + stats->rx_fcs_errs++; + break; + case CQ_RX_ERROP_RE_RX_CTL: + stats->rx_bgx_errs++; + break; + case CQ_RX_ERROP_PREL2_ERR: + stats->rx_prel2_errs++; + break; + case CQ_RX_ERROP_L2_MAL: + stats->rx_l2_hdr_malformed++; + break; + case CQ_RX_ERROP_L2_OVERSIZE: + stats->rx_oversize++; + break; + case CQ_RX_ERROP_L2_UNDERSIZE: + stats->rx_undersize++; + break; + case CQ_RX_ERROP_L2_LENMISM: + stats->rx_l2_len_mismatch++; + break; + case CQ_RX_ERROP_L2_PCLP: + stats->rx_l2_pclp++; + break; + case CQ_RX_ERROP_IP_NOT: + stats->rx_ip_ver_errs++; + break; + case CQ_RX_ERROP_IP_CSUM_ERR: + stats->rx_ip_csum_errs++; + break; + case CQ_RX_ERROP_IP_MAL: + stats->rx_ip_hdr_malformed++; + break; + case CQ_RX_ERROP_IP_MALD: + stats->rx_ip_payload_malformed++; + break; + case CQ_RX_ERROP_IP_HOP: + stats->rx_ip_ttl_errs++; + break; + case CQ_RX_ERROP_L3_PCLP: + stats->rx_l3_pclp++; + break; + case CQ_RX_ERROP_L4_MAL: + stats->rx_l4_malformed++; + break; + case CQ_RX_ERROP_L4_CHK: + stats->rx_l4_csum_errs++; + break; + case CQ_RX_ERROP_UDP_LEN: + stats->rx_udp_len_errs++; + break; + case CQ_RX_ERROP_L4_PORT: + stats->rx_l4_port_errs++; + break; + case CQ_RX_ERROP_TCP_FLAG: + stats->rx_tcp_flag_errs++; + break; + case CQ_RX_ERROP_TCP_OFFSET: + stats->rx_tcp_offset_errs++; + break; + case CQ_RX_ERROP_L4_PCLP: + stats->rx_l4_pclp++; + break; + case CQ_RX_ERROP_RBDR_TRUNC: + stats->rx_truncated_pkts++; + break; + } + + return 1; +} + +/* Check for errors in the send cmp.queue entry */ +int nicvf_check_cqe_tx_errs(struct nicvf *nic, + struct cmp_queue *cq, struct cqe_send_t *cqe_tx) +{ + struct cmp_queue_stats *stats = &cq->stats; + + switch (cqe_tx->send_status) { + case CQ_TX_ERROP_GOOD: + stats->tx.good++; + return 0; + case CQ_TX_ERROP_DESC_FAULT: + stats->tx.desc_fault++; + break; + case CQ_TX_ERROP_HDR_CONS_ERR: + stats->tx.hdr_cons_err++; + break; + case CQ_TX_ERROP_SUBDC_ERR: + stats->tx.subdesc_err++; + break; + case CQ_TX_ERROP_IMM_SIZE_OFLOW: + stats->tx.imm_size_oflow++; + break; + case CQ_TX_ERROP_DATA_SEQUENCE_ERR: + stats->tx.data_seq_err++; + break; + case CQ_TX_ERROP_MEM_SEQUENCE_ERR: + stats->tx.mem_seq_err++; + break; + case CQ_TX_ERROP_LOCK_VIOL: + stats->tx.lock_viol++; + break; + case CQ_TX_ERROP_DATA_FAULT: + stats->tx.data_fault++; + break; + case CQ_TX_ERROP_TSTMP_CONFLICT: + stats->tx.tstmp_conflict++; + break; + case CQ_TX_ERROP_TSTMP_TIMEOUT: + stats->tx.tstmp_timeout++; + break; + case CQ_TX_ERROP_MEM_FAULT: + stats->tx.mem_fault++; + break; + case CQ_TX_ERROP_CK_OVERLAP: + stats->tx.csum_overlap++; + break; + case CQ_TX_ERROP_CK_OFLOW: + stats->tx.csum_overflow++; + break; + } + + return 1; +} diff --git a/kernel/drivers/net/ethernet/cavium/thunder/nicvf_queues.h b/kernel/drivers/net/ethernet/cavium/thunder/nicvf_queues.h new file mode 100644 index 000000000..033e8306e --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/thunder/nicvf_queues.h @@ -0,0 +1,351 @@ +/* + * Copyright (C) 2015 Cavium, Inc. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of version 2 of the GNU General Public License + * as published by the Free Software Foundation. + */ + +#ifndef NICVF_QUEUES_H +#define NICVF_QUEUES_H + +#include <linux/netdevice.h> +#include "q_struct.h" + +#define MAX_QUEUE_SET 128 +#define MAX_RCV_QUEUES_PER_QS 8 +#define MAX_RCV_BUF_DESC_RINGS_PER_QS 2 +#define MAX_SND_QUEUES_PER_QS 8 +#define MAX_CMP_QUEUES_PER_QS 8 + +/* VF's queue interrupt ranges */ +#define NICVF_INTR_ID_CQ 0 +#define NICVF_INTR_ID_SQ 8 +#define NICVF_INTR_ID_RBDR 16 +#define NICVF_INTR_ID_MISC 18 +#define NICVF_INTR_ID_QS_ERR 19 + +#define for_each_cq_irq(irq) \ + for (irq = NICVF_INTR_ID_CQ; irq < NICVF_INTR_ID_SQ; irq++) +#define for_each_sq_irq(irq) \ + for (irq = NICVF_INTR_ID_SQ; irq < NICVF_INTR_ID_RBDR; irq++) +#define for_each_rbdr_irq(irq) \ + for (irq = NICVF_INTR_ID_RBDR; irq < NICVF_INTR_ID_MISC; irq++) + +#define RBDR_SIZE0 0ULL /* 8K entries */ +#define RBDR_SIZE1 1ULL /* 16K entries */ +#define RBDR_SIZE2 2ULL /* 32K entries */ +#define RBDR_SIZE3 3ULL /* 64K entries */ +#define RBDR_SIZE4 4ULL /* 126K entries */ +#define RBDR_SIZE5 5ULL /* 256K entries */ +#define RBDR_SIZE6 6ULL /* 512K entries */ + +#define SND_QUEUE_SIZE0 0ULL /* 1K entries */ +#define SND_QUEUE_SIZE1 1ULL /* 2K entries */ +#define SND_QUEUE_SIZE2 2ULL /* 4K entries */ +#define SND_QUEUE_SIZE3 3ULL /* 8K entries */ +#define SND_QUEUE_SIZE4 4ULL /* 16K entries */ +#define SND_QUEUE_SIZE5 5ULL /* 32K entries */ +#define SND_QUEUE_SIZE6 6ULL /* 64K entries */ + +#define CMP_QUEUE_SIZE0 0ULL /* 1K entries */ +#define CMP_QUEUE_SIZE1 1ULL /* 2K entries */ +#define CMP_QUEUE_SIZE2 2ULL /* 4K entries */ +#define CMP_QUEUE_SIZE3 3ULL /* 8K entries */ +#define CMP_QUEUE_SIZE4 4ULL /* 16K entries */ +#define CMP_QUEUE_SIZE5 5ULL /* 32K entries */ +#define CMP_QUEUE_SIZE6 6ULL /* 64K entries */ + +/* Default queue count per QS, its lengths and threshold values */ +#define RBDR_CNT 1 +#define RCV_QUEUE_CNT 8 +#define SND_QUEUE_CNT 8 +#define CMP_QUEUE_CNT 8 /* Max of RCV and SND qcount */ + +#define SND_QSIZE SND_QUEUE_SIZE2 +#define SND_QUEUE_LEN (1ULL << (SND_QSIZE + 10)) +#define MAX_SND_QUEUE_LEN (1ULL << (SND_QUEUE_SIZE6 + 10)) +#define SND_QUEUE_THRESH 2ULL +#define MIN_SQ_DESC_PER_PKT_XMIT 2 +/* Since timestamp not enabled, otherwise 2 */ +#define MAX_CQE_PER_PKT_XMIT 1 + +/* Keep CQ and SQ sizes same, if timestamping + * is enabled this equation will change. + */ +#define CMP_QSIZE CMP_QUEUE_SIZE2 +#define CMP_QUEUE_LEN (1ULL << (CMP_QSIZE + 10)) +#define CMP_QUEUE_CQE_THRESH 0 +#define CMP_QUEUE_TIMER_THRESH 80 /* ~2usec */ + +#define RBDR_SIZE RBDR_SIZE0 +#define RCV_BUF_COUNT (1ULL << (RBDR_SIZE + 13)) +#define MAX_RCV_BUF_COUNT (1ULL << (RBDR_SIZE6 + 13)) +#define RBDR_THRESH (RCV_BUF_COUNT / 2) +#define DMA_BUFFER_LEN 2048 /* In multiples of 128bytes */ +#define RCV_FRAG_LEN (SKB_DATA_ALIGN(DMA_BUFFER_LEN + NET_SKB_PAD) + \ + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + \ + (NICVF_RCV_BUF_ALIGN_BYTES * 2)) +#define RCV_DATA_OFFSET NICVF_RCV_BUF_ALIGN_BYTES + +#define MAX_CQES_FOR_TX ((SND_QUEUE_LEN / MIN_SQ_DESC_PER_PKT_XMIT) * \ + MAX_CQE_PER_PKT_XMIT) +/* Calculate number of CQEs to reserve for all SQEs. + * Its 1/256th level of CQ size. + * '+ 1' to account for pipelining + */ +#define RQ_CQ_DROP ((256 / (CMP_QUEUE_LEN / \ + (CMP_QUEUE_LEN - MAX_CQES_FOR_TX))) + 1) + +/* Descriptor size in bytes */ +#define SND_QUEUE_DESC_SIZE 16 +#define CMP_QUEUE_DESC_SIZE 512 + +/* Buffer / descriptor alignments */ +#define NICVF_RCV_BUF_ALIGN 7 +#define NICVF_RCV_BUF_ALIGN_BYTES (1ULL << NICVF_RCV_BUF_ALIGN) +#define NICVF_CQ_BASE_ALIGN_BYTES 512 /* 9 bits */ +#define NICVF_SQ_BASE_ALIGN_BYTES 128 /* 7 bits */ + +#define NICVF_ALIGNED_ADDR(ADDR, ALIGN_BYTES) ALIGN(ADDR, ALIGN_BYTES) +#define NICVF_ADDR_ALIGN_LEN(ADDR, BYTES)\ + (NICVF_ALIGNED_ADDR(ADDR, BYTES) - BYTES) +#define NICVF_RCV_BUF_ALIGN_LEN(X)\ + (NICVF_ALIGNED_ADDR(X, NICVF_RCV_BUF_ALIGN_BYTES) - X) + +/* Queue enable/disable */ +#define NICVF_SQ_EN BIT_ULL(19) + +/* Queue reset */ +#define NICVF_CQ_RESET BIT_ULL(41) +#define NICVF_SQ_RESET BIT_ULL(17) +#define NICVF_RBDR_RESET BIT_ULL(43) + +enum CQ_RX_ERRLVL_E { + CQ_ERRLVL_MAC, + CQ_ERRLVL_L2, + CQ_ERRLVL_L3, + CQ_ERRLVL_L4, +}; + +enum CQ_RX_ERROP_E { + CQ_RX_ERROP_RE_NONE = 0x0, + CQ_RX_ERROP_RE_PARTIAL = 0x1, + CQ_RX_ERROP_RE_JABBER = 0x2, + CQ_RX_ERROP_RE_FCS = 0x7, + CQ_RX_ERROP_RE_TERMINATE = 0x9, + CQ_RX_ERROP_RE_RX_CTL = 0xb, + CQ_RX_ERROP_PREL2_ERR = 0x1f, + CQ_RX_ERROP_L2_FRAGMENT = 0x20, + CQ_RX_ERROP_L2_OVERRUN = 0x21, + CQ_RX_ERROP_L2_PFCS = 0x22, + CQ_RX_ERROP_L2_PUNY = 0x23, + CQ_RX_ERROP_L2_MAL = 0x24, + CQ_RX_ERROP_L2_OVERSIZE = 0x25, + CQ_RX_ERROP_L2_UNDERSIZE = 0x26, + CQ_RX_ERROP_L2_LENMISM = 0x27, + CQ_RX_ERROP_L2_PCLP = 0x28, + CQ_RX_ERROP_IP_NOT = 0x41, + CQ_RX_ERROP_IP_CSUM_ERR = 0x42, + CQ_RX_ERROP_IP_MAL = 0x43, + CQ_RX_ERROP_IP_MALD = 0x44, + CQ_RX_ERROP_IP_HOP = 0x45, + CQ_RX_ERROP_L3_ICRC = 0x46, + CQ_RX_ERROP_L3_PCLP = 0x47, + CQ_RX_ERROP_L4_MAL = 0x61, + CQ_RX_ERROP_L4_CHK = 0x62, + CQ_RX_ERROP_UDP_LEN = 0x63, + CQ_RX_ERROP_L4_PORT = 0x64, + CQ_RX_ERROP_TCP_FLAG = 0x65, + CQ_RX_ERROP_TCP_OFFSET = 0x66, + CQ_RX_ERROP_L4_PCLP = 0x67, + CQ_RX_ERROP_RBDR_TRUNC = 0x70, +}; + +enum CQ_TX_ERROP_E { + CQ_TX_ERROP_GOOD = 0x0, + CQ_TX_ERROP_DESC_FAULT = 0x10, + CQ_TX_ERROP_HDR_CONS_ERR = 0x11, + CQ_TX_ERROP_SUBDC_ERR = 0x12, + CQ_TX_ERROP_IMM_SIZE_OFLOW = 0x80, + CQ_TX_ERROP_DATA_SEQUENCE_ERR = 0x81, + CQ_TX_ERROP_MEM_SEQUENCE_ERR = 0x82, + CQ_TX_ERROP_LOCK_VIOL = 0x83, + CQ_TX_ERROP_DATA_FAULT = 0x84, + CQ_TX_ERROP_TSTMP_CONFLICT = 0x85, + CQ_TX_ERROP_TSTMP_TIMEOUT = 0x86, + CQ_TX_ERROP_MEM_FAULT = 0x87, + CQ_TX_ERROP_CK_OVERLAP = 0x88, + CQ_TX_ERROP_CK_OFLOW = 0x89, + CQ_TX_ERROP_ENUM_LAST = 0x8a, +}; + +struct cmp_queue_stats { + struct tx_stats { + u64 good; + u64 desc_fault; + u64 hdr_cons_err; + u64 subdesc_err; + u64 imm_size_oflow; + u64 data_seq_err; + u64 mem_seq_err; + u64 lock_viol; + u64 data_fault; + u64 tstmp_conflict; + u64 tstmp_timeout; + u64 mem_fault; + u64 csum_overlap; + u64 csum_overflow; + } tx; +} ____cacheline_aligned_in_smp; + +enum RQ_SQ_STATS { + RQ_SQ_STATS_OCTS, + RQ_SQ_STATS_PKTS, +}; + +struct rx_tx_queue_stats { + u64 bytes; + u64 pkts; +} ____cacheline_aligned_in_smp; + +struct q_desc_mem { + dma_addr_t dma; + u64 size; + u16 q_len; + dma_addr_t phys_base; + void *base; + void *unalign_base; +}; + +struct rbdr { + bool enable; + u32 dma_size; + u32 frag_len; + u32 thresh; /* Threshold level for interrupt */ + void *desc; + u32 head; + u32 tail; + struct q_desc_mem dmem; +} ____cacheline_aligned_in_smp; + +struct rcv_queue { + bool enable; + struct rbdr *rbdr_start; + struct rbdr *rbdr_cont; + bool en_tcp_reassembly; + u8 cq_qs; /* CQ's QS to which this RQ is assigned */ + u8 cq_idx; /* CQ index (0 to 7) in the QS */ + u8 cont_rbdr_qs; /* Continue buffer ptrs - QS num */ + u8 cont_qs_rbdr_idx; /* RBDR idx in the cont QS */ + u8 start_rbdr_qs; /* First buffer ptrs - QS num */ + u8 start_qs_rbdr_idx; /* RBDR idx in the above QS */ + u8 caching; + struct rx_tx_queue_stats stats; +} ____cacheline_aligned_in_smp; + +struct cmp_queue { + bool enable; + u16 thresh; + spinlock_t lock; /* lock to serialize processing CQEs */ + void *desc; + struct q_desc_mem dmem; + struct cmp_queue_stats stats; + int irq; +} ____cacheline_aligned_in_smp; + +struct snd_queue { + bool enable; + u8 cq_qs; /* CQ's QS to which this SQ is pointing */ + u8 cq_idx; /* CQ index (0 to 7) in the above QS */ + u16 thresh; + atomic_t free_cnt; + u32 head; + u32 tail; + u64 *skbuff; + void *desc; + +#define TSO_HEADER_SIZE 128 + /* For TSO segment's header */ + char *tso_hdrs; + dma_addr_t tso_hdrs_phys; + + cpumask_t affinity_mask; + struct q_desc_mem dmem; + struct rx_tx_queue_stats stats; +} ____cacheline_aligned_in_smp; + +struct queue_set { + bool enable; + bool be_en; + u8 vnic_id; + u8 rq_cnt; + u8 cq_cnt; + u64 cq_len; + u8 sq_cnt; + u64 sq_len; + u8 rbdr_cnt; + u64 rbdr_len; + struct rcv_queue rq[MAX_RCV_QUEUES_PER_QS]; + struct cmp_queue cq[MAX_CMP_QUEUES_PER_QS]; + struct snd_queue sq[MAX_SND_QUEUES_PER_QS]; + struct rbdr rbdr[MAX_RCV_BUF_DESC_RINGS_PER_QS]; +} ____cacheline_aligned_in_smp; + +#define GET_RBDR_DESC(RING, idx)\ + (&(((struct rbdr_entry_t *)((RING)->desc))[idx])) +#define GET_SQ_DESC(RING, idx)\ + (&(((struct sq_hdr_subdesc *)((RING)->desc))[idx])) +#define GET_CQ_DESC(RING, idx)\ + (&(((union cq_desc_t *)((RING)->desc))[idx])) + +/* CQ status bits */ +#define CQ_WR_FULL BIT(26) +#define CQ_WR_DISABLE BIT(25) +#define CQ_WR_FAULT BIT(24) +#define CQ_CQE_COUNT (0xFFFF << 0) + +#define CQ_ERR_MASK (CQ_WR_FULL | CQ_WR_DISABLE | CQ_WR_FAULT) + +void nicvf_config_vlan_stripping(struct nicvf *nic, + netdev_features_t features); +int nicvf_set_qset_resources(struct nicvf *nic); +int nicvf_config_data_transfer(struct nicvf *nic, bool enable); +void nicvf_qset_config(struct nicvf *nic, bool enable); +void nicvf_cmp_queue_config(struct nicvf *nic, struct queue_set *qs, + int qidx, bool enable); + +void nicvf_sq_enable(struct nicvf *nic, struct snd_queue *sq, int qidx); +void nicvf_sq_disable(struct nicvf *nic, int qidx); +void nicvf_put_sq_desc(struct snd_queue *sq, int desc_cnt); +void nicvf_sq_free_used_descs(struct net_device *netdev, + struct snd_queue *sq, int qidx); +int nicvf_sq_append_skb(struct nicvf *nic, struct sk_buff *skb); + +struct sk_buff *nicvf_get_rcv_skb(struct nicvf *nic, struct cqe_rx_t *cqe_rx); +void nicvf_rbdr_task(unsigned long data); +void nicvf_rbdr_work(struct work_struct *work); + +void nicvf_enable_intr(struct nicvf *nic, int int_type, int q_idx); +void nicvf_disable_intr(struct nicvf *nic, int int_type, int q_idx); +void nicvf_clear_intr(struct nicvf *nic, int int_type, int q_idx); +int nicvf_is_intr_enabled(struct nicvf *nic, int int_type, int q_idx); + +/* Register access APIs */ +void nicvf_reg_write(struct nicvf *nic, u64 offset, u64 val); +u64 nicvf_reg_read(struct nicvf *nic, u64 offset); +void nicvf_qset_reg_write(struct nicvf *nic, u64 offset, u64 val); +u64 nicvf_qset_reg_read(struct nicvf *nic, u64 offset); +void nicvf_queue_reg_write(struct nicvf *nic, u64 offset, + u64 qidx, u64 val); +u64 nicvf_queue_reg_read(struct nicvf *nic, + u64 offset, u64 qidx); + +/* Stats */ +void nicvf_update_rq_stats(struct nicvf *nic, int rq_idx); +void nicvf_update_sq_stats(struct nicvf *nic, int sq_idx); +int nicvf_check_cqe_rx_errs(struct nicvf *nic, + struct cmp_queue *cq, struct cqe_rx_t *cqe_rx); +int nicvf_check_cqe_tx_errs(struct nicvf *nic, + struct cmp_queue *cq, struct cqe_send_t *cqe_tx); +#endif /* NICVF_QUEUES_H */ diff --git a/kernel/drivers/net/ethernet/cavium/thunder/q_struct.h b/kernel/drivers/net/ethernet/cavium/thunder/q_struct.h new file mode 100644 index 000000000..3c1de97b1 --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/thunder/q_struct.h @@ -0,0 +1,701 @@ +/* + * This file contains HW queue descriptor formats, config register + * structures etc + * + * Copyright (C) 2015 Cavium, Inc. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of version 2 of the GNU General Public License + * as published by the Free Software Foundation. + */ + +#ifndef Q_STRUCT_H +#define Q_STRUCT_H + +/* Load transaction types for reading segment bytes specified by + * NIC_SEND_GATHER_S[LD_TYPE]. + */ +enum nic_send_ld_type_e { + NIC_SEND_LD_TYPE_E_LDD = 0x0, + NIC_SEND_LD_TYPE_E_LDT = 0x1, + NIC_SEND_LD_TYPE_E_LDWB = 0x2, + NIC_SEND_LD_TYPE_E_ENUM_LAST = 0x3, +}; + +enum ether_type_algorithm { + ETYPE_ALG_NONE = 0x0, + ETYPE_ALG_SKIP = 0x1, + ETYPE_ALG_ENDPARSE = 0x2, + ETYPE_ALG_VLAN = 0x3, + ETYPE_ALG_VLAN_STRIP = 0x4, +}; + +enum layer3_type { + L3TYPE_NONE = 0x00, + L3TYPE_GRH = 0x01, + L3TYPE_IPV4 = 0x04, + L3TYPE_IPV4_OPTIONS = 0x05, + L3TYPE_IPV6 = 0x06, + L3TYPE_IPV6_OPTIONS = 0x07, + L3TYPE_ET_STOP = 0x0D, + L3TYPE_OTHER = 0x0E, +}; + +enum layer4_type { + L4TYPE_NONE = 0x00, + L4TYPE_IPSEC_ESP = 0x01, + L4TYPE_IPFRAG = 0x02, + L4TYPE_IPCOMP = 0x03, + L4TYPE_TCP = 0x04, + L4TYPE_UDP = 0x05, + L4TYPE_SCTP = 0x06, + L4TYPE_GRE = 0x07, + L4TYPE_ROCE_BTH = 0x08, + L4TYPE_OTHER = 0x0E, +}; + +/* CPI and RSSI configuration */ +enum cpi_algorithm_type { + CPI_ALG_NONE = 0x0, + CPI_ALG_VLAN = 0x1, + CPI_ALG_VLAN16 = 0x2, + CPI_ALG_DIFF = 0x3, +}; + +enum rss_algorithm_type { + RSS_ALG_NONE = 0x00, + RSS_ALG_PORT = 0x01, + RSS_ALG_IP = 0x02, + RSS_ALG_TCP_IP = 0x03, + RSS_ALG_UDP_IP = 0x04, + RSS_ALG_SCTP_IP = 0x05, + RSS_ALG_GRE_IP = 0x06, + RSS_ALG_ROCE = 0x07, +}; + +enum rss_hash_cfg { + RSS_HASH_L2ETC = 0x00, + RSS_HASH_IP = 0x01, + RSS_HASH_TCP = 0x02, + RSS_HASH_TCP_SYN_DIS = 0x03, + RSS_HASH_UDP = 0x04, + RSS_HASH_L4ETC = 0x05, + RSS_HASH_ROCE = 0x06, + RSS_L3_BIDI = 0x07, + RSS_L4_BIDI = 0x08, +}; + +/* Completion queue entry types */ +enum cqe_type { + CQE_TYPE_INVALID = 0x0, + CQE_TYPE_RX = 0x2, + CQE_TYPE_RX_SPLIT = 0x3, + CQE_TYPE_RX_TCP = 0x4, + CQE_TYPE_SEND = 0x8, + CQE_TYPE_SEND_PTP = 0x9, +}; + +enum cqe_rx_tcp_status { + CQE_RX_STATUS_VALID_TCP_CNXT = 0x00, + CQE_RX_STATUS_INVALID_TCP_CNXT = 0x0F, +}; + +enum cqe_send_status { + CQE_SEND_STATUS_GOOD = 0x00, + CQE_SEND_STATUS_DESC_FAULT = 0x01, + CQE_SEND_STATUS_HDR_CONS_ERR = 0x11, + CQE_SEND_STATUS_SUBDESC_ERR = 0x12, + CQE_SEND_STATUS_IMM_SIZE_OFLOW = 0x80, + CQE_SEND_STATUS_CRC_SEQ_ERR = 0x81, + CQE_SEND_STATUS_DATA_SEQ_ERR = 0x82, + CQE_SEND_STATUS_MEM_SEQ_ERR = 0x83, + CQE_SEND_STATUS_LOCK_VIOL = 0x84, + CQE_SEND_STATUS_LOCK_UFLOW = 0x85, + CQE_SEND_STATUS_DATA_FAULT = 0x86, + CQE_SEND_STATUS_TSTMP_CONFLICT = 0x87, + CQE_SEND_STATUS_TSTMP_TIMEOUT = 0x88, + CQE_SEND_STATUS_MEM_FAULT = 0x89, + CQE_SEND_STATUS_CSUM_OVERLAP = 0x8A, + CQE_SEND_STATUS_CSUM_OVERFLOW = 0x8B, +}; + +enum cqe_rx_tcp_end_reason { + CQE_RX_TCP_END_FIN_FLAG_DET = 0, + CQE_RX_TCP_END_INVALID_FLAG = 1, + CQE_RX_TCP_END_TIMEOUT = 2, + CQE_RX_TCP_END_OUT_OF_SEQ = 3, + CQE_RX_TCP_END_PKT_ERR = 4, + CQE_RX_TCP_END_QS_DISABLED = 0x0F, +}; + +/* Packet protocol level error enumeration */ +enum cqe_rx_err_level { + CQE_RX_ERRLVL_RE = 0x0, + CQE_RX_ERRLVL_L2 = 0x1, + CQE_RX_ERRLVL_L3 = 0x2, + CQE_RX_ERRLVL_L4 = 0x3, +}; + +/* Packet protocol level error type enumeration */ +enum cqe_rx_err_opcode { + CQE_RX_ERR_RE_NONE = 0x0, + CQE_RX_ERR_RE_PARTIAL = 0x1, + CQE_RX_ERR_RE_JABBER = 0x2, + CQE_RX_ERR_RE_FCS = 0x7, + CQE_RX_ERR_RE_TERMINATE = 0x9, + CQE_RX_ERR_RE_RX_CTL = 0xb, + CQE_RX_ERR_PREL2_ERR = 0x1f, + CQE_RX_ERR_L2_FRAGMENT = 0x20, + CQE_RX_ERR_L2_OVERRUN = 0x21, + CQE_RX_ERR_L2_PFCS = 0x22, + CQE_RX_ERR_L2_PUNY = 0x23, + CQE_RX_ERR_L2_MAL = 0x24, + CQE_RX_ERR_L2_OVERSIZE = 0x25, + CQE_RX_ERR_L2_UNDERSIZE = 0x26, + CQE_RX_ERR_L2_LENMISM = 0x27, + CQE_RX_ERR_L2_PCLP = 0x28, + CQE_RX_ERR_IP_NOT = 0x41, + CQE_RX_ERR_IP_CHK = 0x42, + CQE_RX_ERR_IP_MAL = 0x43, + CQE_RX_ERR_IP_MALD = 0x44, + CQE_RX_ERR_IP_HOP = 0x45, + CQE_RX_ERR_L3_ICRC = 0x46, + CQE_RX_ERR_L3_PCLP = 0x47, + CQE_RX_ERR_L4_MAL = 0x61, + CQE_RX_ERR_L4_CHK = 0x62, + CQE_RX_ERR_UDP_LEN = 0x63, + CQE_RX_ERR_L4_PORT = 0x64, + CQE_RX_ERR_TCP_FLAG = 0x65, + CQE_RX_ERR_TCP_OFFSET = 0x66, + CQE_RX_ERR_L4_PCLP = 0x67, + CQE_RX_ERR_RBDR_TRUNC = 0x70, +}; + +struct cqe_rx_t { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 cqe_type:4; /* W0 */ + u64 stdn_fault:1; + u64 rsvd0:1; + u64 rq_qs:7; + u64 rq_idx:3; + u64 rsvd1:12; + u64 rss_alg:4; + u64 rsvd2:4; + u64 rb_cnt:4; + u64 vlan_found:1; + u64 vlan_stripped:1; + u64 vlan2_found:1; + u64 vlan2_stripped:1; + u64 l4_type:4; + u64 l3_type:4; + u64 l2_present:1; + u64 err_level:3; + u64 err_opcode:8; + + u64 pkt_len:16; /* W1 */ + u64 l2_ptr:8; + u64 l3_ptr:8; + u64 l4_ptr:8; + u64 cq_pkt_len:8; + u64 align_pad:3; + u64 rsvd3:1; + u64 chan:12; + + u64 rss_tag:32; /* W2 */ + u64 vlan_tci:16; + u64 vlan_ptr:8; + u64 vlan2_ptr:8; + + u64 rb3_sz:16; /* W3 */ + u64 rb2_sz:16; + u64 rb1_sz:16; + u64 rb0_sz:16; + + u64 rb7_sz:16; /* W4 */ + u64 rb6_sz:16; + u64 rb5_sz:16; + u64 rb4_sz:16; + + u64 rb11_sz:16; /* W5 */ + u64 rb10_sz:16; + u64 rb9_sz:16; + u64 rb8_sz:16; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 err_opcode:8; + u64 err_level:3; + u64 l2_present:1; + u64 l3_type:4; + u64 l4_type:4; + u64 vlan2_stripped:1; + u64 vlan2_found:1; + u64 vlan_stripped:1; + u64 vlan_found:1; + u64 rb_cnt:4; + u64 rsvd2:4; + u64 rss_alg:4; + u64 rsvd1:12; + u64 rq_idx:3; + u64 rq_qs:7; + u64 rsvd0:1; + u64 stdn_fault:1; + u64 cqe_type:4; /* W0 */ + u64 chan:12; + u64 rsvd3:1; + u64 align_pad:3; + u64 cq_pkt_len:8; + u64 l4_ptr:8; + u64 l3_ptr:8; + u64 l2_ptr:8; + u64 pkt_len:16; /* W1 */ + u64 vlan2_ptr:8; + u64 vlan_ptr:8; + u64 vlan_tci:16; + u64 rss_tag:32; /* W2 */ + u64 rb0_sz:16; + u64 rb1_sz:16; + u64 rb2_sz:16; + u64 rb3_sz:16; /* W3 */ + u64 rb4_sz:16; + u64 rb5_sz:16; + u64 rb6_sz:16; + u64 rb7_sz:16; /* W4 */ + u64 rb8_sz:16; + u64 rb9_sz:16; + u64 rb10_sz:16; + u64 rb11_sz:16; /* W5 */ +#endif + u64 rb0_ptr:64; + u64 rb1_ptr:64; + u64 rb2_ptr:64; + u64 rb3_ptr:64; + u64 rb4_ptr:64; + u64 rb5_ptr:64; + u64 rb6_ptr:64; + u64 rb7_ptr:64; + u64 rb8_ptr:64; + u64 rb9_ptr:64; + u64 rb10_ptr:64; + u64 rb11_ptr:64; +}; + +struct cqe_rx_tcp_err_t { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 cqe_type:4; /* W0 */ + u64 rsvd0:60; + + u64 rsvd1:4; /* W1 */ + u64 partial_first:1; + u64 rsvd2:27; + u64 rbdr_bytes:8; + u64 rsvd3:24; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 rsvd0:60; + u64 cqe_type:4; + + u64 rsvd3:24; + u64 rbdr_bytes:8; + u64 rsvd2:27; + u64 partial_first:1; + u64 rsvd1:4; +#endif +}; + +struct cqe_rx_tcp_t { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 cqe_type:4; /* W0 */ + u64 rsvd0:52; + u64 cq_tcp_status:8; + + u64 rsvd1:32; /* W1 */ + u64 tcp_cntx_bytes:8; + u64 rsvd2:8; + u64 tcp_err_bytes:16; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 cq_tcp_status:8; + u64 rsvd0:52; + u64 cqe_type:4; /* W0 */ + + u64 tcp_err_bytes:16; + u64 rsvd2:8; + u64 tcp_cntx_bytes:8; + u64 rsvd1:32; /* W1 */ +#endif +}; + +struct cqe_send_t { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 cqe_type:4; /* W0 */ + u64 rsvd0:4; + u64 sqe_ptr:16; + u64 rsvd1:4; + u64 rsvd2:10; + u64 sq_qs:7; + u64 sq_idx:3; + u64 rsvd3:8; + u64 send_status:8; + + u64 ptp_timestamp:64; /* W1 */ +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 send_status:8; + u64 rsvd3:8; + u64 sq_idx:3; + u64 sq_qs:7; + u64 rsvd2:10; + u64 rsvd1:4; + u64 sqe_ptr:16; + u64 rsvd0:4; + u64 cqe_type:4; /* W0 */ + + u64 ptp_timestamp:64; /* W1 */ +#endif +}; + +union cq_desc_t { + u64 u[64]; + struct cqe_send_t snd_hdr; + struct cqe_rx_t rx_hdr; + struct cqe_rx_tcp_t rx_tcp_hdr; + struct cqe_rx_tcp_err_t rx_tcp_err_hdr; +}; + +struct rbdr_entry_t { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 rsvd0:15; + u64 buf_addr:42; + u64 cache_align:7; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 cache_align:7; + u64 buf_addr:42; + u64 rsvd0:15; +#endif +}; + +/* TCP reassembly context */ +struct rbe_tcp_cnxt_t { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 tcp_pkt_cnt:12; + u64 rsvd1:4; + u64 align_hdr_bytes:4; + u64 align_ptr_bytes:4; + u64 ptr_bytes:16; + u64 rsvd2:24; + u64 cqe_type:4; + u64 rsvd0:54; + u64 tcp_end_reason:2; + u64 tcp_status:4; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 tcp_status:4; + u64 tcp_end_reason:2; + u64 rsvd0:54; + u64 cqe_type:4; + u64 rsvd2:24; + u64 ptr_bytes:16; + u64 align_ptr_bytes:4; + u64 align_hdr_bytes:4; + u64 rsvd1:4; + u64 tcp_pkt_cnt:12; +#endif +}; + +/* Always Big endian */ +struct rx_hdr_t { + u64 opaque:32; + u64 rss_flow:8; + u64 skip_length:6; + u64 disable_rss:1; + u64 disable_tcp_reassembly:1; + u64 nodrop:1; + u64 dest_alg:2; + u64 rsvd0:2; + u64 dest_rq:11; +}; + +enum send_l4_csum_type { + SEND_L4_CSUM_DISABLE = 0x00, + SEND_L4_CSUM_UDP = 0x01, + SEND_L4_CSUM_TCP = 0x02, + SEND_L4_CSUM_SCTP = 0x03, +}; + +enum send_crc_alg { + SEND_CRCALG_CRC32 = 0x00, + SEND_CRCALG_CRC32C = 0x01, + SEND_CRCALG_ICRC = 0x02, +}; + +enum send_load_type { + SEND_LD_TYPE_LDD = 0x00, + SEND_LD_TYPE_LDT = 0x01, + SEND_LD_TYPE_LDWB = 0x02, +}; + +enum send_mem_alg_type { + SEND_MEMALG_SET = 0x00, + SEND_MEMALG_ADD = 0x08, + SEND_MEMALG_SUB = 0x09, + SEND_MEMALG_ADDLEN = 0x0A, + SEND_MEMALG_SUBLEN = 0x0B, +}; + +enum send_mem_dsz_type { + SEND_MEMDSZ_B64 = 0x00, + SEND_MEMDSZ_B32 = 0x01, + SEND_MEMDSZ_B8 = 0x03, +}; + +enum sq_subdesc_type { + SQ_DESC_TYPE_INVALID = 0x00, + SQ_DESC_TYPE_HEADER = 0x01, + SQ_DESC_TYPE_CRC = 0x02, + SQ_DESC_TYPE_IMMEDIATE = 0x03, + SQ_DESC_TYPE_GATHER = 0x04, + SQ_DESC_TYPE_MEMORY = 0x05, +}; + +struct sq_crc_subdesc { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 rsvd1:32; + u64 crc_ival:32; + u64 subdesc_type:4; + u64 crc_alg:2; + u64 rsvd0:10; + u64 crc_insert_pos:16; + u64 hdr_start:16; + u64 crc_len:16; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 crc_len:16; + u64 hdr_start:16; + u64 crc_insert_pos:16; + u64 rsvd0:10; + u64 crc_alg:2; + u64 subdesc_type:4; + u64 crc_ival:32; + u64 rsvd1:32; +#endif +}; + +struct sq_gather_subdesc { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 subdesc_type:4; /* W0 */ + u64 ld_type:2; + u64 rsvd0:42; + u64 size:16; + + u64 rsvd1:15; /* W1 */ + u64 addr:49; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 size:16; + u64 rsvd0:42; + u64 ld_type:2; + u64 subdesc_type:4; /* W0 */ + + u64 addr:49; + u64 rsvd1:15; /* W1 */ +#endif +}; + +/* SQ immediate subdescriptor */ +struct sq_imm_subdesc { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 subdesc_type:4; /* W0 */ + u64 rsvd0:46; + u64 len:14; + + u64 data:64; /* W1 */ +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 len:14; + u64 rsvd0:46; + u64 subdesc_type:4; /* W0 */ + + u64 data:64; /* W1 */ +#endif +}; + +struct sq_mem_subdesc { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 subdesc_type:4; /* W0 */ + u64 mem_alg:4; + u64 mem_dsz:2; + u64 wmem:1; + u64 rsvd0:21; + u64 offset:32; + + u64 rsvd1:15; /* W1 */ + u64 addr:49; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 offset:32; + u64 rsvd0:21; + u64 wmem:1; + u64 mem_dsz:2; + u64 mem_alg:4; + u64 subdesc_type:4; /* W0 */ + + u64 addr:49; + u64 rsvd1:15; /* W1 */ +#endif +}; + +struct sq_hdr_subdesc { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 subdesc_type:4; + u64 tso:1; + u64 post_cqe:1; /* Post CQE on no error also */ + u64 dont_send:1; + u64 tstmp:1; + u64 subdesc_cnt:8; + u64 csum_l4:2; + u64 csum_l3:1; + u64 rsvd0:5; + u64 l4_offset:8; + u64 l3_offset:8; + u64 rsvd1:4; + u64 tot_len:20; /* W0 */ + + u64 tso_sdc_cont:8; + u64 tso_sdc_first:8; + u64 tso_l4_offset:8; + u64 tso_flags_last:12; + u64 tso_flags_first:12; + u64 rsvd2:2; + u64 tso_max_paysize:14; /* W1 */ +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 tot_len:20; + u64 rsvd1:4; + u64 l3_offset:8; + u64 l4_offset:8; + u64 rsvd0:5; + u64 csum_l3:1; + u64 csum_l4:2; + u64 subdesc_cnt:8; + u64 tstmp:1; + u64 dont_send:1; + u64 post_cqe:1; /* Post CQE on no error also */ + u64 tso:1; + u64 subdesc_type:4; /* W0 */ + + u64 tso_max_paysize:14; + u64 rsvd2:2; + u64 tso_flags_first:12; + u64 tso_flags_last:12; + u64 tso_l4_offset:8; + u64 tso_sdc_first:8; + u64 tso_sdc_cont:8; /* W1 */ +#endif +}; + +/* Queue config register formats */ +struct rq_cfg { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_2_63:62; + u64 ena:1; + u64 tcp_ena:1; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 tcp_ena:1; + u64 ena:1; + u64 reserved_2_63:62; +#endif +}; + +struct cq_cfg { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_43_63:21; + u64 ena:1; + u64 reset:1; + u64 caching:1; + u64 reserved_35_39:5; + u64 qsize:3; + u64 reserved_25_31:7; + u64 avg_con:9; + u64 reserved_0_15:16; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 reserved_0_15:16; + u64 avg_con:9; + u64 reserved_25_31:7; + u64 qsize:3; + u64 reserved_35_39:5; + u64 caching:1; + u64 reset:1; + u64 ena:1; + u64 reserved_43_63:21; +#endif +}; + +struct sq_cfg { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_20_63:44; + u64 ena:1; + u64 reserved_18_18:1; + u64 reset:1; + u64 ldwb:1; + u64 reserved_11_15:5; + u64 qsize:3; + u64 reserved_3_7:5; + u64 tstmp_bgx_intf:3; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 tstmp_bgx_intf:3; + u64 reserved_3_7:5; + u64 qsize:3; + u64 reserved_11_15:5; + u64 ldwb:1; + u64 reset:1; + u64 reserved_18_18:1; + u64 ena:1; + u64 reserved_20_63:44; +#endif +}; + +struct rbdr_cfg { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_45_63:19; + u64 ena:1; + u64 reset:1; + u64 ldwb:1; + u64 reserved_36_41:6; + u64 qsize:4; + u64 reserved_25_31:7; + u64 avg_con:9; + u64 reserved_12_15:4; + u64 lines:12; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 lines:12; + u64 reserved_12_15:4; + u64 avg_con:9; + u64 reserved_25_31:7; + u64 qsize:4; + u64 reserved_36_41:6; + u64 ldwb:1; + u64 reset:1; + u64 ena: 1; + u64 reserved_45_63:19; +#endif +}; + +struct qs_cfg { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved_32_63:32; + u64 ena:1; + u64 reserved_27_30:4; + u64 sq_ins_ena:1; + u64 sq_ins_pos:6; + u64 lock_ena:1; + u64 lock_viol_cqe_ena:1; + u64 send_tstmp_ena:1; + u64 be:1; + u64 reserved_7_15:9; + u64 vnic:7; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u64 vnic:7; + u64 reserved_7_15:9; + u64 be:1; + u64 send_tstmp_ena:1; + u64 lock_viol_cqe_ena:1; + u64 lock_ena:1; + u64 sq_ins_pos:6; + u64 sq_ins_ena:1; + u64 reserved_27_30:4; + u64 ena:1; + u64 reserved_32_63:32; +#endif +}; + +#endif /* Q_STRUCT_H */ diff --git a/kernel/drivers/net/ethernet/cavium/thunder/thunder_bgx.c b/kernel/drivers/net/ethernet/cavium/thunder/thunder_bgx.c new file mode 100644 index 000000000..9df26c226 --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/thunder/thunder_bgx.c @@ -0,0 +1,1128 @@ +/* + * Copyright (C) 2015 Cavium, Inc. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of version 2 of the GNU General Public License + * as published by the Free Software Foundation. + */ + +#include <linux/acpi.h> +#include <linux/module.h> +#include <linux/interrupt.h> +#include <linux/pci.h> +#include <linux/netdevice.h> +#include <linux/etherdevice.h> +#include <linux/phy.h> +#include <linux/of.h> +#include <linux/of_mdio.h> +#include <linux/of_net.h> + +#include "nic_reg.h" +#include "nic.h" +#include "thunder_bgx.h" + +#define DRV_NAME "thunder-BGX" +#define DRV_VERSION "1.0" + +struct lmac { + struct bgx *bgx; + int dmac; + u8 mac[ETH_ALEN]; + bool link_up; + int lmacid; /* ID within BGX */ + int lmacid_bd; /* ID on board */ + struct net_device netdev; + struct phy_device *phydev; + unsigned int last_duplex; + unsigned int last_link; + unsigned int last_speed; + bool is_sgmii; + struct delayed_work dwork; + struct workqueue_struct *check_link; +}; + +struct bgx { + u8 bgx_id; + u8 qlm_mode; + struct lmac lmac[MAX_LMAC_PER_BGX]; + int lmac_count; + int lmac_type; + int lane_to_sds; + int use_training; + void __iomem *reg_base; + struct pci_dev *pdev; +}; + +static struct bgx *bgx_vnic[MAX_BGX_THUNDER]; +static int lmac_count; /* Total no of LMACs in system */ + +static int bgx_xaui_check_link(struct lmac *lmac); + +/* Supported devices */ +static const struct pci_device_id bgx_id_table[] = { + { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_BGX) }, + { 0, } /* end of table */ +}; + +MODULE_AUTHOR("Cavium Inc"); +MODULE_DESCRIPTION("Cavium Thunder BGX/MAC Driver"); +MODULE_LICENSE("GPL v2"); +MODULE_VERSION(DRV_VERSION); +MODULE_DEVICE_TABLE(pci, bgx_id_table); + +/* The Cavium ThunderX network controller can *only* be found in SoCs + * containing the ThunderX ARM64 CPU implementation. All accesses to the device + * registers on this platform are implicitly strongly ordered with respect + * to memory accesses. So writeq_relaxed() and readq_relaxed() are safe to use + * with no memory barriers in this driver. The readq()/writeq() functions add + * explicit ordering operation which in this case are redundant, and only + * add overhead. + */ + +/* Register read/write APIs */ +static u64 bgx_reg_read(struct bgx *bgx, u8 lmac, u64 offset) +{ + void __iomem *addr = bgx->reg_base + ((u32)lmac << 20) + offset; + + return readq_relaxed(addr); +} + +static void bgx_reg_write(struct bgx *bgx, u8 lmac, u64 offset, u64 val) +{ + void __iomem *addr = bgx->reg_base + ((u32)lmac << 20) + offset; + + writeq_relaxed(val, addr); +} + +static void bgx_reg_modify(struct bgx *bgx, u8 lmac, u64 offset, u64 val) +{ + void __iomem *addr = bgx->reg_base + ((u32)lmac << 20) + offset; + + writeq_relaxed(val | readq_relaxed(addr), addr); +} + +static int bgx_poll_reg(struct bgx *bgx, u8 lmac, u64 reg, u64 mask, bool zero) +{ + int timeout = 100; + u64 reg_val; + + while (timeout) { + reg_val = bgx_reg_read(bgx, lmac, reg); + if (zero && !(reg_val & mask)) + return 0; + if (!zero && (reg_val & mask)) + return 0; + usleep_range(1000, 2000); + timeout--; + } + return 1; +} + +/* Return number of BGX present in HW */ +unsigned bgx_get_map(int node) +{ + int i; + unsigned map = 0; + + for (i = 0; i < MAX_BGX_PER_CN88XX; i++) { + if (bgx_vnic[(node * MAX_BGX_PER_CN88XX) + i]) + map |= (1 << i); + } + + return map; +} +EXPORT_SYMBOL(bgx_get_map); + +/* Return number of LMAC configured for this BGX */ +int bgx_get_lmac_count(int node, int bgx_idx) +{ + struct bgx *bgx; + + bgx = bgx_vnic[(node * MAX_BGX_PER_CN88XX) + bgx_idx]; + if (bgx) + return bgx->lmac_count; + + return 0; +} +EXPORT_SYMBOL(bgx_get_lmac_count); + +/* Returns the current link status of LMAC */ +void bgx_get_lmac_link_state(int node, int bgx_idx, int lmacid, void *status) +{ + struct bgx_link_status *link = (struct bgx_link_status *)status; + struct bgx *bgx; + struct lmac *lmac; + + bgx = bgx_vnic[(node * MAX_BGX_PER_CN88XX) + bgx_idx]; + if (!bgx) + return; + + lmac = &bgx->lmac[lmacid]; + link->link_up = lmac->link_up; + link->duplex = lmac->last_duplex; + link->speed = lmac->last_speed; +} +EXPORT_SYMBOL(bgx_get_lmac_link_state); + +const u8 *bgx_get_lmac_mac(int node, int bgx_idx, int lmacid) +{ + struct bgx *bgx = bgx_vnic[(node * MAX_BGX_PER_CN88XX) + bgx_idx]; + + if (bgx) + return bgx->lmac[lmacid].mac; + + return NULL; +} +EXPORT_SYMBOL(bgx_get_lmac_mac); + +void bgx_set_lmac_mac(int node, int bgx_idx, int lmacid, const u8 *mac) +{ + struct bgx *bgx = bgx_vnic[(node * MAX_BGX_PER_CN88XX) + bgx_idx]; + + if (!bgx) + return; + + ether_addr_copy(bgx->lmac[lmacid].mac, mac); +} +EXPORT_SYMBOL(bgx_set_lmac_mac); + +void bgx_lmac_rx_tx_enable(int node, int bgx_idx, int lmacid, bool enable) +{ + struct bgx *bgx = bgx_vnic[(node * MAX_BGX_PER_CN88XX) + bgx_idx]; + u64 cfg; + + if (!bgx) + return; + + cfg = bgx_reg_read(bgx, lmacid, BGX_CMRX_CFG); + if (enable) + cfg |= CMR_PKT_RX_EN | CMR_PKT_TX_EN; + else + cfg &= ~(CMR_PKT_RX_EN | CMR_PKT_TX_EN); + bgx_reg_write(bgx, lmacid, BGX_CMRX_CFG, cfg); +} +EXPORT_SYMBOL(bgx_lmac_rx_tx_enable); + +static void bgx_sgmii_change_link_state(struct lmac *lmac) +{ + struct bgx *bgx = lmac->bgx; + u64 cmr_cfg; + u64 port_cfg = 0; + u64 misc_ctl = 0; + + cmr_cfg = bgx_reg_read(bgx, lmac->lmacid, BGX_CMRX_CFG); + cmr_cfg &= ~CMR_EN; + bgx_reg_write(bgx, lmac->lmacid, BGX_CMRX_CFG, cmr_cfg); + + port_cfg = bgx_reg_read(bgx, lmac->lmacid, BGX_GMP_GMI_PRTX_CFG); + misc_ctl = bgx_reg_read(bgx, lmac->lmacid, BGX_GMP_PCS_MISCX_CTL); + + if (lmac->link_up) { + misc_ctl &= ~PCS_MISC_CTL_GMX_ENO; + port_cfg &= ~GMI_PORT_CFG_DUPLEX; + port_cfg |= (lmac->last_duplex << 2); + } else { + misc_ctl |= PCS_MISC_CTL_GMX_ENO; + } + + switch (lmac->last_speed) { + case 10: + port_cfg &= ~GMI_PORT_CFG_SPEED; /* speed 0 */ + port_cfg |= GMI_PORT_CFG_SPEED_MSB; /* speed_msb 1 */ + port_cfg &= ~GMI_PORT_CFG_SLOT_TIME; /* slottime 0 */ + misc_ctl &= ~PCS_MISC_CTL_SAMP_PT_MASK; + misc_ctl |= 50; /* samp_pt */ + bgx_reg_write(bgx, lmac->lmacid, BGX_GMP_GMI_TXX_SLOT, 64); + bgx_reg_write(bgx, lmac->lmacid, BGX_GMP_GMI_TXX_BURST, 0); + break; + case 100: + port_cfg &= ~GMI_PORT_CFG_SPEED; /* speed 0 */ + port_cfg &= ~GMI_PORT_CFG_SPEED_MSB; /* speed_msb 0 */ + port_cfg &= ~GMI_PORT_CFG_SLOT_TIME; /* slottime 0 */ + misc_ctl &= ~PCS_MISC_CTL_SAMP_PT_MASK; + misc_ctl |= 5; /* samp_pt */ + bgx_reg_write(bgx, lmac->lmacid, BGX_GMP_GMI_TXX_SLOT, 64); + bgx_reg_write(bgx, lmac->lmacid, BGX_GMP_GMI_TXX_BURST, 0); + break; + case 1000: + port_cfg |= GMI_PORT_CFG_SPEED; /* speed 1 */ + port_cfg &= ~GMI_PORT_CFG_SPEED_MSB; /* speed_msb 0 */ + port_cfg |= GMI_PORT_CFG_SLOT_TIME; /* slottime 1 */ + misc_ctl &= ~PCS_MISC_CTL_SAMP_PT_MASK; + misc_ctl |= 1; /* samp_pt */ + bgx_reg_write(bgx, lmac->lmacid, BGX_GMP_GMI_TXX_SLOT, 512); + if (lmac->last_duplex) + bgx_reg_write(bgx, lmac->lmacid, + BGX_GMP_GMI_TXX_BURST, 0); + else + bgx_reg_write(bgx, lmac->lmacid, + BGX_GMP_GMI_TXX_BURST, 8192); + break; + default: + break; + } + bgx_reg_write(bgx, lmac->lmacid, BGX_GMP_PCS_MISCX_CTL, misc_ctl); + bgx_reg_write(bgx, lmac->lmacid, BGX_GMP_GMI_PRTX_CFG, port_cfg); + + port_cfg = bgx_reg_read(bgx, lmac->lmacid, BGX_GMP_GMI_PRTX_CFG); + + /* renable lmac */ + cmr_cfg |= CMR_EN; + bgx_reg_write(bgx, lmac->lmacid, BGX_CMRX_CFG, cmr_cfg); +} + +static void bgx_lmac_handler(struct net_device *netdev) +{ + struct lmac *lmac = container_of(netdev, struct lmac, netdev); + struct phy_device *phydev = lmac->phydev; + int link_changed = 0; + + if (!lmac) + return; + + if (!phydev->link && lmac->last_link) + link_changed = -1; + + if (phydev->link && + (lmac->last_duplex != phydev->duplex || + lmac->last_link != phydev->link || + lmac->last_speed != phydev->speed)) { + link_changed = 1; + } + + lmac->last_link = phydev->link; + lmac->last_speed = phydev->speed; + lmac->last_duplex = phydev->duplex; + + if (!link_changed) + return; + + if (link_changed > 0) + lmac->link_up = true; + else + lmac->link_up = false; + + if (lmac->is_sgmii) + bgx_sgmii_change_link_state(lmac); + else + bgx_xaui_check_link(lmac); +} + +u64 bgx_get_rx_stats(int node, int bgx_idx, int lmac, int idx) +{ + struct bgx *bgx; + + bgx = bgx_vnic[(node * MAX_BGX_PER_CN88XX) + bgx_idx]; + if (!bgx) + return 0; + + if (idx > 8) + lmac = 0; + return bgx_reg_read(bgx, lmac, BGX_CMRX_RX_STAT0 + (idx * 8)); +} +EXPORT_SYMBOL(bgx_get_rx_stats); + +u64 bgx_get_tx_stats(int node, int bgx_idx, int lmac, int idx) +{ + struct bgx *bgx; + + bgx = bgx_vnic[(node * MAX_BGX_PER_CN88XX) + bgx_idx]; + if (!bgx) + return 0; + + return bgx_reg_read(bgx, lmac, BGX_CMRX_TX_STAT0 + (idx * 8)); +} +EXPORT_SYMBOL(bgx_get_tx_stats); + +static void bgx_flush_dmac_addrs(struct bgx *bgx, int lmac) +{ + u64 offset; + + while (bgx->lmac[lmac].dmac > 0) { + offset = ((bgx->lmac[lmac].dmac - 1) * sizeof(u64)) + + (lmac * MAX_DMAC_PER_LMAC * sizeof(u64)); + bgx_reg_write(bgx, 0, BGX_CMR_RX_DMACX_CAM + offset, 0); + bgx->lmac[lmac].dmac--; + } +} + +/* Configure BGX LMAC in internal loopback mode */ +void bgx_lmac_internal_loopback(int node, int bgx_idx, + int lmac_idx, bool enable) +{ + struct bgx *bgx; + struct lmac *lmac; + u64 cfg; + + bgx = bgx_vnic[(node * MAX_BGX_PER_CN88XX) + bgx_idx]; + if (!bgx) + return; + + lmac = &bgx->lmac[lmac_idx]; + if (lmac->is_sgmii) { + cfg = bgx_reg_read(bgx, lmac_idx, BGX_GMP_PCS_MRX_CTL); + if (enable) + cfg |= PCS_MRX_CTL_LOOPBACK1; + else + cfg &= ~PCS_MRX_CTL_LOOPBACK1; + bgx_reg_write(bgx, lmac_idx, BGX_GMP_PCS_MRX_CTL, cfg); + } else { + cfg = bgx_reg_read(bgx, lmac_idx, BGX_SPUX_CONTROL1); + if (enable) + cfg |= SPU_CTL_LOOPBACK; + else + cfg &= ~SPU_CTL_LOOPBACK; + bgx_reg_write(bgx, lmac_idx, BGX_SPUX_CONTROL1, cfg); + } +} +EXPORT_SYMBOL(bgx_lmac_internal_loopback); + +static int bgx_lmac_sgmii_init(struct bgx *bgx, int lmacid) +{ + u64 cfg; + + bgx_reg_modify(bgx, lmacid, BGX_GMP_GMI_TXX_THRESH, 0x30); + /* max packet size */ + bgx_reg_modify(bgx, lmacid, BGX_GMP_GMI_RXX_JABBER, MAX_FRAME_SIZE); + + /* Disable frame alignment if using preamble */ + cfg = bgx_reg_read(bgx, lmacid, BGX_GMP_GMI_TXX_APPEND); + if (cfg & 1) + bgx_reg_write(bgx, lmacid, BGX_GMP_GMI_TXX_SGMII_CTL, 0); + + /* Enable lmac */ + bgx_reg_modify(bgx, lmacid, BGX_CMRX_CFG, CMR_EN); + + /* PCS reset */ + bgx_reg_modify(bgx, lmacid, BGX_GMP_PCS_MRX_CTL, PCS_MRX_CTL_RESET); + if (bgx_poll_reg(bgx, lmacid, BGX_GMP_PCS_MRX_CTL, + PCS_MRX_CTL_RESET, true)) { + dev_err(&bgx->pdev->dev, "BGX PCS reset not completed\n"); + return -1; + } + + /* power down, reset autoneg, autoneg enable */ + cfg = bgx_reg_read(bgx, lmacid, BGX_GMP_PCS_MRX_CTL); + cfg &= ~PCS_MRX_CTL_PWR_DN; + cfg |= (PCS_MRX_CTL_RST_AN | PCS_MRX_CTL_AN_EN); + bgx_reg_write(bgx, lmacid, BGX_GMP_PCS_MRX_CTL, cfg); + + if (bgx_poll_reg(bgx, lmacid, BGX_GMP_PCS_MRX_STATUS, + PCS_MRX_STATUS_AN_CPT, false)) { + dev_err(&bgx->pdev->dev, "BGX AN_CPT not completed\n"); + return -1; + } + + return 0; +} + +static int bgx_lmac_xaui_init(struct bgx *bgx, int lmacid, int lmac_type) +{ + u64 cfg; + + /* Reset SPU */ + bgx_reg_modify(bgx, lmacid, BGX_SPUX_CONTROL1, SPU_CTL_RESET); + if (bgx_poll_reg(bgx, lmacid, BGX_SPUX_CONTROL1, SPU_CTL_RESET, true)) { + dev_err(&bgx->pdev->dev, "BGX SPU reset not completed\n"); + return -1; + } + + /* Disable LMAC */ + cfg = bgx_reg_read(bgx, lmacid, BGX_CMRX_CFG); + cfg &= ~CMR_EN; + bgx_reg_write(bgx, lmacid, BGX_CMRX_CFG, cfg); + + bgx_reg_modify(bgx, lmacid, BGX_SPUX_CONTROL1, SPU_CTL_LOW_POWER); + /* Set interleaved running disparity for RXAUI */ + if (bgx->lmac_type != BGX_MODE_RXAUI) + bgx_reg_modify(bgx, lmacid, + BGX_SPUX_MISC_CONTROL, SPU_MISC_CTL_RX_DIS); + else + bgx_reg_modify(bgx, lmacid, BGX_SPUX_MISC_CONTROL, + SPU_MISC_CTL_RX_DIS | SPU_MISC_CTL_INTLV_RDISP); + + /* clear all interrupts */ + cfg = bgx_reg_read(bgx, lmacid, BGX_SMUX_RX_INT); + bgx_reg_write(bgx, lmacid, BGX_SMUX_RX_INT, cfg); + cfg = bgx_reg_read(bgx, lmacid, BGX_SMUX_TX_INT); + bgx_reg_write(bgx, lmacid, BGX_SMUX_TX_INT, cfg); + cfg = bgx_reg_read(bgx, lmacid, BGX_SPUX_INT); + bgx_reg_write(bgx, lmacid, BGX_SPUX_INT, cfg); + + if (bgx->use_training) { + bgx_reg_write(bgx, lmacid, BGX_SPUX_BR_PMD_LP_CUP, 0x00); + bgx_reg_write(bgx, lmacid, BGX_SPUX_BR_PMD_LD_CUP, 0x00); + bgx_reg_write(bgx, lmacid, BGX_SPUX_BR_PMD_LD_REP, 0x00); + /* training enable */ + bgx_reg_modify(bgx, lmacid, + BGX_SPUX_BR_PMD_CRTL, SPU_PMD_CRTL_TRAIN_EN); + } + + /* Append FCS to each packet */ + bgx_reg_modify(bgx, lmacid, BGX_SMUX_TX_APPEND, SMU_TX_APPEND_FCS_D); + + /* Disable forward error correction */ + cfg = bgx_reg_read(bgx, lmacid, BGX_SPUX_FEC_CONTROL); + cfg &= ~SPU_FEC_CTL_FEC_EN; + bgx_reg_write(bgx, lmacid, BGX_SPUX_FEC_CONTROL, cfg); + + /* Disable autoneg */ + cfg = bgx_reg_read(bgx, lmacid, BGX_SPUX_AN_CONTROL); + cfg = cfg & ~(SPU_AN_CTL_AN_EN | SPU_AN_CTL_XNP_EN); + bgx_reg_write(bgx, lmacid, BGX_SPUX_AN_CONTROL, cfg); + + cfg = bgx_reg_read(bgx, lmacid, BGX_SPUX_AN_ADV); + if (bgx->lmac_type == BGX_MODE_10G_KR) + cfg |= (1 << 23); + else if (bgx->lmac_type == BGX_MODE_40G_KR) + cfg |= (1 << 24); + else + cfg &= ~((1 << 23) | (1 << 24)); + cfg = cfg & (~((1ULL << 25) | (1ULL << 22) | (1ULL << 12))); + bgx_reg_write(bgx, lmacid, BGX_SPUX_AN_ADV, cfg); + + cfg = bgx_reg_read(bgx, 0, BGX_SPU_DBG_CONTROL); + cfg &= ~SPU_DBG_CTL_AN_ARB_LINK_CHK_EN; + bgx_reg_write(bgx, 0, BGX_SPU_DBG_CONTROL, cfg); + + /* Enable lmac */ + bgx_reg_modify(bgx, lmacid, BGX_CMRX_CFG, CMR_EN); + + cfg = bgx_reg_read(bgx, lmacid, BGX_SPUX_CONTROL1); + cfg &= ~SPU_CTL_LOW_POWER; + bgx_reg_write(bgx, lmacid, BGX_SPUX_CONTROL1, cfg); + + cfg = bgx_reg_read(bgx, lmacid, BGX_SMUX_TX_CTL); + cfg &= ~SMU_TX_CTL_UNI_EN; + cfg |= SMU_TX_CTL_DIC_EN; + bgx_reg_write(bgx, lmacid, BGX_SMUX_TX_CTL, cfg); + + /* take lmac_count into account */ + bgx_reg_modify(bgx, lmacid, BGX_SMUX_TX_THRESH, (0x100 - 1)); + /* max packet size */ + bgx_reg_modify(bgx, lmacid, BGX_SMUX_RX_JABBER, MAX_FRAME_SIZE); + + return 0; +} + +static int bgx_xaui_check_link(struct lmac *lmac) +{ + struct bgx *bgx = lmac->bgx; + int lmacid = lmac->lmacid; + int lmac_type = bgx->lmac_type; + u64 cfg; + + bgx_reg_modify(bgx, lmacid, BGX_SPUX_MISC_CONTROL, SPU_MISC_CTL_RX_DIS); + if (bgx->use_training) { + cfg = bgx_reg_read(bgx, lmacid, BGX_SPUX_INT); + if (!(cfg & (1ull << 13))) { + cfg = (1ull << 13) | (1ull << 14); + bgx_reg_write(bgx, lmacid, BGX_SPUX_INT, cfg); + cfg = bgx_reg_read(bgx, lmacid, BGX_SPUX_BR_PMD_CRTL); + cfg |= (1ull << 0); + bgx_reg_write(bgx, lmacid, BGX_SPUX_BR_PMD_CRTL, cfg); + return -1; + } + } + + /* wait for PCS to come out of reset */ + if (bgx_poll_reg(bgx, lmacid, BGX_SPUX_CONTROL1, SPU_CTL_RESET, true)) { + dev_err(&bgx->pdev->dev, "BGX SPU reset not completed\n"); + return -1; + } + + if ((lmac_type == BGX_MODE_10G_KR) || (lmac_type == BGX_MODE_XFI) || + (lmac_type == BGX_MODE_40G_KR) || (lmac_type == BGX_MODE_XLAUI)) { + if (bgx_poll_reg(bgx, lmacid, BGX_SPUX_BR_STATUS1, + SPU_BR_STATUS_BLK_LOCK, false)) { + dev_err(&bgx->pdev->dev, + "SPU_BR_STATUS_BLK_LOCK not completed\n"); + return -1; + } + } else { + if (bgx_poll_reg(bgx, lmacid, BGX_SPUX_BX_STATUS, + SPU_BX_STATUS_RX_ALIGN, false)) { + dev_err(&bgx->pdev->dev, + "SPU_BX_STATUS_RX_ALIGN not completed\n"); + return -1; + } + } + + /* Clear rcvflt bit (latching high) and read it back */ + bgx_reg_modify(bgx, lmacid, BGX_SPUX_STATUS2, SPU_STATUS2_RCVFLT); + if (bgx_reg_read(bgx, lmacid, BGX_SPUX_STATUS2) & SPU_STATUS2_RCVFLT) { + dev_err(&bgx->pdev->dev, "Receive fault, retry training\n"); + if (bgx->use_training) { + cfg = bgx_reg_read(bgx, lmacid, BGX_SPUX_INT); + if (!(cfg & (1ull << 13))) { + cfg = (1ull << 13) | (1ull << 14); + bgx_reg_write(bgx, lmacid, BGX_SPUX_INT, cfg); + cfg = bgx_reg_read(bgx, lmacid, + BGX_SPUX_BR_PMD_CRTL); + cfg |= (1ull << 0); + bgx_reg_write(bgx, lmacid, + BGX_SPUX_BR_PMD_CRTL, cfg); + return -1; + } + } + return -1; + } + + /* Wait for MAC RX to be ready */ + if (bgx_poll_reg(bgx, lmacid, BGX_SMUX_RX_CTL, + SMU_RX_CTL_STATUS, true)) { + dev_err(&bgx->pdev->dev, "SMU RX link not okay\n"); + return -1; + } + + /* Wait for BGX RX to be idle */ + if (bgx_poll_reg(bgx, lmacid, BGX_SMUX_CTL, SMU_CTL_RX_IDLE, false)) { + dev_err(&bgx->pdev->dev, "SMU RX not idle\n"); + return -1; + } + + /* Wait for BGX TX to be idle */ + if (bgx_poll_reg(bgx, lmacid, BGX_SMUX_CTL, SMU_CTL_TX_IDLE, false)) { + dev_err(&bgx->pdev->dev, "SMU TX not idle\n"); + return -1; + } + + if (bgx_reg_read(bgx, lmacid, BGX_SPUX_STATUS2) & SPU_STATUS2_RCVFLT) { + dev_err(&bgx->pdev->dev, "Receive fault\n"); + return -1; + } + + /* Receive link is latching low. Force it high and verify it */ + bgx_reg_modify(bgx, lmacid, BGX_SPUX_STATUS1, SPU_STATUS1_RCV_LNK); + if (bgx_poll_reg(bgx, lmacid, BGX_SPUX_STATUS1, + SPU_STATUS1_RCV_LNK, false)) { + dev_err(&bgx->pdev->dev, "SPU receive link down\n"); + return -1; + } + + cfg = bgx_reg_read(bgx, lmacid, BGX_SPUX_MISC_CONTROL); + cfg &= ~SPU_MISC_CTL_RX_DIS; + bgx_reg_write(bgx, lmacid, BGX_SPUX_MISC_CONTROL, cfg); + return 0; +} + +static void bgx_poll_for_link(struct work_struct *work) +{ + struct lmac *lmac; + u64 link; + + lmac = container_of(work, struct lmac, dwork.work); + + /* Receive link is latching low. Force it high and verify it */ + bgx_reg_modify(lmac->bgx, lmac->lmacid, + BGX_SPUX_STATUS1, SPU_STATUS1_RCV_LNK); + bgx_poll_reg(lmac->bgx, lmac->lmacid, BGX_SPUX_STATUS1, + SPU_STATUS1_RCV_LNK, false); + + link = bgx_reg_read(lmac->bgx, lmac->lmacid, BGX_SPUX_STATUS1); + if (link & SPU_STATUS1_RCV_LNK) { + lmac->link_up = 1; + if (lmac->bgx->lmac_type == BGX_MODE_XLAUI) + lmac->last_speed = 40000; + else + lmac->last_speed = 10000; + lmac->last_duplex = 1; + } else { + lmac->link_up = 0; + lmac->last_speed = SPEED_UNKNOWN; + lmac->last_duplex = DUPLEX_UNKNOWN; + } + + if (lmac->last_link != lmac->link_up) { + lmac->last_link = lmac->link_up; + if (lmac->link_up) + bgx_xaui_check_link(lmac); + } + + queue_delayed_work(lmac->check_link, &lmac->dwork, HZ * 2); +} + +static int bgx_lmac_enable(struct bgx *bgx, u8 lmacid) +{ + struct lmac *lmac; + u64 cfg; + + lmac = &bgx->lmac[lmacid]; + lmac->bgx = bgx; + + if (bgx->lmac_type == BGX_MODE_SGMII) { + lmac->is_sgmii = 1; + if (bgx_lmac_sgmii_init(bgx, lmacid)) + return -1; + } else { + lmac->is_sgmii = 0; + if (bgx_lmac_xaui_init(bgx, lmacid, bgx->lmac_type)) + return -1; + } + + if (lmac->is_sgmii) { + cfg = bgx_reg_read(bgx, lmacid, BGX_GMP_GMI_TXX_APPEND); + cfg |= ((1ull << 2) | (1ull << 1)); /* FCS and PAD */ + bgx_reg_modify(bgx, lmacid, BGX_GMP_GMI_TXX_APPEND, cfg); + bgx_reg_write(bgx, lmacid, BGX_GMP_GMI_TXX_MIN_PKT, 60 - 1); + } else { + cfg = bgx_reg_read(bgx, lmacid, BGX_SMUX_TX_APPEND); + cfg |= ((1ull << 2) | (1ull << 1)); /* FCS and PAD */ + bgx_reg_modify(bgx, lmacid, BGX_SMUX_TX_APPEND, cfg); + bgx_reg_write(bgx, lmacid, BGX_SMUX_TX_MIN_PKT, 60 + 4); + } + + /* Enable lmac */ + bgx_reg_modify(bgx, lmacid, BGX_CMRX_CFG, CMR_EN); + + /* Restore default cfg, incase low level firmware changed it */ + bgx_reg_write(bgx, lmacid, BGX_CMRX_RX_DMAC_CTL, 0x03); + + if ((bgx->lmac_type != BGX_MODE_XFI) && + (bgx->lmac_type != BGX_MODE_XLAUI) && + (bgx->lmac_type != BGX_MODE_40G_KR) && + (bgx->lmac_type != BGX_MODE_10G_KR)) { + if (!lmac->phydev) + return -ENODEV; + + lmac->phydev->dev_flags = 0; + + if (phy_connect_direct(&lmac->netdev, lmac->phydev, + bgx_lmac_handler, + PHY_INTERFACE_MODE_SGMII)) + return -ENODEV; + + phy_start_aneg(lmac->phydev); + } else { + lmac->check_link = alloc_workqueue("check_link", WQ_UNBOUND | + WQ_MEM_RECLAIM, 1); + if (!lmac->check_link) + return -ENOMEM; + INIT_DELAYED_WORK(&lmac->dwork, bgx_poll_for_link); + queue_delayed_work(lmac->check_link, &lmac->dwork, 0); + } + + return 0; +} + +static void bgx_lmac_disable(struct bgx *bgx, u8 lmacid) +{ + struct lmac *lmac; + u64 cmrx_cfg; + + lmac = &bgx->lmac[lmacid]; + if (lmac->check_link) { + /* Destroy work queue */ + cancel_delayed_work_sync(&lmac->dwork); + destroy_workqueue(lmac->check_link); + } + + cmrx_cfg = bgx_reg_read(bgx, lmacid, BGX_CMRX_CFG); + cmrx_cfg &= ~(1 << 15); + bgx_reg_write(bgx, lmacid, BGX_CMRX_CFG, cmrx_cfg); + bgx_flush_dmac_addrs(bgx, lmacid); + + if ((bgx->lmac_type != BGX_MODE_XFI) && + (bgx->lmac_type != BGX_MODE_XLAUI) && + (bgx->lmac_type != BGX_MODE_40G_KR) && + (bgx->lmac_type != BGX_MODE_10G_KR) && lmac->phydev) + phy_disconnect(lmac->phydev); + + lmac->phydev = NULL; +} + +static void bgx_set_num_ports(struct bgx *bgx) +{ + u64 lmac_count; + + switch (bgx->qlm_mode) { + case QLM_MODE_SGMII: + bgx->lmac_count = 4; + bgx->lmac_type = BGX_MODE_SGMII; + bgx->lane_to_sds = 0; + break; + case QLM_MODE_XAUI_1X4: + bgx->lmac_count = 1; + bgx->lmac_type = BGX_MODE_XAUI; + bgx->lane_to_sds = 0xE4; + break; + case QLM_MODE_RXAUI_2X2: + bgx->lmac_count = 2; + bgx->lmac_type = BGX_MODE_RXAUI; + bgx->lane_to_sds = 0xE4; + break; + case QLM_MODE_XFI_4X1: + bgx->lmac_count = 4; + bgx->lmac_type = BGX_MODE_XFI; + bgx->lane_to_sds = 0; + break; + case QLM_MODE_XLAUI_1X4: + bgx->lmac_count = 1; + bgx->lmac_type = BGX_MODE_XLAUI; + bgx->lane_to_sds = 0xE4; + break; + case QLM_MODE_10G_KR_4X1: + bgx->lmac_count = 4; + bgx->lmac_type = BGX_MODE_10G_KR; + bgx->lane_to_sds = 0; + bgx->use_training = 1; + break; + case QLM_MODE_40G_KR4_1X4: + bgx->lmac_count = 1; + bgx->lmac_type = BGX_MODE_40G_KR; + bgx->lane_to_sds = 0xE4; + bgx->use_training = 1; + break; + default: + bgx->lmac_count = 0; + break; + } + + /* Check if low level firmware has programmed LMAC count + * based on board type, if yes consider that otherwise + * the default static values + */ + lmac_count = bgx_reg_read(bgx, 0, BGX_CMR_RX_LMACS) & 0x7; + if (lmac_count != 4) + bgx->lmac_count = lmac_count; +} + +static void bgx_init_hw(struct bgx *bgx) +{ + int i; + + bgx_set_num_ports(bgx); + + bgx_reg_modify(bgx, 0, BGX_CMR_GLOBAL_CFG, CMR_GLOBAL_CFG_FCS_STRIP); + if (bgx_reg_read(bgx, 0, BGX_CMR_BIST_STATUS)) + dev_err(&bgx->pdev->dev, "BGX%d BIST failed\n", bgx->bgx_id); + + /* Set lmac type and lane2serdes mapping */ + for (i = 0; i < bgx->lmac_count; i++) { + if (bgx->lmac_type == BGX_MODE_RXAUI) { + if (i) + bgx->lane_to_sds = 0x0e; + else + bgx->lane_to_sds = 0x04; + bgx_reg_write(bgx, i, BGX_CMRX_CFG, + (bgx->lmac_type << 8) | bgx->lane_to_sds); + continue; + } + bgx_reg_write(bgx, i, BGX_CMRX_CFG, + (bgx->lmac_type << 8) | (bgx->lane_to_sds + i)); + bgx->lmac[i].lmacid_bd = lmac_count; + lmac_count++; + } + + bgx_reg_write(bgx, 0, BGX_CMR_TX_LMACS, bgx->lmac_count); + bgx_reg_write(bgx, 0, BGX_CMR_RX_LMACS, bgx->lmac_count); + + /* Set the backpressure AND mask */ + for (i = 0; i < bgx->lmac_count; i++) + bgx_reg_modify(bgx, 0, BGX_CMR_CHAN_MSK_AND, + ((1ULL << MAX_BGX_CHANS_PER_LMAC) - 1) << + (i * MAX_BGX_CHANS_PER_LMAC)); + + /* Disable all MAC filtering */ + for (i = 0; i < RX_DMAC_COUNT; i++) + bgx_reg_write(bgx, 0, BGX_CMR_RX_DMACX_CAM + (i * 8), 0x00); + + /* Disable MAC steering (NCSI traffic) */ + for (i = 0; i < RX_TRAFFIC_STEER_RULE_COUNT; i++) + bgx_reg_write(bgx, 0, BGX_CMR_RX_STREERING + (i * 8), 0x00); +} + +static void bgx_get_qlm_mode(struct bgx *bgx) +{ + struct device *dev = &bgx->pdev->dev; + int lmac_type; + int train_en; + + /* Read LMAC0 type to figure out QLM mode + * This is configured by low level firmware + */ + lmac_type = bgx_reg_read(bgx, 0, BGX_CMRX_CFG); + lmac_type = (lmac_type >> 8) & 0x07; + + train_en = bgx_reg_read(bgx, 0, BGX_SPUX_BR_PMD_CRTL) & + SPU_PMD_CRTL_TRAIN_EN; + + switch (lmac_type) { + case BGX_MODE_SGMII: + bgx->qlm_mode = QLM_MODE_SGMII; + dev_info(dev, "BGX%d QLM mode: SGMII\n", bgx->bgx_id); + break; + case BGX_MODE_XAUI: + bgx->qlm_mode = QLM_MODE_XAUI_1X4; + dev_info(dev, "BGX%d QLM mode: XAUI\n", bgx->bgx_id); + break; + case BGX_MODE_RXAUI: + bgx->qlm_mode = QLM_MODE_RXAUI_2X2; + dev_info(dev, "BGX%d QLM mode: RXAUI\n", bgx->bgx_id); + break; + case BGX_MODE_XFI: + if (!train_en) { + bgx->qlm_mode = QLM_MODE_XFI_4X1; + dev_info(dev, "BGX%d QLM mode: XFI\n", bgx->bgx_id); + } else { + bgx->qlm_mode = QLM_MODE_10G_KR_4X1; + dev_info(dev, "BGX%d QLM mode: 10G_KR\n", bgx->bgx_id); + } + break; + case BGX_MODE_XLAUI: + if (!train_en) { + bgx->qlm_mode = QLM_MODE_XLAUI_1X4; + dev_info(dev, "BGX%d QLM mode: XLAUI\n", bgx->bgx_id); + } else { + bgx->qlm_mode = QLM_MODE_40G_KR4_1X4; + dev_info(dev, "BGX%d QLM mode: 40G_KR4\n", bgx->bgx_id); + } + break; + default: + bgx->qlm_mode = QLM_MODE_SGMII; + dev_info(dev, "BGX%d QLM default mode: SGMII\n", bgx->bgx_id); + } +} + +#ifdef CONFIG_ACPI + +static int acpi_get_mac_address(struct acpi_device *adev, u8 *dst) +{ + u8 mac[ETH_ALEN]; + int ret; + + ret = fwnode_property_read_u8_array(acpi_fwnode_handle(adev), + "mac-address", mac, ETH_ALEN); + if (ret) + goto out; + + if (!is_valid_ether_addr(mac)) { + ret = -EINVAL; + goto out; + } + + memcpy(dst, mac, ETH_ALEN); +out: + return ret; +} + +/* Currently only sets the MAC address. */ +static acpi_status bgx_acpi_register_phy(acpi_handle handle, + u32 lvl, void *context, void **rv) +{ + struct bgx *bgx = context; + struct acpi_device *adev; + + if (acpi_bus_get_device(handle, &adev)) + goto out; + + acpi_get_mac_address(adev, bgx->lmac[bgx->lmac_count].mac); + + SET_NETDEV_DEV(&bgx->lmac[bgx->lmac_count].netdev, &bgx->pdev->dev); + + bgx->lmac[bgx->lmac_count].lmacid = bgx->lmac_count; +out: + bgx->lmac_count++; + return AE_OK; +} + +static acpi_status bgx_acpi_match_id(acpi_handle handle, u32 lvl, + void *context, void **ret_val) +{ + struct acpi_buffer string = { ACPI_ALLOCATE_BUFFER, NULL }; + struct bgx *bgx = context; + char bgx_sel[5]; + + snprintf(bgx_sel, 5, "BGX%d", bgx->bgx_id); + if (ACPI_FAILURE(acpi_get_name(handle, ACPI_SINGLE_NAME, &string))) { + pr_warn("Invalid link device\n"); + return AE_OK; + } + + if (strncmp(string.pointer, bgx_sel, 4)) + return AE_OK; + + acpi_walk_namespace(ACPI_TYPE_DEVICE, handle, 1, + bgx_acpi_register_phy, NULL, bgx, NULL); + + kfree(string.pointer); + return AE_CTRL_TERMINATE; +} + +static int bgx_init_acpi_phy(struct bgx *bgx) +{ + acpi_get_devices(NULL, bgx_acpi_match_id, bgx, (void **)NULL); + return 0; +} + +#else + +static int bgx_init_acpi_phy(struct bgx *bgx) +{ + return -ENODEV; +} + +#endif /* CONFIG_ACPI */ + +#if IS_ENABLED(CONFIG_OF_MDIO) + +static int bgx_init_of_phy(struct bgx *bgx) +{ + struct device_node *np; + struct device_node *np_child; + u8 lmac = 0; + char bgx_sel[5]; + const char *mac; + + /* Get BGX node from DT */ + snprintf(bgx_sel, 5, "bgx%d", bgx->bgx_id); + np = of_find_node_by_name(NULL, bgx_sel); + if (!np) + return -ENODEV; + + for_each_child_of_node(np, np_child) { + struct device_node *phy_np = of_parse_phandle(np_child, + "phy-handle", 0); + if (!phy_np) + continue; + bgx->lmac[lmac].phydev = of_phy_find_device(phy_np); + + mac = of_get_mac_address(np_child); + if (mac) + ether_addr_copy(bgx->lmac[lmac].mac, mac); + + SET_NETDEV_DEV(&bgx->lmac[lmac].netdev, &bgx->pdev->dev); + bgx->lmac[lmac].lmacid = lmac; + lmac++; + if (lmac == MAX_LMAC_PER_BGX) { + of_node_put(np_child); + break; + } + } + return 0; +} + +#else + +static int bgx_init_of_phy(struct bgx *bgx) +{ + return -ENODEV; +} + +#endif /* CONFIG_OF_MDIO */ + +static int bgx_init_phy(struct bgx *bgx) +{ + if (!acpi_disabled) + return bgx_init_acpi_phy(bgx); + + return bgx_init_of_phy(bgx); +} + +static int bgx_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + int err; + struct device *dev = &pdev->dev; + struct bgx *bgx = NULL; + u8 lmac; + + /* Load octeon mdio driver */ + octeon_mdiobus_force_mod_depencency(); + + bgx = devm_kzalloc(dev, sizeof(*bgx), GFP_KERNEL); + if (!bgx) + return -ENOMEM; + bgx->pdev = pdev; + + pci_set_drvdata(pdev, bgx); + + err = pci_enable_device(pdev); + if (err) { + dev_err(dev, "Failed to enable PCI device\n"); + pci_set_drvdata(pdev, NULL); + return err; + } + + err = pci_request_regions(pdev, DRV_NAME); + if (err) { + dev_err(dev, "PCI request regions failed 0x%x\n", err); + goto err_disable_device; + } + + /* MAP configuration registers */ + bgx->reg_base = pcim_iomap(pdev, PCI_CFG_REG_BAR_NUM, 0); + if (!bgx->reg_base) { + dev_err(dev, "BGX: Cannot map CSR memory space, aborting\n"); + err = -ENOMEM; + goto err_release_regions; + } + bgx->bgx_id = (pci_resource_start(pdev, PCI_CFG_REG_BAR_NUM) >> 24) & 1; + bgx->bgx_id += nic_get_node_id(pdev) * MAX_BGX_PER_CN88XX; + + bgx_vnic[bgx->bgx_id] = bgx; + bgx_get_qlm_mode(bgx); + + err = bgx_init_phy(bgx); + if (err) + goto err_enable; + + bgx_init_hw(bgx); + + /* Enable all LMACs */ + for (lmac = 0; lmac < bgx->lmac_count; lmac++) { + err = bgx_lmac_enable(bgx, lmac); + if (err) { + dev_err(dev, "BGX%d failed to enable lmac%d\n", + bgx->bgx_id, lmac); + goto err_enable; + } + } + + return 0; + +err_enable: + bgx_vnic[bgx->bgx_id] = NULL; +err_release_regions: + pci_release_regions(pdev); +err_disable_device: + pci_disable_device(pdev); + pci_set_drvdata(pdev, NULL); + return err; +} + +static void bgx_remove(struct pci_dev *pdev) +{ + struct bgx *bgx = pci_get_drvdata(pdev); + u8 lmac; + + /* Disable all LMACs */ + for (lmac = 0; lmac < bgx->lmac_count; lmac++) + bgx_lmac_disable(bgx, lmac); + + bgx_vnic[bgx->bgx_id] = NULL; + pci_release_regions(pdev); + pci_disable_device(pdev); + pci_set_drvdata(pdev, NULL); +} + +static struct pci_driver bgx_driver = { + .name = DRV_NAME, + .id_table = bgx_id_table, + .probe = bgx_probe, + .remove = bgx_remove, +}; + +static int __init bgx_init_module(void) +{ + pr_info("%s, ver %s\n", DRV_NAME, DRV_VERSION); + + return pci_register_driver(&bgx_driver); +} + +static void __exit bgx_cleanup_module(void) +{ + pci_unregister_driver(&bgx_driver); +} + +module_init(bgx_init_module); +module_exit(bgx_cleanup_module); diff --git a/kernel/drivers/net/ethernet/cavium/thunder/thunder_bgx.h b/kernel/drivers/net/ethernet/cavium/thunder/thunder_bgx.h new file mode 100644 index 000000000..149e17936 --- /dev/null +++ b/kernel/drivers/net/ethernet/cavium/thunder/thunder_bgx.h @@ -0,0 +1,226 @@ +/* + * Copyright (C) 2015 Cavium, Inc. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of version 2 of the GNU General Public License + * as published by the Free Software Foundation. + */ + +#ifndef THUNDER_BGX_H +#define THUNDER_BGX_H + +#define MAX_BGX_THUNDER 8 /* Max 4 nodes, 2 per node */ +#define MAX_BGX_PER_CN88XX 2 +#define MAX_LMAC_PER_BGX 4 +#define MAX_BGX_CHANS_PER_LMAC 16 +#define MAX_DMAC_PER_LMAC 8 +#define MAX_FRAME_SIZE 9216 + +#define MAX_DMAC_PER_LMAC_TNS_BYPASS_MODE 2 + +#define MAX_LMAC (MAX_BGX_PER_CN88XX * MAX_LMAC_PER_BGX) + +/* Registers */ +#define BGX_CMRX_CFG 0x00 +#define CMR_PKT_TX_EN BIT_ULL(13) +#define CMR_PKT_RX_EN BIT_ULL(14) +#define CMR_EN BIT_ULL(15) +#define BGX_CMR_GLOBAL_CFG 0x08 +#define CMR_GLOBAL_CFG_FCS_STRIP BIT_ULL(6) +#define BGX_CMRX_RX_ID_MAP 0x60 +#define BGX_CMRX_RX_STAT0 0x70 +#define BGX_CMRX_RX_STAT1 0x78 +#define BGX_CMRX_RX_STAT2 0x80 +#define BGX_CMRX_RX_STAT3 0x88 +#define BGX_CMRX_RX_STAT4 0x90 +#define BGX_CMRX_RX_STAT5 0x98 +#define BGX_CMRX_RX_STAT6 0xA0 +#define BGX_CMRX_RX_STAT7 0xA8 +#define BGX_CMRX_RX_STAT8 0xB0 +#define BGX_CMRX_RX_STAT9 0xB8 +#define BGX_CMRX_RX_STAT10 0xC0 +#define BGX_CMRX_RX_BP_DROP 0xC8 +#define BGX_CMRX_RX_DMAC_CTL 0x0E8 +#define BGX_CMR_RX_DMACX_CAM 0x200 +#define RX_DMACX_CAM_EN BIT_ULL(48) +#define RX_DMACX_CAM_LMACID(x) (x << 49) +#define RX_DMAC_COUNT 32 +#define BGX_CMR_RX_STREERING 0x300 +#define RX_TRAFFIC_STEER_RULE_COUNT 8 +#define BGX_CMR_CHAN_MSK_AND 0x450 +#define BGX_CMR_BIST_STATUS 0x460 +#define BGX_CMR_RX_LMACS 0x468 +#define BGX_CMRX_TX_STAT0 0x600 +#define BGX_CMRX_TX_STAT1 0x608 +#define BGX_CMRX_TX_STAT2 0x610 +#define BGX_CMRX_TX_STAT3 0x618 +#define BGX_CMRX_TX_STAT4 0x620 +#define BGX_CMRX_TX_STAT5 0x628 +#define BGX_CMRX_TX_STAT6 0x630 +#define BGX_CMRX_TX_STAT7 0x638 +#define BGX_CMRX_TX_STAT8 0x640 +#define BGX_CMRX_TX_STAT9 0x648 +#define BGX_CMRX_TX_STAT10 0x650 +#define BGX_CMRX_TX_STAT11 0x658 +#define BGX_CMRX_TX_STAT12 0x660 +#define BGX_CMRX_TX_STAT13 0x668 +#define BGX_CMRX_TX_STAT14 0x670 +#define BGX_CMRX_TX_STAT15 0x678 +#define BGX_CMRX_TX_STAT16 0x680 +#define BGX_CMRX_TX_STAT17 0x688 +#define BGX_CMR_TX_LMACS 0x1000 + +#define BGX_SPUX_CONTROL1 0x10000 +#define SPU_CTL_LOW_POWER BIT_ULL(11) +#define SPU_CTL_LOOPBACK BIT_ULL(14) +#define SPU_CTL_RESET BIT_ULL(15) +#define BGX_SPUX_STATUS1 0x10008 +#define SPU_STATUS1_RCV_LNK BIT_ULL(2) +#define BGX_SPUX_STATUS2 0x10020 +#define SPU_STATUS2_RCVFLT BIT_ULL(10) +#define BGX_SPUX_BX_STATUS 0x10028 +#define SPU_BX_STATUS_RX_ALIGN BIT_ULL(12) +#define BGX_SPUX_BR_STATUS1 0x10030 +#define SPU_BR_STATUS_BLK_LOCK BIT_ULL(0) +#define SPU_BR_STATUS_RCV_LNK BIT_ULL(12) +#define BGX_SPUX_BR_PMD_CRTL 0x10068 +#define SPU_PMD_CRTL_TRAIN_EN BIT_ULL(1) +#define BGX_SPUX_BR_PMD_LP_CUP 0x10078 +#define BGX_SPUX_BR_PMD_LD_CUP 0x10088 +#define BGX_SPUX_BR_PMD_LD_REP 0x10090 +#define BGX_SPUX_FEC_CONTROL 0x100A0 +#define SPU_FEC_CTL_FEC_EN BIT_ULL(0) +#define SPU_FEC_CTL_ERR_EN BIT_ULL(1) +#define BGX_SPUX_AN_CONTROL 0x100C8 +#define SPU_AN_CTL_AN_EN BIT_ULL(12) +#define SPU_AN_CTL_XNP_EN BIT_ULL(13) +#define BGX_SPUX_AN_ADV 0x100D8 +#define BGX_SPUX_MISC_CONTROL 0x10218 +#define SPU_MISC_CTL_INTLV_RDISP BIT_ULL(10) +#define SPU_MISC_CTL_RX_DIS BIT_ULL(12) +#define BGX_SPUX_INT 0x10220 /* +(0..3) << 20 */ +#define BGX_SPUX_INT_W1S 0x10228 +#define BGX_SPUX_INT_ENA_W1C 0x10230 +#define BGX_SPUX_INT_ENA_W1S 0x10238 +#define BGX_SPU_DBG_CONTROL 0x10300 +#define SPU_DBG_CTL_AN_ARB_LINK_CHK_EN BIT_ULL(18) +#define SPU_DBG_CTL_AN_NONCE_MCT_DIS BIT_ULL(29) + +#define BGX_SMUX_RX_INT 0x20000 +#define BGX_SMUX_RX_JABBER 0x20030 +#define BGX_SMUX_RX_CTL 0x20048 +#define SMU_RX_CTL_STATUS (3ull << 0) +#define BGX_SMUX_TX_APPEND 0x20100 +#define SMU_TX_APPEND_FCS_D BIT_ULL(2) +#define BGX_SMUX_TX_MIN_PKT 0x20118 +#define BGX_SMUX_TX_INT 0x20140 +#define BGX_SMUX_TX_CTL 0x20178 +#define SMU_TX_CTL_DIC_EN BIT_ULL(0) +#define SMU_TX_CTL_UNI_EN BIT_ULL(1) +#define SMU_TX_CTL_LNK_STATUS (3ull << 4) +#define BGX_SMUX_TX_THRESH 0x20180 +#define BGX_SMUX_CTL 0x20200 +#define SMU_CTL_RX_IDLE BIT_ULL(0) +#define SMU_CTL_TX_IDLE BIT_ULL(1) + +#define BGX_GMP_PCS_MRX_CTL 0x30000 +#define PCS_MRX_CTL_RST_AN BIT_ULL(9) +#define PCS_MRX_CTL_PWR_DN BIT_ULL(11) +#define PCS_MRX_CTL_AN_EN BIT_ULL(12) +#define PCS_MRX_CTL_LOOPBACK1 BIT_ULL(14) +#define PCS_MRX_CTL_RESET BIT_ULL(15) +#define BGX_GMP_PCS_MRX_STATUS 0x30008 +#define PCS_MRX_STATUS_AN_CPT BIT_ULL(5) +#define BGX_GMP_PCS_ANX_AN_RESULTS 0x30020 +#define BGX_GMP_PCS_SGM_AN_ADV 0x30068 +#define BGX_GMP_PCS_MISCX_CTL 0x30078 +#define PCS_MISC_CTL_GMX_ENO BIT_ULL(11) +#define PCS_MISC_CTL_SAMP_PT_MASK 0x7Full +#define BGX_GMP_GMI_PRTX_CFG 0x38020 +#define GMI_PORT_CFG_SPEED BIT_ULL(1) +#define GMI_PORT_CFG_DUPLEX BIT_ULL(2) +#define GMI_PORT_CFG_SLOT_TIME BIT_ULL(3) +#define GMI_PORT_CFG_SPEED_MSB BIT_ULL(8) +#define BGX_GMP_GMI_RXX_JABBER 0x38038 +#define BGX_GMP_GMI_TXX_THRESH 0x38210 +#define BGX_GMP_GMI_TXX_APPEND 0x38218 +#define BGX_GMP_GMI_TXX_SLOT 0x38220 +#define BGX_GMP_GMI_TXX_BURST 0x38228 +#define BGX_GMP_GMI_TXX_MIN_PKT 0x38240 +#define BGX_GMP_GMI_TXX_SGMII_CTL 0x38300 + +#define BGX_MSIX_VEC_0_29_ADDR 0x400000 /* +(0..29) << 4 */ +#define BGX_MSIX_VEC_0_29_CTL 0x400008 +#define BGX_MSIX_PBA_0 0x4F0000 + +/* MSI-X interrupts */ +#define BGX_MSIX_VECTORS 30 +#define BGX_LMAC_VEC_OFFSET 7 +#define BGX_MSIX_VEC_SHIFT 4 + +#define CMRX_INT 0 +#define SPUX_INT 1 +#define SMUX_RX_INT 2 +#define SMUX_TX_INT 3 +#define GMPX_PCS_INT 4 +#define GMPX_GMI_RX_INT 5 +#define GMPX_GMI_TX_INT 6 +#define CMR_MEM_INT 28 +#define SPU_MEM_INT 29 + +#define LMAC_INTR_LINK_UP BIT(0) +#define LMAC_INTR_LINK_DOWN BIT(1) + +/* RX_DMAC_CTL configuration*/ +enum MCAST_MODE { + MCAST_MODE_REJECT, + MCAST_MODE_ACCEPT, + MCAST_MODE_CAM_FILTER, + RSVD +}; + +#define BCAST_ACCEPT 1 +#define CAM_ACCEPT 1 + +void octeon_mdiobus_force_mod_depencency(void); +void bgx_lmac_rx_tx_enable(int node, int bgx_idx, int lmacid, bool enable); +void bgx_add_dmac_addr(u64 dmac, int node, int bgx_idx, int lmac); +unsigned bgx_get_map(int node); +int bgx_get_lmac_count(int node, int bgx); +const u8 *bgx_get_lmac_mac(int node, int bgx_idx, int lmacid); +void bgx_set_lmac_mac(int node, int bgx_idx, int lmacid, const u8 *mac); +void bgx_get_lmac_link_state(int node, int bgx_idx, int lmacid, void *status); +void bgx_lmac_internal_loopback(int node, int bgx_idx, + int lmac_idx, bool enable); +u64 bgx_get_rx_stats(int node, int bgx_idx, int lmac, int idx); +u64 bgx_get_tx_stats(int node, int bgx_idx, int lmac, int idx); +#define BGX_RX_STATS_COUNT 11 +#define BGX_TX_STATS_COUNT 18 + +struct bgx_stats { + u64 rx_stats[BGX_RX_STATS_COUNT]; + u64 tx_stats[BGX_TX_STATS_COUNT]; +}; + +enum LMAC_TYPE { + BGX_MODE_SGMII = 0, /* 1 lane, 1.250 Gbaud */ + BGX_MODE_XAUI = 1, /* 4 lanes, 3.125 Gbaud */ + BGX_MODE_DXAUI = 1, /* 4 lanes, 6.250 Gbaud */ + BGX_MODE_RXAUI = 2, /* 2 lanes, 6.250 Gbaud */ + BGX_MODE_XFI = 3, /* 1 lane, 10.3125 Gbaud */ + BGX_MODE_XLAUI = 4, /* 4 lanes, 10.3125 Gbaud */ + BGX_MODE_10G_KR = 3,/* 1 lane, 10.3125 Gbaud */ + BGX_MODE_40G_KR = 4,/* 4 lanes, 10.3125 Gbaud */ +}; + +enum qlm_mode { + QLM_MODE_SGMII, /* SGMII, each lane independent */ + QLM_MODE_XAUI_1X4, /* 1 XAUI or DXAUI, 4 lanes */ + QLM_MODE_RXAUI_2X2, /* 2 RXAUI, 2 lanes each */ + QLM_MODE_XFI_4X1, /* 4 XFI, 1 lane each */ + QLM_MODE_XLAUI_1X4, /* 1 XLAUI, 4 lanes each */ + QLM_MODE_10G_KR_4X1, /* 4 10GBASE-KR, 1 lane each */ + QLM_MODE_40G_KR4_1X4, /* 1 40GBASE-KR4, 4 lanes each */ +}; + +#endif /* THUNDER_BGX_H */ |