diff options
Diffstat (limited to 'qemu/roms/ipxe/src/drivers/usb/xhci.c')
| -rw-r--r-- | qemu/roms/ipxe/src/drivers/usb/xhci.c | 3321 |
1 files changed, 3321 insertions, 0 deletions
diff --git a/qemu/roms/ipxe/src/drivers/usb/xhci.c b/qemu/roms/ipxe/src/drivers/usb/xhci.c new file mode 100644 index 000000000..49e67316b --- /dev/null +++ b/qemu/roms/ipxe/src/drivers/usb/xhci.c @@ -0,0 +1,3321 @@ +/* + * Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + * + * You can also choose to distribute this program under the terms of + * the Unmodified Binary Distribution Licence (as given in the file + * COPYING.UBDL), provided that you have satisfied its requirements. + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +#include <stdlib.h> +#include <stdio.h> +#include <unistd.h> +#include <string.h> +#include <strings.h> +#include <errno.h> +#include <byteswap.h> +#include <ipxe/malloc.h> +#include <ipxe/umalloc.h> +#include <ipxe/pci.h> +#include <ipxe/usb.h> +#include <ipxe/init.h> +#include <ipxe/profile.h> +#include "xhci.h" + +/** @file + * + * USB eXtensible Host Controller Interface (xHCI) driver + * + */ + +/** Message transfer profiler */ +static struct profiler xhci_message_profiler __profiler = + { .name = "xhci.message" }; + +/** Stream transfer profiler */ +static struct profiler xhci_stream_profiler __profiler = + { .name = "xhci.stream" }; + +/** Event ring profiler */ +static struct profiler xhci_event_profiler __profiler = + { .name = "xhci.event" }; + +/** Transfer event profiler */ +static struct profiler xhci_transfer_profiler __profiler = + { .name = "xhci.transfer" }; + +/* Disambiguate the various error causes */ +#define EIO_DATA \ + __einfo_error ( EINFO_EIO_DATA ) +#define EINFO_EIO_DATA \ + __einfo_uniqify ( EINFO_EIO, ( 2 - 0 ), \ + "Data buffer error" ) +#define EIO_BABBLE \ + __einfo_error ( EINFO_EIO_BABBLE ) +#define EINFO_EIO_BABBLE \ + __einfo_uniqify ( EINFO_EIO, ( 3 - 0 ), \ + "Babble detected" ) +#define EIO_USB \ + __einfo_error ( EINFO_EIO_USB ) +#define EINFO_EIO_USB \ + __einfo_uniqify ( EINFO_EIO, ( 4 - 0 ), \ + "USB transaction error" ) +#define EIO_TRB \ + __einfo_error ( EINFO_EIO_TRB ) +#define EINFO_EIO_TRB \ + __einfo_uniqify ( EINFO_EIO, ( 5 - 0 ), \ + "TRB error" ) +#define EIO_STALL \ + __einfo_error ( EINFO_EIO_STALL ) +#define EINFO_EIO_STALL \ + __einfo_uniqify ( EINFO_EIO, ( 6 - 0 ), \ + "Stall error" ) +#define EIO_RESOURCE \ + __einfo_error ( EINFO_EIO_RESOURCE ) +#define EINFO_EIO_RESOURCE \ + __einfo_uniqify ( EINFO_EIO, ( 7 - 0 ), \ + "Resource error" ) +#define EIO_BANDWIDTH \ + __einfo_error ( EINFO_EIO_BANDWIDTH ) +#define EINFO_EIO_BANDWIDTH \ + __einfo_uniqify ( EINFO_EIO, ( 8 - 0 ), \ + "Bandwidth error" ) +#define EIO_NO_SLOTS \ + __einfo_error ( EINFO_EIO_NO_SLOTS ) +#define EINFO_EIO_NO_SLOTS \ + __einfo_uniqify ( EINFO_EIO, ( 9 - 0 ), \ + "No slots available" ) +#define EIO_STREAM_TYPE \ + __einfo_error ( EINFO_EIO_STREAM_TYPE ) +#define EINFO_EIO_STREAM_TYPE \ + __einfo_uniqify ( EINFO_EIO, ( 10 - 0 ), \ + "Invalid stream type" ) +#define EIO_SLOT \ + __einfo_error ( EINFO_EIO_SLOT ) +#define EINFO_EIO_SLOT \ + __einfo_uniqify ( EINFO_EIO, ( 11 - 0 ), \ + "Slot not enabled" ) +#define EIO_ENDPOINT \ + __einfo_error ( EINFO_EIO_ENDPOINT ) +#define EINFO_EIO_ENDPOINT \ + __einfo_uniqify ( EINFO_EIO, ( 12 - 0 ), \ + "Endpoint not enabled" ) +#define EIO_SHORT \ + __einfo_error ( EINFO_EIO_SHORT ) +#define EINFO_EIO_SHORT \ + __einfo_uniqify ( EINFO_EIO, ( 13 - 0 ), \ + "Short packet" ) +#define EIO_UNDERRUN \ + __einfo_error ( EINFO_EIO_UNDERRUN ) +#define EINFO_EIO_UNDERRUN \ + __einfo_uniqify ( EINFO_EIO, ( 14 - 0 ), \ + "Ring underrun" ) +#define EIO_OVERRUN \ + __einfo_error ( EINFO_EIO_OVERRUN ) +#define EINFO_EIO_OVERRUN \ + __einfo_uniqify ( EINFO_EIO, ( 15 - 0 ), \ + "Ring overrun" ) +#define EIO_VF_RING_FULL \ + __einfo_error ( EINFO_EIO_VF_RING_FULL ) +#define EINFO_EIO_VF_RING_FULL \ + __einfo_uniqify ( EINFO_EIO, ( 16 - 0 ), \ + "Virtual function event ring full" ) +#define EIO_PARAMETER \ + __einfo_error ( EINFO_EIO_PARAMETER ) +#define EINFO_EIO_PARAMETER \ + __einfo_uniqify ( EINFO_EIO, ( 17 - 0 ), \ + "Parameter error" ) +#define EIO_BANDWIDTH_OVERRUN \ + __einfo_error ( EINFO_EIO_BANDWIDTH_OVERRUN ) +#define EINFO_EIO_BANDWIDTH_OVERRUN \ + __einfo_uniqify ( EINFO_EIO, ( 18 - 0 ), \ + "Bandwidth overrun" ) +#define EIO_CONTEXT \ + __einfo_error ( EINFO_EIO_CONTEXT ) +#define EINFO_EIO_CONTEXT \ + __einfo_uniqify ( EINFO_EIO, ( 19 - 0 ), \ + "Context state error" ) +#define EIO_NO_PING \ + __einfo_error ( EINFO_EIO_NO_PING ) +#define EINFO_EIO_NO_PING \ + __einfo_uniqify ( EINFO_EIO, ( 20 - 0 ), \ + "No ping response" ) +#define EIO_RING_FULL \ + __einfo_error ( EINFO_EIO_RING_FULL ) +#define EINFO_EIO_RING_FULL \ + __einfo_uniqify ( EINFO_EIO, ( 21 - 0 ), \ + "Event ring full" ) +#define EIO_INCOMPATIBLE \ + __einfo_error ( EINFO_EIO_INCOMPATIBLE ) +#define EINFO_EIO_INCOMPATIBLE \ + __einfo_uniqify ( EINFO_EIO, ( 22 - 0 ), \ + "Incompatible device" ) +#define EIO_MISSED \ + __einfo_error ( EINFO_EIO_MISSED ) +#define EINFO_EIO_MISSED \ + __einfo_uniqify ( EINFO_EIO, ( 23 - 0 ), \ + "Missed service error" ) +#define EIO_CMD_STOPPED \ + __einfo_error ( EINFO_EIO_CMD_STOPPED ) +#define EINFO_EIO_CMD_STOPPED \ + __einfo_uniqify ( EINFO_EIO, ( 24 - 0 ), \ + "Command ring stopped" ) +#define EIO_CMD_ABORTED \ + __einfo_error ( EINFO_EIO_CMD_ABORTED ) +#define EINFO_EIO_CMD_ABORTED \ + __einfo_uniqify ( EINFO_EIO, ( 25 - 0 ), \ + "Command aborted" ) +#define EIO_STOP \ + __einfo_error ( EINFO_EIO_STOP ) +#define EINFO_EIO_STOP \ + __einfo_uniqify ( EINFO_EIO, ( 26 - 0 ), \ + "Stopped" ) +#define EIO_STOP_LEN \ + __einfo_error ( EINFO_EIO_STOP_LEN ) +#define EINFO_EIO_STOP_LEN \ + __einfo_uniqify ( EINFO_EIO, ( 27 - 0 ), \ + "Stopped - length invalid" ) +#define EIO_STOP_SHORT \ + __einfo_error ( EINFO_EIO_STOP_SHORT ) +#define EINFO_EIO_STOP_SHORT \ + __einfo_uniqify ( EINFO_EIO, ( 28 - 0 ), \ + "Stopped - short packet" ) +#define EIO_LATENCY \ + __einfo_error ( EINFO_EIO_LATENCY ) +#define EINFO_EIO_LATENCY \ + __einfo_uniqify ( EINFO_EIO, ( 29 - 0 ), \ + "Maximum exit latency too large" ) +#define EIO_ISOCH \ + __einfo_error ( EINFO_EIO_ISOCH ) +#define EINFO_EIO_ISOCH \ + __einfo_uniqify ( EINFO_EIO, ( 31 - 0 ), \ + "Isochronous buffer overrun" ) +#define EPROTO_LOST \ + __einfo_error ( EINFO_EPROTO_LOST ) +#define EINFO_EPROTO_LOST \ + __einfo_uniqify ( EINFO_EPROTO, ( 32 - 32 ), \ + "Event lost" ) +#define EPROTO_UNDEFINED \ + __einfo_error ( EINFO_EPROTO_UNDEFINED ) +#define EINFO_EPROTO_UNDEFINED \ + __einfo_uniqify ( EINFO_EPROTO, ( 33 - 32 ), \ + "Undefined error" ) +#define EPROTO_STREAM_ID \ + __einfo_error ( EINFO_EPROTO_STREAM_ID ) +#define EINFO_EPROTO_STREAM_ID \ + __einfo_uniqify ( EINFO_EPROTO, ( 34 - 32 ), \ + "Invalid stream ID" ) +#define EPROTO_SECONDARY \ + __einfo_error ( EINFO_EPROTO_SECONDARY ) +#define EINFO_EPROTO_SECONDARY \ + __einfo_uniqify ( EINFO_EPROTO, ( 35 - 32 ), \ + "Secondary bandwidth error" ) +#define EPROTO_SPLIT \ + __einfo_error ( EINFO_EPROTO_SPLIT ) +#define EINFO_EPROTO_SPLIT \ + __einfo_uniqify ( EINFO_EPROTO, ( 36 - 32 ), \ + "Split transaction error" ) +#define ECODE(code) \ + ( ( (code) < 32 ) ? \ + EUNIQ ( EINFO_EIO, ( (code) & 31 ), EIO_DATA, EIO_BABBLE, \ + EIO_USB, EIO_TRB, EIO_STALL, EIO_RESOURCE, \ + EIO_BANDWIDTH, EIO_NO_SLOTS, EIO_STREAM_TYPE, \ + EIO_SLOT, EIO_ENDPOINT, EIO_SHORT, EIO_UNDERRUN, \ + EIO_OVERRUN, EIO_VF_RING_FULL, EIO_PARAMETER, \ + EIO_BANDWIDTH_OVERRUN, EIO_CONTEXT, EIO_NO_PING, \ + EIO_RING_FULL, EIO_INCOMPATIBLE, EIO_MISSED, \ + EIO_CMD_STOPPED, EIO_CMD_ABORTED, EIO_STOP, \ + EIO_STOP_LEN, EIO_STOP_SHORT, EIO_LATENCY, \ + EIO_ISOCH ) : \ + ( (code) < 64 ) ? \ + EUNIQ ( EINFO_EPROTO, ( (code) & 31 ), EPROTO_LOST, \ + EPROTO_UNDEFINED, EPROTO_STREAM_ID, \ + EPROTO_SECONDARY, EPROTO_SPLIT ) : \ + EFAULT ) + +/****************************************************************************** + * + * Register access + * + ****************************************************************************** + */ + +/** + * Initialise device + * + * @v xhci xHCI device + * @v regs MMIO registers + */ +static void xhci_init ( struct xhci_device *xhci, void *regs ) { + uint32_t hcsparams1; + uint32_t hcsparams2; + uint32_t hccparams1; + uint32_t pagesize; + size_t caplength; + size_t rtsoff; + size_t dboff; + + /* Locate capability, operational, runtime, and doorbell registers */ + xhci->cap = regs; + caplength = readb ( xhci->cap + XHCI_CAP_CAPLENGTH ); + rtsoff = readl ( xhci->cap + XHCI_CAP_RTSOFF ); + dboff = readl ( xhci->cap + XHCI_CAP_DBOFF ); + xhci->op = ( xhci->cap + caplength ); + xhci->run = ( xhci->cap + rtsoff ); + xhci->db = ( xhci->cap + dboff ); + DBGC2 ( xhci, "XHCI %s cap %08lx op %08lx run %08lx db %08lx\n", + xhci->name, virt_to_phys ( xhci->cap ), + virt_to_phys ( xhci->op ), virt_to_phys ( xhci->run ), + virt_to_phys ( xhci->db ) ); + + /* Read structural parameters 1 */ + hcsparams1 = readl ( xhci->cap + XHCI_CAP_HCSPARAMS1 ); + xhci->slots = XHCI_HCSPARAMS1_SLOTS ( hcsparams1 ); + xhci->intrs = XHCI_HCSPARAMS1_INTRS ( hcsparams1 ); + xhci->ports = XHCI_HCSPARAMS1_PORTS ( hcsparams1 ); + DBGC ( xhci, "XHCI %s has %d slots %d intrs %d ports\n", + xhci->name, xhci->slots, xhci->intrs, xhci->ports ); + + /* Read structural parameters 2 */ + hcsparams2 = readl ( xhci->cap + XHCI_CAP_HCSPARAMS2 ); + xhci->scratchpads = XHCI_HCSPARAMS2_SCRATCHPADS ( hcsparams2 ); + DBGC2 ( xhci, "XHCI %s needs %d scratchpads\n", + xhci->name, xhci->scratchpads ); + + /* Read capability parameters 1 */ + hccparams1 = readl ( xhci->cap + XHCI_CAP_HCCPARAMS1 ); + xhci->addr64 = XHCI_HCCPARAMS1_ADDR64 ( hccparams1 ); + xhci->csz_shift = XHCI_HCCPARAMS1_CSZ_SHIFT ( hccparams1 ); + xhci->xecp = XHCI_HCCPARAMS1_XECP ( hccparams1 ); + + /* Read page size */ + pagesize = readl ( xhci->op + XHCI_OP_PAGESIZE ); + xhci->pagesize = XHCI_PAGESIZE ( pagesize ); + assert ( xhci->pagesize != 0 ); + assert ( ( ( xhci->pagesize ) & ( xhci->pagesize - 1 ) ) == 0 ); + DBGC2 ( xhci, "XHCI %s page size %zd bytes\n", + xhci->name, xhci->pagesize ); +} + +/** + * Find extended capability + * + * @v xhci xHCI device + * @v id Capability ID + * @v offset Offset to previous extended capability instance, or zero + * @ret offset Offset to extended capability, or zero if not found + */ +static unsigned int xhci_extended_capability ( struct xhci_device *xhci, + unsigned int id, + unsigned int offset ) { + uint32_t xecp; + unsigned int next; + + /* Locate the extended capability */ + while ( 1 ) { + + /* Locate first or next capability as applicable */ + if ( offset ) { + xecp = readl ( xhci->cap + offset ); + next = XHCI_XECP_NEXT ( xecp ); + } else { + next = xhci->xecp; + } + if ( ! next ) + return 0; + offset += next; + + /* Check if this is the requested capability */ + xecp = readl ( xhci->cap + offset ); + if ( XHCI_XECP_ID ( xecp ) == id ) + return offset; + } +} + +/** + * Write potentially 64-bit register + * + * @v xhci xHCI device + * @v value Value + * @v reg Register address + * @ret rc Return status code + */ +static inline __attribute__ (( always_inline )) int +xhci_writeq ( struct xhci_device *xhci, physaddr_t value, void *reg ) { + + /* If this is a 32-bit build, then this can never fail + * (allowing the compiler to optimise out the error path). + */ + if ( sizeof ( value ) <= sizeof ( uint32_t ) ) { + writel ( value, reg ); + writel ( 0, ( reg + sizeof ( uint32_t ) ) ); + return 0; + } + + /* If the device does not support 64-bit addresses and this + * address is outside the 32-bit address space, then fail. + */ + if ( ( value & ~0xffffffffULL ) && ! xhci->addr64 ) { + DBGC ( xhci, "XHCI %s cannot access address %lx\n", + xhci->name, value ); + return -ENOTSUP; + } + + /* If this is a 64-bit build, then writeq() is available */ + writeq ( value, reg ); + return 0; +} + +/** + * Calculate buffer alignment + * + * @v len Length + * @ret align Buffer alignment + * + * Determine alignment required for a buffer which must be aligned to + * at least XHCI_MIN_ALIGN and which must not cross a page boundary. + */ +static inline size_t xhci_align ( size_t len ) { + size_t align; + + /* Align to own length (rounded up to a power of two) */ + align = ( 1 << fls ( len - 1 ) ); + + /* Round up to XHCI_MIN_ALIGN if needed */ + if ( align < XHCI_MIN_ALIGN ) + align = XHCI_MIN_ALIGN; + + return align; +} + +/** + * Calculate device context offset + * + * @v xhci xHCI device + * @v ctx Context index + */ +static inline size_t xhci_device_context_offset ( struct xhci_device *xhci, + unsigned int ctx ) { + + return ( XHCI_DCI ( ctx ) << xhci->csz_shift ); +} + +/** + * Calculate input context offset + * + * @v xhci xHCI device + * @v ctx Context index + */ +static inline size_t xhci_input_context_offset ( struct xhci_device *xhci, + unsigned int ctx ) { + + return ( XHCI_ICI ( ctx ) << xhci->csz_shift ); +} + +/****************************************************************************** + * + * Diagnostics + * + ****************************************************************************** + */ + +/** + * Dump host controller registers + * + * @v xhci xHCI device + */ +static inline void xhci_dump ( struct xhci_device *xhci ) { + uint32_t usbcmd; + uint32_t usbsts; + uint32_t pagesize; + uint32_t dnctrl; + uint32_t config; + + /* Do nothing unless debugging is enabled */ + if ( ! DBG_LOG ) + return; + + /* Dump USBCMD */ + usbcmd = readl ( xhci->op + XHCI_OP_USBCMD ); + DBGC ( xhci, "XHCI %s USBCMD %08x%s%s\n", xhci->name, usbcmd, + ( ( usbcmd & XHCI_USBCMD_RUN ) ? " run" : "" ), + ( ( usbcmd & XHCI_USBCMD_HCRST ) ? " hcrst" : "" ) ); + + /* Dump USBSTS */ + usbsts = readl ( xhci->op + XHCI_OP_USBSTS ); + DBGC ( xhci, "XHCI %s USBSTS %08x%s\n", xhci->name, usbsts, + ( ( usbsts & XHCI_USBSTS_HCH ) ? " hch" : "" ) ); + + /* Dump PAGESIZE */ + pagesize = readl ( xhci->op + XHCI_OP_PAGESIZE ); + DBGC ( xhci, "XHCI %s PAGESIZE %08x\n", xhci->name, pagesize ); + + /* Dump DNCTRL */ + dnctrl = readl ( xhci->op + XHCI_OP_DNCTRL ); + DBGC ( xhci, "XHCI %s DNCTRL %08x\n", xhci->name, dnctrl ); + + /* Dump CONFIG */ + config = readl ( xhci->op + XHCI_OP_CONFIG ); + DBGC ( xhci, "XHCI %s CONFIG %08x\n", xhci->name, config ); +} + +/** + * Dump port registers + * + * @v xhci xHCI device + * @v port Port number + */ +static inline void xhci_dump_port ( struct xhci_device *xhci, + unsigned int port ) { + uint32_t portsc; + uint32_t portpmsc; + uint32_t portli; + uint32_t porthlpmc; + + /* Do nothing unless debugging is enabled */ + if ( ! DBG_LOG ) + return; + + /* Dump PORTSC */ + portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port ) ); + DBGC ( xhci, "XHCI %s-%d PORTSC %08x%s%s%s%s psiv=%d\n", + xhci->name, port, portsc, + ( ( portsc & XHCI_PORTSC_CCS ) ? " ccs" : "" ), + ( ( portsc & XHCI_PORTSC_PED ) ? " ped" : "" ), + ( ( portsc & XHCI_PORTSC_PR ) ? " pr" : "" ), + ( ( portsc & XHCI_PORTSC_PP ) ? " pp" : "" ), + XHCI_PORTSC_PSIV ( portsc ) ); + + /* Dump PORTPMSC */ + portpmsc = readl ( xhci->op + XHCI_OP_PORTPMSC ( port ) ); + DBGC ( xhci, "XHCI %s-%d PORTPMSC %08x\n", xhci->name, port, portpmsc ); + + /* Dump PORTLI */ + portli = readl ( xhci->op + XHCI_OP_PORTLI ( port ) ); + DBGC ( xhci, "XHCI %s-%d PORTLI %08x\n", xhci->name, port, portli ); + + /* Dump PORTHLPMC */ + porthlpmc = readl ( xhci->op + XHCI_OP_PORTHLPMC ( port ) ); + DBGC ( xhci, "XHCI %s-%d PORTHLPMC %08x\n", + xhci->name, port, porthlpmc ); +} + +/****************************************************************************** + * + * USB legacy support + * + ****************************************************************************** + */ + +/** Prevent the release of ownership back to BIOS */ +static int xhci_legacy_prevent_release; + +/** + * Initialise USB legacy support + * + * @v xhci xHCI device + */ +static void xhci_legacy_init ( struct xhci_device *xhci ) { + unsigned int legacy; + uint8_t bios; + + /* Locate USB legacy support capability (if present) */ + legacy = xhci_extended_capability ( xhci, XHCI_XECP_ID_LEGACY, 0 ); + if ( ! legacy ) { + /* Not an error; capability may not be present */ + DBGC ( xhci, "XHCI %s has no USB legacy support capability\n", + xhci->name ); + return; + } + + /* Check if legacy USB support is enabled */ + bios = readb ( xhci->cap + legacy + XHCI_USBLEGSUP_BIOS ); + if ( ! ( bios & XHCI_USBLEGSUP_BIOS_OWNED ) ) { + /* Not an error; already owned by OS */ + DBGC ( xhci, "XHCI %s USB legacy support already disabled\n", + xhci->name ); + return; + } + + /* Record presence of USB legacy support capability */ + xhci->legacy = legacy; +} + +/** + * Claim ownership from BIOS + * + * @v xhci xHCI device + */ +static void xhci_legacy_claim ( struct xhci_device *xhci ) { + uint32_t ctlsts; + uint8_t bios; + unsigned int i; + + /* Do nothing unless legacy support capability is present */ + if ( ! xhci->legacy ) + return; + + /* Claim ownership */ + writeb ( XHCI_USBLEGSUP_OS_OWNED, + xhci->cap + xhci->legacy + XHCI_USBLEGSUP_OS ); + + /* Wait for BIOS to release ownership */ + for ( i = 0 ; i < XHCI_USBLEGSUP_MAX_WAIT_MS ; i++ ) { + + /* Check if BIOS has released ownership */ + bios = readb ( xhci->cap + xhci->legacy + XHCI_USBLEGSUP_BIOS ); + if ( ! ( bios & XHCI_USBLEGSUP_BIOS_OWNED ) ) { + DBGC ( xhci, "XHCI %s claimed ownership from BIOS\n", + xhci->name ); + ctlsts = readl ( xhci->cap + xhci->legacy + + XHCI_USBLEGSUP_CTLSTS ); + if ( ctlsts ) { + DBGC ( xhci, "XHCI %s warning: BIOS retained " + "SMIs: %08x\n", xhci->name, ctlsts ); + } + return; + } + + /* Delay */ + mdelay ( 1 ); + } + + /* BIOS did not release ownership. Claim it forcibly by + * disabling all SMIs. + */ + DBGC ( xhci, "XHCI %s could not claim ownership from BIOS: forcibly " + "disabling SMIs\n", xhci->name ); + writel ( 0, xhci->cap + xhci->legacy + XHCI_USBLEGSUP_CTLSTS ); +} + +/** + * Release ownership back to BIOS + * + * @v xhci xHCI device + */ +static void xhci_legacy_release ( struct xhci_device *xhci ) { + + /* Do nothing unless legacy support capability is present */ + if ( ! xhci->legacy ) + return; + + /* Do nothing if releasing ownership is prevented */ + if ( xhci_legacy_prevent_release ) { + DBGC ( xhci, "XHCI %s not releasing ownership to BIOS\n", + xhci->name ); + return; + } + + /* Release ownership */ + writeb ( 0, xhci->cap + xhci->legacy + XHCI_USBLEGSUP_OS ); + DBGC ( xhci, "XHCI %s released ownership to BIOS\n", xhci->name ); +} + +/****************************************************************************** + * + * Supported protocols + * + ****************************************************************************** + */ + +/** + * Transcribe port speed (for debugging) + * + * @v psi Protocol speed ID + * @ret speed Transcribed speed + */ +static inline const char * xhci_speed_name ( uint32_t psi ) { + static const char *exponents[4] = { "", "k", "M", "G" }; + static char buf[ 10 /* "xxxxxXbps" + NUL */ ]; + unsigned int mantissa; + unsigned int exponent; + + /* Extract mantissa and exponent */ + mantissa = XHCI_SUPPORTED_PSI_MANTISSA ( psi ); + exponent = XHCI_SUPPORTED_PSI_EXPONENT ( psi ); + + /* Transcribe speed */ + snprintf ( buf, sizeof ( buf ), "%d%sbps", + mantissa, exponents[exponent] ); + return buf; +} + +/** + * Find supported protocol extended capability for a port + * + * @v xhci xHCI device + * @v port Port number + * @ret supported Offset to extended capability, or zero if not found + */ +static unsigned int xhci_supported_protocol ( struct xhci_device *xhci, + unsigned int port ) { + unsigned int supported = 0; + unsigned int offset; + unsigned int count; + uint32_t ports; + + /* Iterate over all supported protocol structures */ + while ( ( supported = xhci_extended_capability ( xhci, + XHCI_XECP_ID_SUPPORTED, + supported ) ) ) { + + /* Determine port range */ + ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS ); + offset = XHCI_SUPPORTED_PORTS_OFFSET ( ports ); + count = XHCI_SUPPORTED_PORTS_COUNT ( ports ); + + /* Check if port lies within this range */ + if ( ( port - offset ) < count ) + return supported; + } + + DBGC ( xhci, "XHCI %s-%d has no supported protocol\n", + xhci->name, port ); + return 0; +} + +/** + * Find port protocol + * + * @v xhci xHCI device + * @v port Port number + * @ret protocol USB protocol, or zero if not found + */ +static unsigned int xhci_port_protocol ( struct xhci_device *xhci, + unsigned int port ) { + unsigned int supported = xhci_supported_protocol ( xhci, port ); + union { + uint32_t raw; + char text[5]; + } name; + unsigned int protocol; + unsigned int type; + unsigned int psic; + unsigned int psiv; + unsigned int i; + uint32_t revision; + uint32_t ports; + uint32_t slot; + uint32_t psi; + + /* Fail if there is no supported protocol */ + if ( ! supported ) + return 0; + + /* Determine protocol version */ + revision = readl ( xhci->cap + supported + XHCI_SUPPORTED_REVISION ); + protocol = XHCI_SUPPORTED_REVISION_VER ( revision ); + + /* Describe port protocol */ + if ( DBG_EXTRA ) { + name.raw = cpu_to_le32 ( readl ( xhci->cap + supported + + XHCI_SUPPORTED_NAME ) ); + name.text[4] = '\0'; + slot = readl ( xhci->cap + supported + XHCI_SUPPORTED_SLOT ); + type = XHCI_SUPPORTED_SLOT_TYPE ( slot ); + DBGC2 ( xhci, "XHCI %s-%d %sv%04x type %d", + xhci->name, port, name.text, protocol, type ); + ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS ); + psic = XHCI_SUPPORTED_PORTS_PSIC ( ports ); + if ( psic ) { + DBGC2 ( xhci, " speeds" ); + for ( i = 0 ; i < psic ; i++ ) { + psi = readl ( xhci->cap + supported + + XHCI_SUPPORTED_PSI ( i ) ); + psiv = XHCI_SUPPORTED_PSI_VALUE ( psi ); + DBGC2 ( xhci, " %d:%s", psiv, + xhci_speed_name ( psi ) ); + } + } + if ( xhci->quirks & XHCI_BAD_PSIV ) + DBGC2 ( xhci, " (ignored)" ); + DBGC2 ( xhci, "\n" ); + } + + return protocol; +} + +/** + * Find port slot type + * + * @v xhci xHCI device + * @v port Port number + * @ret type Slot type, or negative error + */ +static int xhci_port_slot_type ( struct xhci_device *xhci, unsigned int port ) { + unsigned int supported = xhci_supported_protocol ( xhci, port ); + unsigned int type; + uint32_t slot; + + /* Fail if there is no supported protocol */ + if ( ! supported ) + return -ENOTSUP; + + /* Get slot type */ + slot = readl ( xhci->cap + supported + XHCI_SUPPORTED_SLOT ); + type = XHCI_SUPPORTED_SLOT_TYPE ( slot ); + + return type; +} + +/** + * Find port speed + * + * @v xhci xHCI device + * @v port Port number + * @v psiv Protocol speed ID value + * @ret speed Port speed, or negative error + */ +static int xhci_port_speed ( struct xhci_device *xhci, unsigned int port, + unsigned int psiv ) { + unsigned int supported = xhci_supported_protocol ( xhci, port ); + unsigned int psic; + unsigned int mantissa; + unsigned int exponent; + unsigned int speed; + unsigned int i; + uint32_t ports; + uint32_t psi; + + /* Fail if there is no supported protocol */ + if ( ! supported ) + return -ENOTSUP; + + /* Get protocol speed ID count */ + ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS ); + psic = XHCI_SUPPORTED_PORTS_PSIC ( ports ); + + /* Use the default mappings if applicable */ + if ( ( psic == 0 ) || ( xhci->quirks & XHCI_BAD_PSIV ) ) { + switch ( psiv ) { + case XHCI_SPEED_LOW : return USB_SPEED_LOW; + case XHCI_SPEED_FULL : return USB_SPEED_FULL; + case XHCI_SPEED_HIGH : return USB_SPEED_HIGH; + case XHCI_SPEED_SUPER : return USB_SPEED_SUPER; + default: + DBGC ( xhci, "XHCI %s-%d non-standard PSI value %d\n", + xhci->name, port, psiv ); + return -ENOTSUP; + } + } + + /* Iterate over PSI dwords looking for a match */ + for ( i = 0 ; i < psic ; i++ ) { + psi = readl ( xhci->cap + supported + XHCI_SUPPORTED_PSI ( i )); + if ( psiv == XHCI_SUPPORTED_PSI_VALUE ( psi ) ) { + mantissa = XHCI_SUPPORTED_PSI_MANTISSA ( psi ); + exponent = XHCI_SUPPORTED_PSI_EXPONENT ( psi ); + speed = USB_SPEED ( mantissa, exponent ); + return speed; + } + } + + DBGC ( xhci, "XHCI %s-%d spurious PSI value %d\n", + xhci->name, port, psiv ); + return -ENOENT; +} + +/** + * Find protocol speed ID value + * + * @v xhci xHCI device + * @v port Port number + * @v speed USB speed + * @ret psiv Protocol speed ID value, or negative error + */ +static int xhci_port_psiv ( struct xhci_device *xhci, unsigned int port, + unsigned int speed ) { + unsigned int supported = xhci_supported_protocol ( xhci, port ); + unsigned int psic; + unsigned int mantissa; + unsigned int exponent; + unsigned int psiv; + unsigned int i; + uint32_t ports; + uint32_t psi; + + /* Fail if there is no supported protocol */ + if ( ! supported ) + return -ENOTSUP; + + /* Get protocol speed ID count */ + ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS ); + psic = XHCI_SUPPORTED_PORTS_PSIC ( ports ); + + /* Use the default mappings if applicable */ + if ( ( psic == 0 ) || ( xhci->quirks & XHCI_BAD_PSIV ) ) { + switch ( speed ) { + case USB_SPEED_LOW : return XHCI_SPEED_LOW; + case USB_SPEED_FULL : return XHCI_SPEED_FULL; + case USB_SPEED_HIGH : return XHCI_SPEED_HIGH; + case USB_SPEED_SUPER : return XHCI_SPEED_SUPER; + default: + DBGC ( xhci, "XHCI %s-%d non-standard speed %d\n", + xhci->name, port, speed ); + return -ENOTSUP; + } + } + + /* Iterate over PSI dwords looking for a match */ + for ( i = 0 ; i < psic ; i++ ) { + psi = readl ( xhci->cap + supported + XHCI_SUPPORTED_PSI ( i )); + mantissa = XHCI_SUPPORTED_PSI_MANTISSA ( psi ); + exponent = XHCI_SUPPORTED_PSI_EXPONENT ( psi ); + if ( speed == USB_SPEED ( mantissa, exponent ) ) { + psiv = XHCI_SUPPORTED_PSI_VALUE ( psi ); + return psiv; + } + } + + DBGC ( xhci, "XHCI %s-%d unrepresentable speed %#x\n", + xhci->name, port, speed ); + return -ENOENT; +} + +/****************************************************************************** + * + * Device context base address array + * + ****************************************************************************** + */ + +/** + * Allocate device context base address array + * + * @v xhci xHCI device + * @ret rc Return status code + */ +static int xhci_dcbaa_alloc ( struct xhci_device *xhci ) { + size_t len; + physaddr_t dcbaap; + int rc; + + /* Allocate and initialise structure. Must be at least + * 64-byte aligned and must not cross a page boundary, so + * align on its own size (rounded up to a power of two and + * with a minimum of 64 bytes). + */ + len = ( ( xhci->slots + 1 ) * sizeof ( xhci->dcbaa[0] ) ); + xhci->dcbaa = malloc_dma ( len, xhci_align ( len ) ); + if ( ! xhci->dcbaa ) { + DBGC ( xhci, "XHCI %s could not allocate DCBAA\n", xhci->name ); + rc = -ENOMEM; + goto err_alloc; + } + memset ( xhci->dcbaa, 0, len ); + + /* Program DCBAA pointer */ + dcbaap = virt_to_phys ( xhci->dcbaa ); + if ( ( rc = xhci_writeq ( xhci, dcbaap, + xhci->op + XHCI_OP_DCBAAP ) ) != 0 ) + goto err_writeq; + + DBGC2 ( xhci, "XHCI %s DCBAA at [%08lx,%08lx)\n", + xhci->name, dcbaap, ( dcbaap + len ) ); + return 0; + + err_writeq: + free_dma ( xhci->dcbaa, len ); + err_alloc: + return rc; +} + +/** + * Free device context base address array + * + * @v xhci xHCI device + */ +static void xhci_dcbaa_free ( struct xhci_device *xhci ) { + size_t len; + unsigned int i; + + /* Sanity check */ + for ( i = 0 ; i <= xhci->slots ; i++ ) + assert ( xhci->dcbaa[i] == 0 ); + + /* Clear DCBAA pointer */ + xhci_writeq ( xhci, 0, xhci->op + XHCI_OP_DCBAAP ); + + /* Free DCBAA */ + len = ( ( xhci->slots + 1 ) * sizeof ( xhci->dcbaa[0] ) ); + free_dma ( xhci->dcbaa, len ); +} + +/****************************************************************************** + * + * Scratchpad buffers + * + ****************************************************************************** + */ + +/** + * Allocate scratchpad buffers + * + * @v xhci xHCI device + * @ret rc Return status code + */ +static int xhci_scratchpad_alloc ( struct xhci_device *xhci ) { + size_t array_len; + size_t len; + physaddr_t phys; + unsigned int i; + int rc; + + /* Do nothing if no scratchpad buffers are used */ + if ( ! xhci->scratchpads ) + return 0; + + /* Allocate scratchpads */ + len = ( xhci->scratchpads * xhci->pagesize ); + xhci->scratchpad = umalloc ( len ); + if ( ! xhci->scratchpad ) { + DBGC ( xhci, "XHCI %s could not allocate scratchpad buffers\n", + xhci->name ); + rc = -ENOMEM; + goto err_alloc; + } + memset_user ( xhci->scratchpad, 0, 0, len ); + + /* Allocate scratchpad array */ + array_len = ( xhci->scratchpads * sizeof ( xhci->scratchpad_array[0] )); + xhci->scratchpad_array = + malloc_dma ( array_len, xhci_align ( array_len ) ); + if ( ! xhci->scratchpad_array ) { + DBGC ( xhci, "XHCI %s could not allocate scratchpad buffer " + "array\n", xhci->name ); + rc = -ENOMEM; + goto err_alloc_array; + } + + /* Populate scratchpad array */ + for ( i = 0 ; i < xhci->scratchpads ; i++ ) { + phys = user_to_phys ( xhci->scratchpad, ( i * xhci->pagesize )); + xhci->scratchpad_array[i] = phys; + } + + /* Set scratchpad array pointer */ + assert ( xhci->dcbaa != NULL ); + xhci->dcbaa[0] = cpu_to_le64 ( virt_to_phys ( xhci->scratchpad_array )); + + DBGC2 ( xhci, "XHCI %s scratchpad [%08lx,%08lx) array [%08lx,%08lx)\n", + xhci->name, user_to_phys ( xhci->scratchpad, 0 ), + user_to_phys ( xhci->scratchpad, len ), + virt_to_phys ( xhci->scratchpad_array ), + ( virt_to_phys ( xhci->scratchpad_array ) + array_len ) ); + return 0; + + free_dma ( xhci->scratchpad_array, array_len ); + err_alloc_array: + ufree ( xhci->scratchpad ); + err_alloc: + return rc; +} + +/** + * Free scratchpad buffers + * + * @v xhci xHCI device + */ +static void xhci_scratchpad_free ( struct xhci_device *xhci ) { + size_t array_len; + + /* Do nothing if no scratchpad buffers are used */ + if ( ! xhci->scratchpads ) + return; + + /* Clear scratchpad array pointer */ + assert ( xhci->dcbaa != NULL ); + xhci->dcbaa[0] = 0; + + /* Free scratchpad array */ + array_len = ( xhci->scratchpads * sizeof ( xhci->scratchpad_array[0] )); + free_dma ( xhci->scratchpad_array, array_len ); + + /* Free scratchpads */ + ufree ( xhci->scratchpad ); +} + +/****************************************************************************** + * + * Run / stop / reset + * + ****************************************************************************** + */ + +/** + * Start xHCI device + * + * @v xhci xHCI device + */ +static void xhci_run ( struct xhci_device *xhci ) { + uint32_t config; + uint32_t usbcmd; + + /* Configure number of device slots */ + config = readl ( xhci->op + XHCI_OP_CONFIG ); + config &= ~XHCI_CONFIG_MAX_SLOTS_EN_MASK; + config |= XHCI_CONFIG_MAX_SLOTS_EN ( xhci->slots ); + writel ( config, xhci->op + XHCI_OP_CONFIG ); + + /* Set run/stop bit */ + usbcmd = readl ( xhci->op + XHCI_OP_USBCMD ); + usbcmd |= XHCI_USBCMD_RUN; + writel ( usbcmd, xhci->op + XHCI_OP_USBCMD ); +} + +/** + * Stop xHCI device + * + * @v xhci xHCI device + * @ret rc Return status code + */ +static int xhci_stop ( struct xhci_device *xhci ) { + uint32_t usbcmd; + uint32_t usbsts; + unsigned int i; + + /* Clear run/stop bit */ + usbcmd = readl ( xhci->op + XHCI_OP_USBCMD ); + usbcmd &= ~XHCI_USBCMD_RUN; + writel ( usbcmd, xhci->op + XHCI_OP_USBCMD ); + + /* Wait for device to stop */ + for ( i = 0 ; i < XHCI_STOP_MAX_WAIT_MS ; i++ ) { + + /* Check if device is stopped */ + usbsts = readl ( xhci->op + XHCI_OP_USBSTS ); + if ( usbsts & XHCI_USBSTS_HCH ) + return 0; + + /* Delay */ + mdelay ( 1 ); + } + + DBGC ( xhci, "XHCI %s timed out waiting for stop\n", xhci->name ); + return -ETIMEDOUT; +} + +/** + * Reset xHCI device + * + * @v xhci xHCI device + * @ret rc Return status code + */ +static int xhci_reset ( struct xhci_device *xhci ) { + uint32_t usbcmd; + unsigned int i; + int rc; + + /* The xHCI specification states that resetting a running + * device may result in undefined behaviour, so try stopping + * it first. + */ + if ( ( rc = xhci_stop ( xhci ) ) != 0 ) { + /* Ignore errors and attempt to reset the device anyway */ + } + + /* Reset device */ + writel ( XHCI_USBCMD_HCRST, xhci->op + XHCI_OP_USBCMD ); + + /* Wait for reset to complete */ + for ( i = 0 ; i < XHCI_RESET_MAX_WAIT_MS ; i++ ) { + + /* Check if reset is complete */ + usbcmd = readl ( xhci->op + XHCI_OP_USBCMD ); + if ( ! ( usbcmd & XHCI_USBCMD_HCRST ) ) + return 0; + + /* Delay */ + mdelay ( 1 ); + } + + DBGC ( xhci, "XHCI %s timed out waiting for reset\n", xhci->name ); + return -ETIMEDOUT; +} + +/****************************************************************************** + * + * Transfer request blocks + * + ****************************************************************************** + */ + +/** + * Allocate transfer request block ring + * + * @v xhci xHCI device + * @v ring TRB ring + * @v shift Ring size (log2) + * @v slot Device slot + * @v target Doorbell target + * @v stream Doorbell stream ID + * @ret rc Return status code + */ +static int xhci_ring_alloc ( struct xhci_device *xhci, + struct xhci_trb_ring *ring, + unsigned int shift, unsigned int slot, + unsigned int target, unsigned int stream ) { + struct xhci_trb_link *link; + unsigned int count; + int rc; + + /* Sanity check */ + assert ( shift > 0 ); + + /* Initialise structure */ + memset ( ring, 0, sizeof ( *ring ) ); + ring->shift = shift; + count = ( 1U << shift ); + ring->mask = ( count - 1 ); + ring->len = ( ( count + 1 /* Link TRB */ ) * sizeof ( ring->trb[0] ) ); + ring->db = ( xhci->db + ( slot * sizeof ( ring->dbval ) ) ); + ring->dbval = XHCI_DBVAL ( target, stream ); + + /* Allocate I/O buffers */ + ring->iobuf = zalloc ( count * sizeof ( ring->iobuf[0] ) ); + if ( ! ring->iobuf ) { + rc = -ENOMEM; + goto err_alloc_iobuf; + } + + /* Allocate TRBs */ + ring->trb = malloc_dma ( ring->len, xhci_align ( ring->len ) ); + if ( ! ring->trb ) { + rc = -ENOMEM; + goto err_alloc_trb; + } + memset ( ring->trb, 0, ring->len ); + + /* Initialise Link TRB */ + link = &ring->trb[count].link; + link->next = cpu_to_le64 ( virt_to_phys ( ring->trb ) ); + link->flags = XHCI_TRB_TC; + link->type = XHCI_TRB_LINK; + ring->link = link; + + return 0; + + free_dma ( ring->trb, ring->len ); + err_alloc_trb: + free ( ring->iobuf ); + err_alloc_iobuf: + return rc; +} + +/** + * Reset transfer request block ring + * + * @v ring TRB ring + */ +static void xhci_ring_reset ( struct xhci_trb_ring *ring ) { + unsigned int count = ( 1U << ring->shift ); + + /* Reset producer and consumer counters */ + ring->prod = 0; + ring->cons = 0; + + /* Reset TRBs (except Link TRB) */ + memset ( ring->trb, 0, ( count * sizeof ( ring->trb[0] ) ) ); +} + +/** + * Free transfer request block ring + * + * @v ring TRB ring + */ +static void xhci_ring_free ( struct xhci_trb_ring *ring ) { + unsigned int count = ( 1U << ring->shift ); + unsigned int i; + + /* Sanity checks */ + assert ( ring->cons == ring->prod ); + for ( i = 0 ; i < count ; i++ ) + assert ( ring->iobuf[i] == NULL ); + + /* Free TRBs */ + free_dma ( ring->trb, ring->len ); + + /* Free I/O buffers */ + free ( ring->iobuf ); +} + +/** + * Enqueue a transfer request block + * + * @v ring TRB ring + * @v iobuf I/O buffer (if any) + * @v trb Transfer request block (with empty Cycle flag) + * @ret rc Return status code + * + * This operation does not implicitly ring the doorbell register. + */ +static int xhci_enqueue ( struct xhci_trb_ring *ring, struct io_buffer *iobuf, + const union xhci_trb *trb ) { + union xhci_trb *dest; + unsigned int prod; + unsigned int mask; + unsigned int index; + unsigned int cycle; + + /* Sanity check */ + assert ( ! ( trb->common.flags & XHCI_TRB_C ) ); + + /* Fail if ring is full */ + if ( ! xhci_ring_remaining ( ring ) ) + return -ENOBUFS; + + /* Update producer counter (and link TRB, if applicable) */ + prod = ring->prod++; + mask = ring->mask; + cycle = ( ( ~( prod >> ring->shift ) ) & XHCI_TRB_C ); + index = ( prod & mask ); + if ( index == 0 ) + ring->link->flags = ( XHCI_TRB_TC | ( cycle ^ XHCI_TRB_C ) ); + + /* Record I/O buffer */ + ring->iobuf[index] = iobuf; + + /* Enqueue TRB */ + dest = &ring->trb[index]; + dest->template.parameter = trb->template.parameter; + dest->template.status = trb->template.status; + wmb(); + dest->template.control = ( trb->template.control | + cpu_to_le32 ( cycle ) ); + + return 0; +} + +/** + * Dequeue a transfer request block + * + * @v ring TRB ring + * @ret iobuf I/O buffer + */ +static struct io_buffer * xhci_dequeue ( struct xhci_trb_ring *ring ) { + struct io_buffer *iobuf; + unsigned int cons; + unsigned int mask; + unsigned int index; + + /* Sanity check */ + assert ( xhci_ring_fill ( ring ) != 0 ); + + /* Update consumer counter */ + cons = ring->cons++; + mask = ring->mask; + index = ( cons & mask ); + + /* Retrieve I/O buffer */ + iobuf = ring->iobuf[index]; + ring->iobuf[index] = NULL; + + return iobuf; +} + +/** + * Enqueue multiple transfer request blocks + * + * @v ring TRB ring + * @v iobuf I/O buffer + * @v trbs Transfer request blocks (with empty Cycle flag) + * @v count Number of transfer request blocks + * @ret rc Return status code + * + * This operation does not implicitly ring the doorbell register. + */ +static int xhci_enqueue_multi ( struct xhci_trb_ring *ring, + struct io_buffer *iobuf, + const union xhci_trb *trbs, + unsigned int count ) { + const union xhci_trb *trb = trbs; + int rc; + + /* Sanity check */ + assert ( iobuf != NULL ); + + /* Fail if ring does not have sufficient space */ + if ( xhci_ring_remaining ( ring ) < count ) + return -ENOBUFS; + + /* Enqueue each TRB, recording the I/O buffer with the final TRB */ + while ( count-- ) { + rc = xhci_enqueue ( ring, ( count ? NULL : iobuf ), trb++ ); + assert ( rc == 0 ); /* Should never be able to fail */ + } + + return 0; +} + +/** + * Dequeue multiple transfer request blocks + * + * @v ring TRB ring + * @ret iobuf I/O buffer + */ +static struct io_buffer * xhci_dequeue_multi ( struct xhci_trb_ring *ring ) { + struct io_buffer *iobuf; + + /* Dequeue TRBs until we reach the final TRB for an I/O buffer */ + do { + iobuf = xhci_dequeue ( ring ); + } while ( iobuf == NULL ); + + return iobuf; +} + +/** + * Ring doorbell register + * + * @v ring TRB ring + */ +static inline __attribute__ (( always_inline )) void +xhci_doorbell ( struct xhci_trb_ring *ring ) { + + wmb(); + writel ( ring->dbval, ring->db ); +} + +/****************************************************************************** + * + * Command and event rings + * + ****************************************************************************** + */ + +/** + * Allocate command ring + * + * @v xhci xHCI device + * @ret rc Return status code + */ +static int xhci_command_alloc ( struct xhci_device *xhci ) { + physaddr_t crp; + int rc; + + /* Allocate TRB ring */ + if ( ( rc = xhci_ring_alloc ( xhci, &xhci->command, XHCI_CMD_TRBS_LOG2, + 0, 0, 0 ) ) != 0 ) + goto err_ring_alloc; + + /* Program command ring control register */ + crp = virt_to_phys ( xhci->command.trb ); + if ( ( rc = xhci_writeq ( xhci, ( crp | XHCI_CRCR_RCS ), + xhci->op + XHCI_OP_CRCR ) ) != 0 ) + goto err_writeq; + + DBGC2 ( xhci, "XHCI %s CRCR at [%08lx,%08lx)\n", + xhci->name, crp, ( crp + xhci->command.len ) ); + return 0; + + err_writeq: + xhci_ring_free ( &xhci->command ); + err_ring_alloc: + return rc; +} + +/** + * Free command ring + * + * @v xhci xHCI device + */ +static void xhci_command_free ( struct xhci_device *xhci ) { + + /* Sanity check */ + assert ( ( readl ( xhci->op + XHCI_OP_CRCR ) & XHCI_CRCR_CRR ) == 0 ); + + /* Clear command ring control register */ + xhci_writeq ( xhci, 0, xhci->op + XHCI_OP_CRCR ); + + /* Free TRB ring */ + xhci_ring_free ( &xhci->command ); +} + +/** + * Allocate event ring + * + * @v xhci xHCI device + * @ret rc Return status code + */ +static int xhci_event_alloc ( struct xhci_device *xhci ) { + struct xhci_event_ring *event = &xhci->event; + unsigned int count; + size_t len; + int rc; + + /* Allocate event ring */ + count = ( 1 << XHCI_EVENT_TRBS_LOG2 ); + len = ( count * sizeof ( event->trb[0] ) ); + event->trb = malloc_dma ( len, xhci_align ( len ) ); + if ( ! event->trb ) { + rc = -ENOMEM; + goto err_alloc_trb; + } + memset ( event->trb, 0, len ); + + /* Allocate event ring segment table */ + event->segment = malloc_dma ( sizeof ( event->segment[0] ), + xhci_align ( sizeof (event->segment[0]))); + if ( ! event->segment ) { + rc = -ENOMEM; + goto err_alloc_segment; + } + memset ( event->segment, 0, sizeof ( event->segment[0] ) ); + event->segment[0].base = cpu_to_le64 ( virt_to_phys ( event->trb ) ); + event->segment[0].count = cpu_to_le32 ( count ); + + /* Program event ring registers */ + writel ( 1, xhci->run + XHCI_RUN_ERSTSZ ( 0 ) ); + if ( ( rc = xhci_writeq ( xhci, virt_to_phys ( event->trb ), + xhci->run + XHCI_RUN_ERDP ( 0 ) ) ) != 0 ) + goto err_writeq_erdp; + if ( ( rc = xhci_writeq ( xhci, virt_to_phys ( event->segment ), + xhci->run + XHCI_RUN_ERSTBA ( 0 ) ) ) != 0 ) + goto err_writeq_erstba; + + DBGC2 ( xhci, "XHCI %s event ring [%08lx,%08lx) table [%08lx,%08lx)\n", + xhci->name, virt_to_phys ( event->trb ), + ( virt_to_phys ( event->trb ) + len ), + virt_to_phys ( event->segment ), + ( virt_to_phys ( event->segment ) + + sizeof (event->segment[0] ) ) ); + return 0; + + xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERSTBA ( 0 ) ); + err_writeq_erstba: + xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERDP ( 0 ) ); + err_writeq_erdp: + free_dma ( event->trb, len ); + err_alloc_segment: + free_dma ( event->segment, sizeof ( event->segment[0] ) ); + err_alloc_trb: + return rc; +} + +/** + * Free event ring + * + * @v xhci xHCI device + */ +static void xhci_event_free ( struct xhci_device *xhci ) { + struct xhci_event_ring *event = &xhci->event; + unsigned int count; + size_t len; + + /* Clear event ring registers */ + writel ( 0, xhci->run + XHCI_RUN_ERSTSZ ( 0 ) ); + xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERSTBA ( 0 ) ); + xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERDP ( 0 ) ); + + /* Free event ring segment table */ + free_dma ( event->segment, sizeof ( event->segment[0] ) ); + + /* Free event ring */ + count = ( 1 << XHCI_EVENT_TRBS_LOG2 ); + len = ( count * sizeof ( event->trb[0] ) ); + free_dma ( event->trb, len ); +} + +/** + * Handle transfer event + * + * @v xhci xHCI device + * @v trb Transfer event TRB + */ +static void xhci_transfer ( struct xhci_device *xhci, + struct xhci_trb_transfer *trb ) { + struct xhci_slot *slot; + struct xhci_endpoint *endpoint; + struct io_buffer *iobuf; + int rc; + + /* Profile transfer events */ + profile_start ( &xhci_transfer_profiler ); + + /* Identify slot */ + if ( ( trb->slot > xhci->slots ) || + ( ( slot = xhci->slot[trb->slot] ) == NULL ) ) { + DBGC ( xhci, "XHCI %s transfer event invalid slot %d:\n", + xhci->name, trb->slot ); + DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) ); + return; + } + + /* Identify endpoint */ + if ( ( trb->endpoint > XHCI_CTX_END ) || + ( ( endpoint = slot->endpoint[trb->endpoint] ) == NULL ) ) { + DBGC ( xhci, "XHCI %s slot %d transfer event invalid epid " + "%d:\n", xhci->name, slot->id, trb->endpoint ); + DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) ); + return; + } + + /* Dequeue TRB(s) */ + iobuf = xhci_dequeue_multi ( &endpoint->ring ); + assert ( iobuf != NULL ); + + /* Check for errors */ + if ( ! ( ( trb->code == XHCI_CMPLT_SUCCESS ) || + ( trb->code == XHCI_CMPLT_SHORT ) ) ) { + + /* Construct error */ + rc = -ECODE ( trb->code ); + DBGC ( xhci, "XHCI %s slot %d ctx %d failed (code %d): %s\n", + xhci->name, slot->id, endpoint->ctx, trb->code, + strerror ( rc ) ); + DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) ); + + /* Sanity check */ + assert ( ( endpoint->context->state & XHCI_ENDPOINT_STATE_MASK ) + != XHCI_ENDPOINT_RUNNING ); + + /* Report failure to USB core */ + usb_complete_err ( endpoint->ep, iobuf, rc ); + return; + } + + /* Record actual transfer size */ + iob_unput ( iobuf, le16_to_cpu ( trb->residual ) ); + + /* Sanity check (for successful completions only) */ + assert ( xhci_ring_consumed ( &endpoint->ring ) == + le64_to_cpu ( trb->transfer ) ); + + /* Report completion to USB core */ + usb_complete ( endpoint->ep, iobuf ); + profile_stop ( &xhci_transfer_profiler ); +} + +/** + * Handle command completion event + * + * @v xhci xHCI device + * @v trb Command completion event + */ +static void xhci_complete ( struct xhci_device *xhci, + struct xhci_trb_complete *trb ) { + int rc; + + /* Ignore "command ring stopped" notifications */ + if ( trb->code == XHCI_CMPLT_CMD_STOPPED ) { + DBGC2 ( xhci, "XHCI %s command ring stopped\n", xhci->name ); + return; + } + + /* Ignore unexpected completions */ + if ( ! xhci->pending ) { + rc = -ECODE ( trb->code ); + DBGC ( xhci, "XHCI %s unexpected completion (code %d): %s\n", + xhci->name, trb->code, strerror ( rc ) ); + DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) ); + return; + } + + /* Dequeue command TRB */ + xhci_dequeue ( &xhci->command ); + + /* Sanity check */ + assert ( xhci_ring_consumed ( &xhci->command ) == + le64_to_cpu ( trb->command ) ); + + /* Record completion */ + memcpy ( xhci->pending, trb, sizeof ( *xhci->pending ) ); + xhci->pending = NULL; +} + +/** + * Handle port status event + * + * @v xhci xHCI device + * @v trb Port status event + */ +static void xhci_port_status ( struct xhci_device *xhci, + struct xhci_trb_port_status *trb ) { + struct usb_port *port = usb_port ( xhci->bus->hub, trb->port ); + uint32_t portsc; + + /* Sanity check */ + assert ( ( trb->port > 0 ) && ( trb->port <= xhci->ports ) ); + + /* Record disconnections and clear changes */ + portsc = readl ( xhci->op + XHCI_OP_PORTSC ( trb->port ) ); + port->disconnected |= ( portsc & XHCI_PORTSC_CSC ); + portsc &= ( XHCI_PORTSC_PRESERVE | XHCI_PORTSC_CHANGE ); + writel ( portsc, xhci->op + XHCI_OP_PORTSC ( trb->port ) ); + + /* Report port status change */ + usb_port_changed ( port ); +} + +/** + * Handle host controller event + * + * @v xhci xHCI device + * @v trb Host controller event + */ +static void xhci_host_controller ( struct xhci_device *xhci, + struct xhci_trb_host_controller *trb ) { + int rc; + + /* Construct error */ + rc = -ECODE ( trb->code ); + DBGC ( xhci, "XHCI %s host controller event (code %d): %s\n", + xhci->name, trb->code, strerror ( rc ) ); +} + +/** + * Poll event ring + * + * @v xhci xHCI device + */ +static void xhci_event_poll ( struct xhci_device *xhci ) { + struct xhci_event_ring *event = &xhci->event; + union xhci_trb *trb; + unsigned int shift = XHCI_EVENT_TRBS_LOG2; + unsigned int count = ( 1 << shift ); + unsigned int mask = ( count - 1 ); + unsigned int consumed; + unsigned int type; + + /* Poll for events */ + profile_start ( &xhci_event_profiler ); + for ( consumed = 0 ; ; consumed++ ) { + + /* Stop if we reach an empty TRB */ + rmb(); + trb = &event->trb[ event->cons & mask ]; + if ( ! ( ( trb->common.flags ^ + ( event->cons >> shift ) ) & XHCI_TRB_C ) ) + break; + + /* Handle TRB */ + type = ( trb->common.type & XHCI_TRB_TYPE_MASK ); + switch ( type ) { + + case XHCI_TRB_TRANSFER : + xhci_transfer ( xhci, &trb->transfer ); + break; + + case XHCI_TRB_COMPLETE : + xhci_complete ( xhci, &trb->complete ); + break; + + case XHCI_TRB_PORT_STATUS: + xhci_port_status ( xhci, &trb->port ); + break; + + case XHCI_TRB_HOST_CONTROLLER: + xhci_host_controller ( xhci, &trb->host ); + break; + + default: + DBGC ( xhci, "XHCI %s unrecognised event %#x\n:", + xhci->name, event->cons ); + DBGC_HDA ( xhci, virt_to_phys ( trb ), + trb, sizeof ( *trb ) ); + break; + } + + /* Consume this TRB */ + event->cons++; + } + + /* Update dequeue pointer if applicable */ + if ( consumed ) { + xhci_writeq ( xhci, virt_to_phys ( trb ), + xhci->run + XHCI_RUN_ERDP ( 0 ) ); + profile_stop ( &xhci_event_profiler ); + } +} + +/** + * Abort command + * + * @v xhci xHCI device + */ +static void xhci_abort ( struct xhci_device *xhci ) { + physaddr_t crp; + + /* Abort the command */ + DBGC2 ( xhci, "XHCI %s aborting command\n", xhci->name ); + xhci_writeq ( xhci, XHCI_CRCR_CA, xhci->op + XHCI_OP_CRCR ); + + /* Allow time for command to abort */ + mdelay ( XHCI_COMMAND_ABORT_DELAY_MS ); + + /* Sanity check */ + assert ( ( readl ( xhci->op + XHCI_OP_CRCR ) & XHCI_CRCR_CRR ) == 0 ); + + /* Consume (and ignore) any final command status */ + xhci_event_poll ( xhci ); + + /* Reset the command ring control register */ + xhci_ring_reset ( &xhci->command ); + crp = virt_to_phys ( xhci->command.trb ); + xhci_writeq ( xhci, ( crp | XHCI_CRCR_RCS ), xhci->op + XHCI_OP_CRCR ); +} + +/** + * Issue command and wait for completion + * + * @v xhci xHCI device + * @v trb Transfer request block (with empty Cycle flag) + * @ret rc Return status code + * + * On a successful completion, the TRB will be overwritten with the + * completion. + */ +static int xhci_command ( struct xhci_device *xhci, union xhci_trb *trb ) { + struct xhci_trb_complete *complete = &trb->complete; + unsigned int i; + int rc; + + /* Record the pending command */ + xhci->pending = trb; + + /* Enqueue the command */ + if ( ( rc = xhci_enqueue ( &xhci->command, NULL, trb ) ) != 0 ) + goto err_enqueue; + + /* Ring the command doorbell */ + xhci_doorbell ( &xhci->command ); + + /* Wait for the command to complete */ + for ( i = 0 ; i < XHCI_COMMAND_MAX_WAIT_MS ; i++ ) { + + /* Poll event ring */ + xhci_event_poll ( xhci ); + + /* Check for completion */ + if ( ! xhci->pending ) { + if ( complete->code != XHCI_CMPLT_SUCCESS ) { + rc = -ECODE ( complete->code ); + DBGC ( xhci, "XHCI %s command failed (code " + "%d): %s\n", xhci->name, complete->code, + strerror ( rc ) ); + DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) ); + return rc; + } + return 0; + } + + /* Delay */ + mdelay ( 1 ); + } + + /* Timeout */ + DBGC ( xhci, "XHCI %s timed out waiting for completion\n", xhci->name ); + rc = -ETIMEDOUT; + + /* Abort command */ + xhci_abort ( xhci ); + + err_enqueue: + xhci->pending = NULL; + return rc; +} + +/** + * Issue NOP and wait for completion + * + * @v xhci xHCI device + * @ret rc Return status code + */ +static inline int xhci_nop ( struct xhci_device *xhci ) { + union xhci_trb trb; + struct xhci_trb_common *nop = &trb.common; + int rc; + + /* Construct command */ + memset ( nop, 0, sizeof ( *nop ) ); + nop->flags = XHCI_TRB_IOC; + nop->type = XHCI_TRB_NOP_CMD; + + /* Issue command and wait for completion */ + if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) + return rc; + + return 0; +} + +/** + * Enable slot + * + * @v xhci xHCI device + * @v type Slot type + * @ret slot Device slot ID, or negative error + */ +static inline int xhci_enable_slot ( struct xhci_device *xhci, + unsigned int type ) { + union xhci_trb trb; + struct xhci_trb_enable_slot *enable = &trb.enable; + struct xhci_trb_complete *enabled = &trb.complete; + unsigned int slot; + int rc; + + /* Construct command */ + memset ( enable, 0, sizeof ( *enable ) ); + enable->slot = type; + enable->type = XHCI_TRB_ENABLE_SLOT; + + /* Issue command and wait for completion */ + if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) { + DBGC ( xhci, "XHCI %s could not enable new slot: %s\n", + xhci->name, strerror ( rc ) ); + return rc; + } + + /* Extract slot number */ + slot = enabled->slot; + + DBGC2 ( xhci, "XHCI %s slot %d enabled\n", xhci->name, slot ); + return slot; +} + +/** + * Disable slot + * + * @v xhci xHCI device + * @v slot Device slot + * @ret rc Return status code + */ +static inline int xhci_disable_slot ( struct xhci_device *xhci, + unsigned int slot ) { + union xhci_trb trb; + struct xhci_trb_disable_slot *disable = &trb.disable; + int rc; + + /* Construct command */ + memset ( disable, 0, sizeof ( *disable ) ); + disable->type = XHCI_TRB_DISABLE_SLOT; + disable->slot = slot; + + /* Issue command and wait for completion */ + if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) { + DBGC ( xhci, "XHCI %s could not disable slot %d: %s\n", + xhci->name, slot, strerror ( rc ) ); + return rc; + } + + DBGC2 ( xhci, "XHCI %s slot %d disabled\n", xhci->name, slot ); + return 0; +} + +/** + * Issue context-based command and wait for completion + * + * @v xhci xHCI device + * @v slot Device slot + * @v endpoint Endpoint + * @v type TRB type + * @v populate Input context populater + * @ret rc Return status code + */ +static int xhci_context ( struct xhci_device *xhci, struct xhci_slot *slot, + struct xhci_endpoint *endpoint, unsigned int type, + void ( * populate ) ( struct xhci_device *xhci, + struct xhci_slot *slot, + struct xhci_endpoint *endpoint, + void *input ) ) { + union xhci_trb trb; + struct xhci_trb_context *context = &trb.context; + size_t len; + void *input; + int rc; + + /* Allocate an input context */ + len = xhci_input_context_offset ( xhci, XHCI_CTX_END ); + input = malloc_dma ( len, xhci_align ( len ) ); + if ( ! input ) { + rc = -ENOMEM; + goto err_alloc; + } + memset ( input, 0, len ); + + /* Populate input context */ + populate ( xhci, slot, endpoint, input ); + + /* Construct command */ + memset ( context, 0, sizeof ( *context ) ); + context->type = type; + context->input = cpu_to_le64 ( virt_to_phys ( input ) ); + context->slot = slot->id; + + /* Issue command and wait for completion */ + if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) + goto err_command; + + err_command: + free_dma ( input, len ); + err_alloc: + return rc; +} + +/** + * Populate address device input context + * + * @v xhci xHCI device + * @v slot Device slot + * @v endpoint Endpoint + * @v input Input context + */ +static void xhci_address_device_input ( struct xhci_device *xhci, + struct xhci_slot *slot, + struct xhci_endpoint *endpoint, + void *input ) { + struct xhci_control_context *control_ctx; + struct xhci_slot_context *slot_ctx; + struct xhci_endpoint_context *ep_ctx; + + /* Sanity checks */ + assert ( endpoint->ctx == XHCI_CTX_EP0 ); + + /* Populate control context */ + control_ctx = input; + control_ctx->add = cpu_to_le32 ( ( 1 << XHCI_CTX_SLOT ) | + ( 1 << XHCI_CTX_EP0 ) ); + + /* Populate slot context */ + slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT )); + slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( 1, 0, slot->psiv, + slot->route ) ); + slot_ctx->port = slot->port; + slot_ctx->tt_id = slot->tt_id; + slot_ctx->tt_port = slot->tt_port; + + /* Populate control endpoint context */ + ep_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_EP0 ) ); + ep_ctx->type = XHCI_EP_TYPE_CONTROL; + ep_ctx->burst = endpoint->ep->burst; + ep_ctx->mtu = cpu_to_le16 ( endpoint->ep->mtu ); + ep_ctx->dequeue = cpu_to_le64 ( virt_to_phys ( endpoint->ring.trb ) | + XHCI_EP_DCS ); + ep_ctx->trb_len = cpu_to_le16 ( XHCI_EP0_TRB_LEN ); +} + +/** + * Address device + * + * @v xhci xHCI device + * @v slot Device slot + * @ret rc Return status code + */ +static inline int xhci_address_device ( struct xhci_device *xhci, + struct xhci_slot *slot ) { + struct usb_device *usb = slot->usb; + struct xhci_slot_context *slot_ctx; + int rc; + + /* Assign device address */ + if ( ( rc = xhci_context ( xhci, slot, slot->endpoint[XHCI_CTX_EP0], + XHCI_TRB_ADDRESS_DEVICE, + xhci_address_device_input ) ) != 0 ) + return rc; + + /* Get assigned address */ + slot_ctx = ( slot->context + + xhci_device_context_offset ( xhci, XHCI_CTX_SLOT ) ); + usb->address = slot_ctx->address; + DBGC2 ( xhci, "XHCI %s assigned address %d to %s\n", + xhci->name, usb->address, usb->name ); + + return 0; +} + +/** + * Populate configure endpoint input context + * + * @v xhci xHCI device + * @v slot Device slot + * @v endpoint Endpoint + * @v input Input context + */ +static void xhci_configure_endpoint_input ( struct xhci_device *xhci, + struct xhci_slot *slot, + struct xhci_endpoint *endpoint, + void *input ) { + struct xhci_control_context *control_ctx; + struct xhci_slot_context *slot_ctx; + struct xhci_endpoint_context *ep_ctx; + + /* Populate control context */ + control_ctx = input; + control_ctx->add = cpu_to_le32 ( ( 1 << XHCI_CTX_SLOT ) | + ( 1 << endpoint->ctx ) ); + + /* Populate slot context */ + slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT )); + slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( ( XHCI_CTX_END - 1 ), + ( slot->ports ? 1 : 0 ), + slot->psiv, 0 ) ); + slot_ctx->ports = slot->ports; + + /* Populate endpoint context */ + ep_ctx = ( input + xhci_input_context_offset ( xhci, endpoint->ctx ) ); + ep_ctx->interval = endpoint->interval; + ep_ctx->type = endpoint->type; + ep_ctx->burst = endpoint->ep->burst; + ep_ctx->mtu = cpu_to_le16 ( endpoint->ep->mtu ); + ep_ctx->dequeue = cpu_to_le64 ( virt_to_phys ( endpoint->ring.trb ) | + XHCI_EP_DCS ); + ep_ctx->trb_len = cpu_to_le16 ( endpoint->ep->mtu ); /* best guess */ +} + +/** + * Configure endpoint + * + * @v xhci xHCI device + * @v slot Device slot + * @v endpoint Endpoint + * @ret rc Return status code + */ +static inline int xhci_configure_endpoint ( struct xhci_device *xhci, + struct xhci_slot *slot, + struct xhci_endpoint *endpoint ) { + int rc; + + /* Configure endpoint */ + if ( ( rc = xhci_context ( xhci, slot, endpoint, + XHCI_TRB_CONFIGURE_ENDPOINT, + xhci_configure_endpoint_input ) ) != 0 ) + return rc; + + DBGC2 ( xhci, "XHCI %s slot %d ctx %d configured\n", + xhci->name, slot->id, endpoint->ctx ); + return 0; +} + +/** + * Populate deconfigure endpoint input context + * + * @v xhci xHCI device + * @v slot Device slot + * @v endpoint Endpoint + * @v input Input context + */ +static void +xhci_deconfigure_endpoint_input ( struct xhci_device *xhci __unused, + struct xhci_slot *slot __unused, + struct xhci_endpoint *endpoint, + void *input ) { + struct xhci_control_context *control_ctx; + struct xhci_slot_context *slot_ctx; + + /* Populate control context */ + control_ctx = input; + control_ctx->add = cpu_to_le32 ( 1 << XHCI_CTX_SLOT ); + control_ctx->drop = cpu_to_le32 ( 1 << endpoint->ctx ); + + /* Populate slot context */ + slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT )); + slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( ( XHCI_CTX_END - 1 ), + 0, 0, 0 ) ); +} + +/** + * Deconfigure endpoint + * + * @v xhci xHCI device + * @v slot Device slot + * @v endpoint Endpoint + * @ret rc Return status code + */ +static inline int xhci_deconfigure_endpoint ( struct xhci_device *xhci, + struct xhci_slot *slot, + struct xhci_endpoint *endpoint ) { + int rc; + + /* Deconfigure endpoint */ + if ( ( rc = xhci_context ( xhci, slot, endpoint, + XHCI_TRB_CONFIGURE_ENDPOINT, + xhci_deconfigure_endpoint_input ) ) != 0 ) + return rc; + + DBGC2 ( xhci, "XHCI %s slot %d ctx %d deconfigured\n", + xhci->name, slot->id, endpoint->ctx ); + return 0; +} + +/** + * Populate evaluate context input context + * + * @v xhci xHCI device + * @v slot Device slot + * @v endpoint Endpoint + * @v input Input context + */ +static void xhci_evaluate_context_input ( struct xhci_device *xhci, + struct xhci_slot *slot __unused, + struct xhci_endpoint *endpoint, + void *input ) { + struct xhci_control_context *control_ctx; + struct xhci_slot_context *slot_ctx; + struct xhci_endpoint_context *ep_ctx; + + /* Populate control context */ + control_ctx = input; + control_ctx->add = cpu_to_le32 ( ( 1 << XHCI_CTX_SLOT ) | + ( 1 << endpoint->ctx ) ); + + /* Populate slot context */ + slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT )); + slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( ( XHCI_CTX_END - 1 ), + 0, 0, 0 ) ); + + /* Populate endpoint context */ + ep_ctx = ( input + xhci_input_context_offset ( xhci, endpoint->ctx ) ); + ep_ctx->mtu = cpu_to_le16 ( endpoint->ep->mtu ); +} + +/** + * Evaluate context + * + * @v xhci xHCI device + * @v slot Device slot + * @v endpoint Endpoint + * @ret rc Return status code + */ +static inline int xhci_evaluate_context ( struct xhci_device *xhci, + struct xhci_slot *slot, + struct xhci_endpoint *endpoint ) { + int rc; + + /* Configure endpoint */ + if ( ( rc = xhci_context ( xhci, slot, endpoint, + XHCI_TRB_EVALUATE_CONTEXT, + xhci_evaluate_context_input ) ) != 0 ) + return rc; + + DBGC2 ( xhci, "XHCI %s slot %d ctx %d (re-)evaluated\n", + xhci->name, slot->id, endpoint->ctx ); + return 0; +} + +/** + * Reset endpoint + * + * @v xhci xHCI device + * @v slot Device slot + * @v endpoint Endpoint + * @ret rc Return status code + */ +static inline int xhci_reset_endpoint ( struct xhci_device *xhci, + struct xhci_slot *slot, + struct xhci_endpoint *endpoint ) { + union xhci_trb trb; + struct xhci_trb_reset_endpoint *reset = &trb.reset; + int rc; + + /* Construct command */ + memset ( reset, 0, sizeof ( *reset ) ); + reset->slot = slot->id; + reset->endpoint = endpoint->ctx; + reset->type = XHCI_TRB_RESET_ENDPOINT; + + /* Issue command and wait for completion */ + if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) { + DBGC ( xhci, "XHCI %s slot %d ctx %d could not reset endpoint " + "in state %d: %s\n", xhci->name, slot->id, endpoint->ctx, + endpoint->context->state, strerror ( rc ) ); + return rc; + } + + return 0; +} + +/** + * Stop endpoint + * + * @v xhci xHCI device + * @v slot Device slot + * @v endpoint Endpoint + * @ret rc Return status code + */ +static inline int xhci_stop_endpoint ( struct xhci_device *xhci, + struct xhci_slot *slot, + struct xhci_endpoint *endpoint ) { + union xhci_trb trb; + struct xhci_trb_stop_endpoint *stop = &trb.stop; + int rc; + + /* Construct command */ + memset ( stop, 0, sizeof ( *stop ) ); + stop->slot = slot->id; + stop->endpoint = endpoint->ctx; + stop->type = XHCI_TRB_STOP_ENDPOINT; + + /* Issue command and wait for completion */ + if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) { + DBGC ( xhci, "XHCI %s slot %d ctx %d could not stop endpoint " + "in state %d: %s\n", xhci->name, slot->id, endpoint->ctx, + endpoint->context->state, strerror ( rc ) ); + return rc; + } + + return 0; +} + +/** + * Set transfer ring dequeue pointer + * + * @v xhci xHCI device + * @v slot Device slot + * @v endpoint Endpoint + * @ret rc Return status code + */ +static inline int +xhci_set_tr_dequeue_pointer ( struct xhci_device *xhci, + struct xhci_slot *slot, + struct xhci_endpoint *endpoint ) { + union xhci_trb trb; + struct xhci_trb_set_tr_dequeue_pointer *dequeue = &trb.dequeue; + struct xhci_trb_ring *ring = &endpoint->ring; + unsigned int cons; + unsigned int mask; + unsigned int index; + unsigned int dcs; + int rc; + + /* Construct command */ + memset ( dequeue, 0, sizeof ( *dequeue ) ); + cons = ring->cons; + mask = ring->mask; + dcs = ( ( ~( cons >> ring->shift ) ) & XHCI_EP_DCS ); + index = ( cons & mask ); + dequeue->dequeue = + cpu_to_le64 ( virt_to_phys ( &ring->trb[index] ) | dcs ); + dequeue->slot = slot->id; + dequeue->endpoint = endpoint->ctx; + dequeue->type = XHCI_TRB_SET_TR_DEQUEUE_POINTER; + + /* Issue command and wait for completion */ + if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) { + DBGC ( xhci, "XHCI %s slot %d ctx %d could not set TR dequeue " + "pointer in state %d: %s\n", xhci->name, slot->id, + endpoint->ctx, endpoint->context->state, strerror ( rc)); + return rc; + } + + return 0; +} + +/****************************************************************************** + * + * Endpoint operations + * + ****************************************************************************** + */ + +/** + * Open endpoint + * + * @v ep USB endpoint + * @ret rc Return status code + */ +static int xhci_endpoint_open ( struct usb_endpoint *ep ) { + struct usb_device *usb = ep->usb; + struct xhci_slot *slot = usb_get_hostdata ( usb ); + struct xhci_device *xhci = slot->xhci; + struct xhci_endpoint *endpoint; + unsigned int ctx; + unsigned int type; + unsigned int interval; + int rc; + + /* Calculate context index */ + ctx = XHCI_CTX ( ep->address ); + assert ( slot->endpoint[ctx] == NULL ); + + /* Calculate endpoint type */ + type = XHCI_EP_TYPE ( ep->attributes & USB_ENDPOINT_ATTR_TYPE_MASK ); + if ( type == XHCI_EP_TYPE ( USB_ENDPOINT_ATTR_CONTROL ) ) + type = XHCI_EP_TYPE_CONTROL; + if ( ep->address & USB_DIR_IN ) + type |= XHCI_EP_TYPE_IN; + + /* Calculate interval */ + if ( type & XHCI_EP_TYPE_PERIODIC ) { + interval = ( fls ( ep->interval ) - 1 ); + } else { + interval = ep->interval; + } + + /* Allocate and initialise structure */ + endpoint = zalloc ( sizeof ( *endpoint ) ); + if ( ! endpoint ) { + rc = -ENOMEM; + goto err_alloc; + } + usb_endpoint_set_hostdata ( ep, endpoint ); + slot->endpoint[ctx] = endpoint; + endpoint->xhci = xhci; + endpoint->slot = slot; + endpoint->ep = ep; + endpoint->ctx = ctx; + endpoint->type = type; + endpoint->interval = interval; + endpoint->context = ( ( ( void * ) slot->context ) + + xhci_device_context_offset ( xhci, ctx ) ); + + /* Allocate transfer ring */ + if ( ( rc = xhci_ring_alloc ( xhci, &endpoint->ring, + XHCI_TRANSFER_TRBS_LOG2, + slot->id, ctx, 0 ) ) != 0 ) + goto err_ring_alloc; + + /* Configure endpoint, if applicable */ + if ( ( ctx != XHCI_CTX_EP0 ) && + ( ( rc = xhci_configure_endpoint ( xhci, slot, endpoint ) ) != 0 )) + goto err_configure_endpoint; + + DBGC2 ( xhci, "XHCI %s slot %d ctx %d ring [%08lx,%08lx)\n", + xhci->name, slot->id, ctx, virt_to_phys ( endpoint->ring.trb ), + ( virt_to_phys ( endpoint->ring.trb ) + endpoint->ring.len ) ); + return 0; + + xhci_deconfigure_endpoint ( xhci, slot, endpoint ); + err_configure_endpoint: + xhci_ring_free ( &endpoint->ring ); + err_ring_alloc: + slot->endpoint[ctx] = NULL; + free ( endpoint ); + err_alloc: + return rc; +} + +/** + * Close endpoint + * + * @v ep USB endpoint + */ +static void xhci_endpoint_close ( struct usb_endpoint *ep ) { + struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep ); + struct xhci_slot *slot = endpoint->slot; + struct xhci_device *xhci = slot->xhci; + struct io_buffer *iobuf; + unsigned int ctx = endpoint->ctx; + + /* Deconfigure endpoint, if applicable */ + if ( ctx != XHCI_CTX_EP0 ) + xhci_deconfigure_endpoint ( xhci, slot, endpoint ); + + /* Cancel any incomplete transfers */ + while ( xhci_ring_fill ( &endpoint->ring ) ) { + iobuf = xhci_dequeue_multi ( &endpoint->ring ); + usb_complete_err ( ep, iobuf, -ECANCELED ); + } + + /* Free endpoint */ + xhci_ring_free ( &endpoint->ring ); + slot->endpoint[ctx] = NULL; + free ( endpoint ); +} + +/** + * Reset endpoint + * + * @v ep USB endpoint + * @ret rc Return status code + */ +static int xhci_endpoint_reset ( struct usb_endpoint *ep ) { + struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep ); + struct xhci_slot *slot = endpoint->slot; + struct xhci_device *xhci = slot->xhci; + int rc; + + /* Reset endpoint context */ + if ( ( rc = xhci_reset_endpoint ( xhci, slot, endpoint ) ) != 0 ) + return rc; + + /* Set transfer ring dequeue pointer */ + if ( ( rc = xhci_set_tr_dequeue_pointer ( xhci, slot, endpoint ) ) != 0) + return rc; + + /* Ring doorbell to resume processing */ + xhci_doorbell ( &endpoint->ring ); + + DBGC ( xhci, "XHCI %s slot %d ctx %d reset\n", + xhci->name, slot->id, endpoint->ctx ); + return 0; +} + +/** + * Update MTU + * + * @v ep USB endpoint + * @ret rc Return status code + */ +static int xhci_endpoint_mtu ( struct usb_endpoint *ep ) { + struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep ); + struct xhci_slot *slot = endpoint->slot; + struct xhci_device *xhci = slot->xhci; + int rc; + + /* Evalulate context */ + if ( ( rc = xhci_evaluate_context ( xhci, slot, endpoint ) ) != 0 ) + return rc; + + return 0; +} + +/** + * Enqueue message transfer + * + * @v ep USB endpoint + * @v iobuf I/O buffer + * @ret rc Return status code + */ +static int xhci_endpoint_message ( struct usb_endpoint *ep, + struct io_buffer *iobuf ) { + struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep ); + struct usb_setup_packet *packet; + unsigned int input; + size_t len; + union xhci_trb trbs[ 1 /* setup */ + 1 /* possible data */ + + 1 /* status */ ]; + union xhci_trb *trb = trbs; + struct xhci_trb_setup *setup; + struct xhci_trb_data *data; + struct xhci_trb_status *status; + int rc; + + /* Profile message transfers */ + profile_start ( &xhci_message_profiler ); + + /* Construct setup stage TRB */ + memset ( trbs, 0, sizeof ( trbs ) ); + assert ( iob_len ( iobuf ) >= sizeof ( *packet ) ); + packet = iobuf->data; + iob_pull ( iobuf, sizeof ( *packet ) ); + setup = &(trb++)->setup; + memcpy ( &setup->packet, packet, sizeof ( setup->packet ) ); + setup->len = cpu_to_le32 ( sizeof ( *packet ) ); + setup->flags = XHCI_TRB_IDT; + setup->type = XHCI_TRB_SETUP; + len = iob_len ( iobuf ); + input = ( packet->request & cpu_to_le16 ( USB_DIR_IN ) ); + if ( len ) + setup->direction = ( input ? XHCI_SETUP_IN : XHCI_SETUP_OUT ); + + /* Construct data stage TRB, if applicable */ + if ( len ) { + data = &(trb++)->data; + data->data = cpu_to_le64 ( virt_to_phys ( iobuf->data ) ); + data->len = cpu_to_le32 ( len ); + data->type = XHCI_TRB_DATA; + data->direction = ( input ? XHCI_DATA_IN : XHCI_DATA_OUT ); + } + + /* Construct status stage TRB */ + status = &(trb++)->status; + status->flags = XHCI_TRB_IOC; + status->type = XHCI_TRB_STATUS; + status->direction = + ( ( len && input ) ? XHCI_STATUS_OUT : XHCI_STATUS_IN ); + + /* Enqueue TRBs */ + if ( ( rc = xhci_enqueue_multi ( &endpoint->ring, iobuf, trbs, + ( trb - trbs ) ) ) != 0 ) + return rc; + + /* Ring the doorbell */ + xhci_doorbell ( &endpoint->ring ); + + profile_stop ( &xhci_message_profiler ); + return 0; +} + +/** + * Enqueue stream transfer + * + * @v ep USB endpoint + * @v iobuf I/O buffer + * @v terminate Terminate using a short packet + * @ret rc Return status code + */ +static int xhci_endpoint_stream ( struct usb_endpoint *ep, + struct io_buffer *iobuf, int terminate ) { + struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep ); + union xhci_trb trbs[ 1 /* Normal */ + 1 /* Possible zero-length */ ]; + union xhci_trb *trb = trbs; + struct xhci_trb_normal *normal; + size_t len = iob_len ( iobuf ); + int rc; + + /* Profile stream transfers */ + profile_start ( &xhci_stream_profiler ); + + /* Construct normal TRBs */ + memset ( &trbs, 0, sizeof ( trbs ) ); + normal = &(trb++)->normal; + normal->data = cpu_to_le64 ( virt_to_phys ( iobuf->data ) ); + normal->len = cpu_to_le32 ( len ); + normal->type = XHCI_TRB_NORMAL; + if ( terminate && ( ( len & ( ep->mtu - 1 ) ) == 0 ) ) { + normal->flags = XHCI_TRB_CH; + normal = &(trb++)->normal; + normal->type = XHCI_TRB_NORMAL; + } + normal->flags = XHCI_TRB_IOC; + + /* Enqueue TRBs */ + if ( ( rc = xhci_enqueue_multi ( &endpoint->ring, iobuf, trbs, + ( trb - trbs ) ) ) != 0 ) + return rc; + + /* Ring the doorbell */ + xhci_doorbell ( &endpoint->ring ); + + profile_stop ( &xhci_stream_profiler ); + return 0; +} + +/****************************************************************************** + * + * Device operations + * + ****************************************************************************** + */ + +/** + * Open device + * + * @v usb USB device + * @ret rc Return status code + */ +static int xhci_device_open ( struct usb_device *usb ) { + struct xhci_device *xhci = usb_bus_get_hostdata ( usb->port->hub->bus ); + struct usb_port *tt = usb_transaction_translator ( usb ); + struct xhci_slot *slot; + struct xhci_slot *tt_slot; + size_t len; + int type; + int id; + int rc; + + /* Determine applicable slot type */ + type = xhci_port_slot_type ( xhci, usb->port->address ); + if ( type < 0 ) { + rc = type; + DBGC ( xhci, "XHCI %s-%d has no slot type\n", + xhci->name, usb->port->address ); + goto err_type; + } + + /* Allocate a device slot number */ + id = xhci_enable_slot ( xhci, type ); + if ( id < 0 ) { + rc = id; + goto err_enable_slot; + } + assert ( ( id > 0 ) && ( ( unsigned int ) id <= xhci->slots ) ); + assert ( xhci->slot[id] == NULL ); + + /* Allocate and initialise structure */ + slot = zalloc ( sizeof ( *slot ) ); + if ( ! slot ) { + rc = -ENOMEM; + goto err_alloc; + } + usb_set_hostdata ( usb, slot ); + xhci->slot[id] = slot; + slot->xhci = xhci; + slot->usb = usb; + slot->id = id; + if ( tt ) { + tt_slot = usb_get_hostdata ( tt->hub->usb ); + slot->tt_id = tt_slot->id; + slot->tt_port = tt->address; + } + + /* Allocate a device context */ + len = xhci_device_context_offset ( xhci, XHCI_CTX_END ); + slot->context = malloc_dma ( len, xhci_align ( len ) ); + if ( ! slot->context ) { + rc = -ENOMEM; + goto err_alloc_context; + } + memset ( slot->context, 0, len ); + + /* Set device context base address */ + assert ( xhci->dcbaa[id] == 0 ); + xhci->dcbaa[id] = cpu_to_le64 ( virt_to_phys ( slot->context ) ); + + DBGC2 ( xhci, "XHCI %s slot %d device context [%08lx,%08lx) for %s\n", + xhci->name, slot->id, virt_to_phys ( slot->context ), + ( virt_to_phys ( slot->context ) + len ), usb->name ); + return 0; + + xhci->dcbaa[id] = 0; + free_dma ( slot->context, len ); + err_alloc_context: + xhci->slot[id] = NULL; + free ( slot ); + err_alloc: + xhci_disable_slot ( xhci, id ); + err_enable_slot: + err_type: + return rc; +} + +/** + * Close device + * + * @v usb USB device + */ +static void xhci_device_close ( struct usb_device *usb ) { + struct xhci_slot *slot = usb_get_hostdata ( usb ); + struct xhci_device *xhci = slot->xhci; + size_t len = xhci_device_context_offset ( xhci, XHCI_CTX_END ); + unsigned int id = slot->id; + int rc; + + /* Disable slot */ + if ( ( rc = xhci_disable_slot ( xhci, id ) ) != 0 ) { + /* Slot is still enabled. Leak the slot context, + * since the controller may still write to this + * memory, and leave the DCBAA entry intact. + * + * If the controller later reports that this same slot + * has been re-enabled, then some assertions will be + * triggered. + */ + DBGC ( xhci, "XHCI %s slot %d leaking context memory\n", + xhci->name, slot->id ); + slot->context = NULL; + } + + /* Free slot */ + if ( slot->context ) { + free_dma ( slot->context, len ); + xhci->dcbaa[id] = 0; + } + xhci->slot[id] = NULL; + free ( slot ); +} + +/** + * Assign device address + * + * @v usb USB device + * @ret rc Return status code + */ +static int xhci_device_address ( struct usb_device *usb ) { + struct xhci_slot *slot = usb_get_hostdata ( usb ); + struct xhci_device *xhci = slot->xhci; + struct usb_port *port = usb->port; + struct usb_port *root_port; + int psiv; + int rc; + + /* Calculate route string */ + slot->route = usb_route_string ( usb ); + + /* Calculate root hub port number */ + root_port = usb_root_hub_port ( usb ); + slot->port = root_port->address; + + /* Calculate protocol speed ID */ + psiv = xhci_port_psiv ( xhci, slot->port, port->speed ); + if ( psiv < 0 ) { + rc = psiv; + return rc; + } + slot->psiv = psiv; + + /* Address device */ + if ( ( rc = xhci_address_device ( xhci, slot ) ) != 0 ) + return rc; + + return 0; +} + +/****************************************************************************** + * + * Bus operations + * + ****************************************************************************** + */ + +/** + * Open USB bus + * + * @v bus USB bus + * @ret rc Return status code + */ +static int xhci_bus_open ( struct usb_bus *bus ) { + struct xhci_device *xhci = usb_bus_get_hostdata ( bus ); + int rc; + + /* Allocate device slot array */ + xhci->slot = zalloc ( ( xhci->slots + 1 ) * sizeof ( xhci->slot[0] ) ); + if ( ! xhci->slot ) { + rc = -ENOMEM; + goto err_slot_alloc; + } + + /* Allocate device context base address array */ + if ( ( rc = xhci_dcbaa_alloc ( xhci ) ) != 0 ) + goto err_dcbaa_alloc; + + /* Allocate scratchpad buffers */ + if ( ( rc = xhci_scratchpad_alloc ( xhci ) ) != 0 ) + goto err_scratchpad_alloc; + + /* Allocate command ring */ + if ( ( rc = xhci_command_alloc ( xhci ) ) != 0 ) + goto err_command_alloc; + + /* Allocate event ring */ + if ( ( rc = xhci_event_alloc ( xhci ) ) != 0 ) + goto err_event_alloc; + + /* Start controller */ + xhci_run ( xhci ); + + return 0; + + xhci_stop ( xhci ); + xhci_event_free ( xhci ); + err_event_alloc: + xhci_command_free ( xhci ); + err_command_alloc: + xhci_scratchpad_free ( xhci ); + err_scratchpad_alloc: + xhci_dcbaa_free ( xhci ); + err_dcbaa_alloc: + free ( xhci->slot ); + err_slot_alloc: + return rc; +} + +/** + * Close USB bus + * + * @v bus USB bus + */ +static void xhci_bus_close ( struct usb_bus *bus ) { + struct xhci_device *xhci = usb_bus_get_hostdata ( bus ); + unsigned int i; + + /* Sanity checks */ + assert ( xhci->slot != NULL ); + for ( i = 0 ; i <= xhci->slots ; i++ ) + assert ( xhci->slot[i] == NULL ); + + xhci_stop ( xhci ); + xhci_event_free ( xhci ); + xhci_command_free ( xhci ); + xhci_scratchpad_free ( xhci ); + xhci_dcbaa_free ( xhci ); + free ( xhci->slot ); +} + +/** + * Poll USB bus + * + * @v bus USB bus + */ +static void xhci_bus_poll ( struct usb_bus *bus ) { + struct xhci_device *xhci = usb_bus_get_hostdata ( bus ); + + /* Poll event ring */ + xhci_event_poll ( xhci ); +} + +/****************************************************************************** + * + * Hub operations + * + ****************************************************************************** + */ + +/** + * Open hub + * + * @v hub USB hub + * @ret rc Return status code + */ +static int xhci_hub_open ( struct usb_hub *hub ) { + struct xhci_slot *slot; + + /* Do nothing if this is the root hub */ + if ( ! hub->usb ) + return 0; + + /* Get device slot */ + slot = usb_get_hostdata ( hub->usb ); + + /* Update device slot hub parameters. We don't inform the + * hardware of this information until the hub's interrupt + * endpoint is opened, since the only mechanism for so doing + * provided by the xHCI specification is a Configure Endpoint + * command, and we can't issue that command until we have a + * non-EP0 endpoint to configure. + */ + slot->ports = hub->ports; + + return 0; +} + +/** + * Close hub + * + * @v hub USB hub + */ +static void xhci_hub_close ( struct usb_hub *hub __unused ) { + + /* Nothing to do */ +} + +/****************************************************************************** + * + * Root hub operations + * + ****************************************************************************** + */ + +/** + * Open root hub + * + * @v hub USB hub + * @ret rc Return status code + */ +static int xhci_root_open ( struct usb_hub *hub ) { + struct usb_bus *bus = hub->bus; + struct xhci_device *xhci = usb_bus_get_hostdata ( bus ); + struct usb_port *port; + uint32_t portsc; + unsigned int i; + + /* Enable power to all ports */ + for ( i = 1 ; i <= xhci->ports ; i++ ) { + portsc = readl ( xhci->op + XHCI_OP_PORTSC ( i ) ); + portsc &= XHCI_PORTSC_PRESERVE; + portsc |= XHCI_PORTSC_PP; + writel ( portsc, xhci->op + XHCI_OP_PORTSC ( i ) ); + } + + /* xHCI spec requires us to potentially wait 20ms after + * enabling power to a port. + */ + mdelay ( XHCI_PORT_POWER_DELAY_MS ); + + /* USB3 ports may power up as Disabled */ + for ( i = 1 ; i <= xhci->ports ; i++ ) { + portsc = readl ( xhci->op + XHCI_OP_PORTSC ( i ) ); + port = usb_port ( hub, i ); + if ( ( port->protocol >= USB_PROTO_3_0 ) && + ( ( portsc & XHCI_PORTSC_PLS_MASK ) == + XHCI_PORTSC_PLS_DISABLED ) ) { + /* Force link state to RxDetect */ + portsc &= XHCI_PORTSC_PRESERVE; + portsc |= ( XHCI_PORTSC_PLS_RXDETECT | XHCI_PORTSC_LWS); + writel ( portsc, xhci->op + XHCI_OP_PORTSC ( i ) ); + } + } + + /* Some xHCI cards seem to require an additional delay after + * setting the link state to RxDetect. + */ + mdelay ( XHCI_LINK_STATE_DELAY_MS ); + + /* Record hub driver private data */ + usb_hub_set_drvdata ( hub, xhci ); + + return 0; +} + +/** + * Close root hub + * + * @v hub USB hub + */ +static void xhci_root_close ( struct usb_hub *hub ) { + + /* Clear hub driver private data */ + usb_hub_set_drvdata ( hub, NULL ); +} + +/** + * Enable port + * + * @v hub USB hub + * @v port USB port + * @ret rc Return status code + */ +static int xhci_root_enable ( struct usb_hub *hub, struct usb_port *port ) { + struct xhci_device *xhci = usb_hub_get_drvdata ( hub ); + uint32_t portsc; + unsigned int i; + + /* Reset port */ + portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) ); + portsc &= XHCI_PORTSC_PRESERVE; + portsc |= XHCI_PORTSC_PR; + writel ( portsc, xhci->op + XHCI_OP_PORTSC ( port->address ) ); + + /* Wait for port to become enabled */ + for ( i = 0 ; i < XHCI_PORT_RESET_MAX_WAIT_MS ; i++ ) { + + /* Check port status */ + portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) ); + if ( portsc & XHCI_PORTSC_PED ) + return 0; + + /* Delay */ + mdelay ( 1 ); + } + + DBGC ( xhci, "XHCI %s-%d timed out waiting for port to enable\n", + xhci->name, port->address ); + return -ETIMEDOUT; +} + +/** + * Disable port + * + * @v hub USB hub + * @v port USB port + * @ret rc Return status code + */ +static int xhci_root_disable ( struct usb_hub *hub, struct usb_port *port ) { + struct xhci_device *xhci = usb_hub_get_drvdata ( hub ); + uint32_t portsc; + + /* Disable port */ + portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) ); + portsc &= XHCI_PORTSC_PRESERVE; + portsc |= XHCI_PORTSC_PED; + writel ( portsc, xhci->op + XHCI_OP_PORTSC ( port->address ) ); + + return 0; +} + +/** + * Update root hub port speed + * + * @v hub USB hub + * @v port USB port + * @ret rc Return status code + */ +static int xhci_root_speed ( struct usb_hub *hub, struct usb_port *port ) { + struct xhci_device *xhci = usb_hub_get_drvdata ( hub ); + uint32_t portsc; + unsigned int psiv; + int ccs; + int ped; + int csc; + int speed; + int rc; + + /* Read port status */ + portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) ); + DBGC2 ( xhci, "XHCI %s-%d status is %08x\n", + xhci->name, port->address, portsc ); + ccs = ( portsc & XHCI_PORTSC_CCS ); + ped = ( portsc & XHCI_PORTSC_PED ); + csc = ( portsc & XHCI_PORTSC_CSC ); + psiv = XHCI_PORTSC_PSIV ( portsc ); + + /* Record disconnections and clear changes */ + port->disconnected |= csc; + portsc &= ( XHCI_PORTSC_PRESERVE | XHCI_PORTSC_CHANGE ); + writel ( portsc, xhci->op + XHCI_OP_PORTSC ( port->address ) ); + + /* Port speed is not valid unless port is connected */ + if ( ! ccs ) { + port->speed = USB_SPEED_NONE; + return 0; + } + + /* For USB2 ports, the PSIV field is not valid until the port + * completes reset and becomes enabled. + */ + if ( ( port->protocol < USB_PROTO_3_0 ) && ! ped ) { + port->speed = USB_SPEED_FULL; + return 0; + } + + /* Get port speed and map to generic USB speed */ + speed = xhci_port_speed ( xhci, port->address, psiv ); + if ( speed < 0 ) { + rc = speed; + return rc; + } + + port->speed = speed; + return 0; +} + +/** + * Clear transaction translator buffer + * + * @v hub USB hub + * @v port USB port + * @v ep USB endpoint + * @ret rc Return status code + */ +static int xhci_root_clear_tt ( struct usb_hub *hub, struct usb_port *port, + struct usb_endpoint *ep ) { + struct xhci_device *xhci = usb_hub_get_drvdata ( hub ); + + /* Should never be called; this is a root hub */ + DBGC ( xhci, "XHCI %s-%d nonsensical CLEAR_TT for %s %s\n", xhci->name, + port->address, ep->usb->name, usb_endpoint_name ( ep ) ); + + return -ENOTSUP; +} + +/****************************************************************************** + * + * PCI interface + * + ****************************************************************************** + */ + +/** USB host controller operations */ +static struct usb_host_operations xhci_operations = { + .endpoint = { + .open = xhci_endpoint_open, + .close = xhci_endpoint_close, + .reset = xhci_endpoint_reset, + .mtu = xhci_endpoint_mtu, + .message = xhci_endpoint_message, + .stream = xhci_endpoint_stream, + }, + .device = { + .open = xhci_device_open, + .close = xhci_device_close, + .address = xhci_device_address, + }, + .bus = { + .open = xhci_bus_open, + .close = xhci_bus_close, + .poll = xhci_bus_poll, + }, + .hub = { + .open = xhci_hub_open, + .close = xhci_hub_close, + }, + .root = { + .open = xhci_root_open, + .close = xhci_root_close, + .enable = xhci_root_enable, + .disable = xhci_root_disable, + .speed = xhci_root_speed, + .clear_tt = xhci_root_clear_tt, + }, +}; + +/** + * Fix Intel PCH-specific quirks + * + * @v xhci xHCI device + * @v pci PCI device + */ +static void xhci_pch_fix ( struct xhci_device *xhci, struct pci_device *pci ) { + struct xhci_pch *pch = &xhci->pch; + uint32_t xusb2pr; + uint32_t xusb2prm; + uint32_t usb3pssen; + uint32_t usb3prm; + + /* Enable SuperSpeed capability. Do this before rerouting + * USB2 ports, so that USB3 devices connect at SuperSpeed. + */ + pci_read_config_dword ( pci, XHCI_PCH_USB3PSSEN, &usb3pssen ); + pci_read_config_dword ( pci, XHCI_PCH_USB3PRM, &usb3prm ); + if ( usb3prm & ~usb3pssen ) { + DBGC ( xhci, "XHCI %s enabling SuperSpeed on ports %08x\n", + xhci->name, ( usb3prm & ~usb3pssen ) ); + } + pch->usb3pssen = usb3pssen; + usb3pssen |= usb3prm; + pci_write_config_dword ( pci, XHCI_PCH_USB3PSSEN, usb3pssen ); + + /* Route USB2 ports from EHCI to xHCI */ + pci_read_config_dword ( pci, XHCI_PCH_XUSB2PR, &xusb2pr ); + pci_read_config_dword ( pci, XHCI_PCH_XUSB2PRM, &xusb2prm ); + if ( xusb2prm & ~xusb2pr ) { + DBGC ( xhci, "XHCI %s routing ports %08x from EHCI to xHCI\n", + xhci->name, ( xusb2prm & ~xusb2pr ) ); + } + pch->xusb2pr = xusb2pr; + xusb2pr |= xusb2prm; + pci_write_config_dword ( pci, XHCI_PCH_XUSB2PR, xusb2pr ); +} + +/** + * Undo Intel PCH-specific quirk fixes + * + * @v xhci xHCI device + * @v pci PCI device + */ +static void xhci_pch_undo ( struct xhci_device *xhci, struct pci_device *pci ) { + struct xhci_pch *pch = &xhci->pch; + + /* Restore USB2 port routing to original state */ + pci_write_config_dword ( pci, XHCI_PCH_XUSB2PR, pch->xusb2pr ); + + /* Restore SuperSpeed capability to original state */ + pci_write_config_dword ( pci, XHCI_PCH_USB3PSSEN, pch->usb3pssen ); +} + +/** + * Probe PCI device + * + * @v pci PCI device + * @ret rc Return status code + */ +static int xhci_probe ( struct pci_device *pci ) { + struct xhci_device *xhci; + struct usb_port *port; + unsigned long bar_start; + size_t bar_size; + unsigned int i; + int rc; + + /* Allocate and initialise structure */ + xhci = zalloc ( sizeof ( *xhci ) ); + if ( ! xhci ) { + rc = -ENOMEM; + goto err_alloc; + } + xhci->name = pci->dev.name; + xhci->quirks = pci->id->driver_data; + + /* Fix up PCI device */ + adjust_pci_device ( pci ); + + /* Map registers */ + bar_start = pci_bar_start ( pci, XHCI_BAR ); + bar_size = pci_bar_size ( pci, XHCI_BAR ); + xhci->regs = ioremap ( bar_start, bar_size ); + if ( ! xhci->regs ) { + rc = -ENODEV; + goto err_ioremap; + } + + /* Initialise xHCI device */ + xhci_init ( xhci, xhci->regs ); + + /* Initialise USB legacy support and claim ownership */ + xhci_legacy_init ( xhci ); + xhci_legacy_claim ( xhci ); + + /* Fix Intel PCH-specific quirks, if applicable */ + if ( xhci->quirks & XHCI_PCH ) + xhci_pch_fix ( xhci, pci ); + + /* Reset device */ + if ( ( rc = xhci_reset ( xhci ) ) != 0 ) + goto err_reset; + + /* Allocate USB bus */ + xhci->bus = alloc_usb_bus ( &pci->dev, xhci->ports, XHCI_MTU, + &xhci_operations ); + if ( ! xhci->bus ) { + rc = -ENOMEM; + goto err_alloc_bus; + } + usb_bus_set_hostdata ( xhci->bus, xhci ); + usb_hub_set_drvdata ( xhci->bus->hub, xhci ); + + /* Set port protocols */ + for ( i = 1 ; i <= xhci->ports ; i++ ) { + port = usb_port ( xhci->bus->hub, i ); + port->protocol = xhci_port_protocol ( xhci, i ); + } + + /* Register USB bus */ + if ( ( rc = register_usb_bus ( xhci->bus ) ) != 0 ) + goto err_register; + + pci_set_drvdata ( pci, xhci ); + return 0; + + unregister_usb_bus ( xhci->bus ); + err_register: + free_usb_bus ( xhci->bus ); + err_alloc_bus: + xhci_reset ( xhci ); + err_reset: + if ( xhci->quirks & XHCI_PCH ) + xhci_pch_undo ( xhci, pci ); + xhci_legacy_release ( xhci ); + iounmap ( xhci->regs ); + err_ioremap: + free ( xhci ); + err_alloc: + return rc; +} + +/** + * Remove PCI device + * + * @v pci PCI device + */ +static void xhci_remove ( struct pci_device *pci ) { + struct xhci_device *xhci = pci_get_drvdata ( pci ); + struct usb_bus *bus = xhci->bus; + + unregister_usb_bus ( bus ); + free_usb_bus ( bus ); + xhci_reset ( xhci ); + if ( xhci->quirks & XHCI_PCH ) + xhci_pch_undo ( xhci, pci ); + xhci_legacy_release ( xhci ); + iounmap ( xhci->regs ); + free ( xhci ); +} + +/** XHCI PCI device IDs */ +static struct pci_device_id xhci_ids[] = { + PCI_ROM ( 0x8086, 0x9d2f, "xhci-skylake", "xHCI (Skylake)", ( XHCI_PCH | XHCI_BAD_PSIV ) ), + PCI_ROM ( 0x8086, 0xffff, "xhci-pch", "xHCI (Intel PCH)", XHCI_PCH ), + PCI_ROM ( 0xffff, 0xffff, "xhci", "xHCI", 0 ), +}; + +/** XHCI PCI driver */ +struct pci_driver xhci_driver __pci_driver = { + .ids = xhci_ids, + .id_count = ( sizeof ( xhci_ids ) / sizeof ( xhci_ids[0] ) ), + .class = PCI_CLASS_ID ( PCI_CLASS_SERIAL, PCI_CLASS_SERIAL_USB, + PCI_CLASS_SERIAL_USB_XHCI ), + .probe = xhci_probe, + .remove = xhci_remove, +}; + +/** + * Prepare for exit + * + * @v booting System is shutting down for OS boot + */ +static void xhci_shutdown ( int booting ) { + /* If we are shutting down to boot an OS, then prevent the + * release of ownership back to BIOS. + */ + xhci_legacy_prevent_release = booting; +} + +/** Startup/shutdown function */ +struct startup_fn xhci_startup __startup_fn ( STARTUP_LATE ) = { + .shutdown = xhci_shutdown, +}; |