diff options
Diffstat (limited to 'qemu/roms/ipxe/src/drivers/usb/xhci.c')
| -rw-r--r-- | qemu/roms/ipxe/src/drivers/usb/xhci.c | 3321 |
1 files changed, 0 insertions, 3321 deletions
diff --git a/qemu/roms/ipxe/src/drivers/usb/xhci.c b/qemu/roms/ipxe/src/drivers/usb/xhci.c deleted file mode 100644 index 49e67316b..000000000 --- a/qemu/roms/ipxe/src/drivers/usb/xhci.c +++ /dev/null @@ -1,3321 +0,0 @@ -/* - * Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation; either version 2 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - * - * You can also choose to distribute this program under the terms of - * the Unmodified Binary Distribution Licence (as given in the file - * COPYING.UBDL), provided that you have satisfied its requirements. - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#include <stdlib.h> -#include <stdio.h> -#include <unistd.h> -#include <string.h> -#include <strings.h> -#include <errno.h> -#include <byteswap.h> -#include <ipxe/malloc.h> -#include <ipxe/umalloc.h> -#include <ipxe/pci.h> -#include <ipxe/usb.h> -#include <ipxe/init.h> -#include <ipxe/profile.h> -#include "xhci.h" - -/** @file - * - * USB eXtensible Host Controller Interface (xHCI) driver - * - */ - -/** Message transfer profiler */ -static struct profiler xhci_message_profiler __profiler = - { .name = "xhci.message" }; - -/** Stream transfer profiler */ -static struct profiler xhci_stream_profiler __profiler = - { .name = "xhci.stream" }; - -/** Event ring profiler */ -static struct profiler xhci_event_profiler __profiler = - { .name = "xhci.event" }; - -/** Transfer event profiler */ -static struct profiler xhci_transfer_profiler __profiler = - { .name = "xhci.transfer" }; - -/* Disambiguate the various error causes */ -#define EIO_DATA \ - __einfo_error ( EINFO_EIO_DATA ) -#define EINFO_EIO_DATA \ - __einfo_uniqify ( EINFO_EIO, ( 2 - 0 ), \ - "Data buffer error" ) -#define EIO_BABBLE \ - __einfo_error ( EINFO_EIO_BABBLE ) -#define EINFO_EIO_BABBLE \ - __einfo_uniqify ( EINFO_EIO, ( 3 - 0 ), \ - "Babble detected" ) -#define EIO_USB \ - __einfo_error ( EINFO_EIO_USB ) -#define EINFO_EIO_USB \ - __einfo_uniqify ( EINFO_EIO, ( 4 - 0 ), \ - "USB transaction error" ) -#define EIO_TRB \ - __einfo_error ( EINFO_EIO_TRB ) -#define EINFO_EIO_TRB \ - __einfo_uniqify ( EINFO_EIO, ( 5 - 0 ), \ - "TRB error" ) -#define EIO_STALL \ - __einfo_error ( EINFO_EIO_STALL ) -#define EINFO_EIO_STALL \ - __einfo_uniqify ( EINFO_EIO, ( 6 - 0 ), \ - "Stall error" ) -#define EIO_RESOURCE \ - __einfo_error ( EINFO_EIO_RESOURCE ) -#define EINFO_EIO_RESOURCE \ - __einfo_uniqify ( EINFO_EIO, ( 7 - 0 ), \ - "Resource error" ) -#define EIO_BANDWIDTH \ - __einfo_error ( EINFO_EIO_BANDWIDTH ) -#define EINFO_EIO_BANDWIDTH \ - __einfo_uniqify ( EINFO_EIO, ( 8 - 0 ), \ - "Bandwidth error" ) -#define EIO_NO_SLOTS \ - __einfo_error ( EINFO_EIO_NO_SLOTS ) -#define EINFO_EIO_NO_SLOTS \ - __einfo_uniqify ( EINFO_EIO, ( 9 - 0 ), \ - "No slots available" ) -#define EIO_STREAM_TYPE \ - __einfo_error ( EINFO_EIO_STREAM_TYPE ) -#define EINFO_EIO_STREAM_TYPE \ - __einfo_uniqify ( EINFO_EIO, ( 10 - 0 ), \ - "Invalid stream type" ) -#define EIO_SLOT \ - __einfo_error ( EINFO_EIO_SLOT ) -#define EINFO_EIO_SLOT \ - __einfo_uniqify ( EINFO_EIO, ( 11 - 0 ), \ - "Slot not enabled" ) -#define EIO_ENDPOINT \ - __einfo_error ( EINFO_EIO_ENDPOINT ) -#define EINFO_EIO_ENDPOINT \ - __einfo_uniqify ( EINFO_EIO, ( 12 - 0 ), \ - "Endpoint not enabled" ) -#define EIO_SHORT \ - __einfo_error ( EINFO_EIO_SHORT ) -#define EINFO_EIO_SHORT \ - __einfo_uniqify ( EINFO_EIO, ( 13 - 0 ), \ - "Short packet" ) -#define EIO_UNDERRUN \ - __einfo_error ( EINFO_EIO_UNDERRUN ) -#define EINFO_EIO_UNDERRUN \ - __einfo_uniqify ( EINFO_EIO, ( 14 - 0 ), \ - "Ring underrun" ) -#define EIO_OVERRUN \ - __einfo_error ( EINFO_EIO_OVERRUN ) -#define EINFO_EIO_OVERRUN \ - __einfo_uniqify ( EINFO_EIO, ( 15 - 0 ), \ - "Ring overrun" ) -#define EIO_VF_RING_FULL \ - __einfo_error ( EINFO_EIO_VF_RING_FULL ) -#define EINFO_EIO_VF_RING_FULL \ - __einfo_uniqify ( EINFO_EIO, ( 16 - 0 ), \ - "Virtual function event ring full" ) -#define EIO_PARAMETER \ - __einfo_error ( EINFO_EIO_PARAMETER ) -#define EINFO_EIO_PARAMETER \ - __einfo_uniqify ( EINFO_EIO, ( 17 - 0 ), \ - "Parameter error" ) -#define EIO_BANDWIDTH_OVERRUN \ - __einfo_error ( EINFO_EIO_BANDWIDTH_OVERRUN ) -#define EINFO_EIO_BANDWIDTH_OVERRUN \ - __einfo_uniqify ( EINFO_EIO, ( 18 - 0 ), \ - "Bandwidth overrun" ) -#define EIO_CONTEXT \ - __einfo_error ( EINFO_EIO_CONTEXT ) -#define EINFO_EIO_CONTEXT \ - __einfo_uniqify ( EINFO_EIO, ( 19 - 0 ), \ - "Context state error" ) -#define EIO_NO_PING \ - __einfo_error ( EINFO_EIO_NO_PING ) -#define EINFO_EIO_NO_PING \ - __einfo_uniqify ( EINFO_EIO, ( 20 - 0 ), \ - "No ping response" ) -#define EIO_RING_FULL \ - __einfo_error ( EINFO_EIO_RING_FULL ) -#define EINFO_EIO_RING_FULL \ - __einfo_uniqify ( EINFO_EIO, ( 21 - 0 ), \ - "Event ring full" ) -#define EIO_INCOMPATIBLE \ - __einfo_error ( EINFO_EIO_INCOMPATIBLE ) -#define EINFO_EIO_INCOMPATIBLE \ - __einfo_uniqify ( EINFO_EIO, ( 22 - 0 ), \ - "Incompatible device" ) -#define EIO_MISSED \ - __einfo_error ( EINFO_EIO_MISSED ) -#define EINFO_EIO_MISSED \ - __einfo_uniqify ( EINFO_EIO, ( 23 - 0 ), \ - "Missed service error" ) -#define EIO_CMD_STOPPED \ - __einfo_error ( EINFO_EIO_CMD_STOPPED ) -#define EINFO_EIO_CMD_STOPPED \ - __einfo_uniqify ( EINFO_EIO, ( 24 - 0 ), \ - "Command ring stopped" ) -#define EIO_CMD_ABORTED \ - __einfo_error ( EINFO_EIO_CMD_ABORTED ) -#define EINFO_EIO_CMD_ABORTED \ - __einfo_uniqify ( EINFO_EIO, ( 25 - 0 ), \ - "Command aborted" ) -#define EIO_STOP \ - __einfo_error ( EINFO_EIO_STOP ) -#define EINFO_EIO_STOP \ - __einfo_uniqify ( EINFO_EIO, ( 26 - 0 ), \ - "Stopped" ) -#define EIO_STOP_LEN \ - __einfo_error ( EINFO_EIO_STOP_LEN ) -#define EINFO_EIO_STOP_LEN \ - __einfo_uniqify ( EINFO_EIO, ( 27 - 0 ), \ - "Stopped - length invalid" ) -#define EIO_STOP_SHORT \ - __einfo_error ( EINFO_EIO_STOP_SHORT ) -#define EINFO_EIO_STOP_SHORT \ - __einfo_uniqify ( EINFO_EIO, ( 28 - 0 ), \ - "Stopped - short packet" ) -#define EIO_LATENCY \ - __einfo_error ( EINFO_EIO_LATENCY ) -#define EINFO_EIO_LATENCY \ - __einfo_uniqify ( EINFO_EIO, ( 29 - 0 ), \ - "Maximum exit latency too large" ) -#define EIO_ISOCH \ - __einfo_error ( EINFO_EIO_ISOCH ) -#define EINFO_EIO_ISOCH \ - __einfo_uniqify ( EINFO_EIO, ( 31 - 0 ), \ - "Isochronous buffer overrun" ) -#define EPROTO_LOST \ - __einfo_error ( EINFO_EPROTO_LOST ) -#define EINFO_EPROTO_LOST \ - __einfo_uniqify ( EINFO_EPROTO, ( 32 - 32 ), \ - "Event lost" ) -#define EPROTO_UNDEFINED \ - __einfo_error ( EINFO_EPROTO_UNDEFINED ) -#define EINFO_EPROTO_UNDEFINED \ - __einfo_uniqify ( EINFO_EPROTO, ( 33 - 32 ), \ - "Undefined error" ) -#define EPROTO_STREAM_ID \ - __einfo_error ( EINFO_EPROTO_STREAM_ID ) -#define EINFO_EPROTO_STREAM_ID \ - __einfo_uniqify ( EINFO_EPROTO, ( 34 - 32 ), \ - "Invalid stream ID" ) -#define EPROTO_SECONDARY \ - __einfo_error ( EINFO_EPROTO_SECONDARY ) -#define EINFO_EPROTO_SECONDARY \ - __einfo_uniqify ( EINFO_EPROTO, ( 35 - 32 ), \ - "Secondary bandwidth error" ) -#define EPROTO_SPLIT \ - __einfo_error ( EINFO_EPROTO_SPLIT ) -#define EINFO_EPROTO_SPLIT \ - __einfo_uniqify ( EINFO_EPROTO, ( 36 - 32 ), \ - "Split transaction error" ) -#define ECODE(code) \ - ( ( (code) < 32 ) ? \ - EUNIQ ( EINFO_EIO, ( (code) & 31 ), EIO_DATA, EIO_BABBLE, \ - EIO_USB, EIO_TRB, EIO_STALL, EIO_RESOURCE, \ - EIO_BANDWIDTH, EIO_NO_SLOTS, EIO_STREAM_TYPE, \ - EIO_SLOT, EIO_ENDPOINT, EIO_SHORT, EIO_UNDERRUN, \ - EIO_OVERRUN, EIO_VF_RING_FULL, EIO_PARAMETER, \ - EIO_BANDWIDTH_OVERRUN, EIO_CONTEXT, EIO_NO_PING, \ - EIO_RING_FULL, EIO_INCOMPATIBLE, EIO_MISSED, \ - EIO_CMD_STOPPED, EIO_CMD_ABORTED, EIO_STOP, \ - EIO_STOP_LEN, EIO_STOP_SHORT, EIO_LATENCY, \ - EIO_ISOCH ) : \ - ( (code) < 64 ) ? \ - EUNIQ ( EINFO_EPROTO, ( (code) & 31 ), EPROTO_LOST, \ - EPROTO_UNDEFINED, EPROTO_STREAM_ID, \ - EPROTO_SECONDARY, EPROTO_SPLIT ) : \ - EFAULT ) - -/****************************************************************************** - * - * Register access - * - ****************************************************************************** - */ - -/** - * Initialise device - * - * @v xhci xHCI device - * @v regs MMIO registers - */ -static void xhci_init ( struct xhci_device *xhci, void *regs ) { - uint32_t hcsparams1; - uint32_t hcsparams2; - uint32_t hccparams1; - uint32_t pagesize; - size_t caplength; - size_t rtsoff; - size_t dboff; - - /* Locate capability, operational, runtime, and doorbell registers */ - xhci->cap = regs; - caplength = readb ( xhci->cap + XHCI_CAP_CAPLENGTH ); - rtsoff = readl ( xhci->cap + XHCI_CAP_RTSOFF ); - dboff = readl ( xhci->cap + XHCI_CAP_DBOFF ); - xhci->op = ( xhci->cap + caplength ); - xhci->run = ( xhci->cap + rtsoff ); - xhci->db = ( xhci->cap + dboff ); - DBGC2 ( xhci, "XHCI %s cap %08lx op %08lx run %08lx db %08lx\n", - xhci->name, virt_to_phys ( xhci->cap ), - virt_to_phys ( xhci->op ), virt_to_phys ( xhci->run ), - virt_to_phys ( xhci->db ) ); - - /* Read structural parameters 1 */ - hcsparams1 = readl ( xhci->cap + XHCI_CAP_HCSPARAMS1 ); - xhci->slots = XHCI_HCSPARAMS1_SLOTS ( hcsparams1 ); - xhci->intrs = XHCI_HCSPARAMS1_INTRS ( hcsparams1 ); - xhci->ports = XHCI_HCSPARAMS1_PORTS ( hcsparams1 ); - DBGC ( xhci, "XHCI %s has %d slots %d intrs %d ports\n", - xhci->name, xhci->slots, xhci->intrs, xhci->ports ); - - /* Read structural parameters 2 */ - hcsparams2 = readl ( xhci->cap + XHCI_CAP_HCSPARAMS2 ); - xhci->scratchpads = XHCI_HCSPARAMS2_SCRATCHPADS ( hcsparams2 ); - DBGC2 ( xhci, "XHCI %s needs %d scratchpads\n", - xhci->name, xhci->scratchpads ); - - /* Read capability parameters 1 */ - hccparams1 = readl ( xhci->cap + XHCI_CAP_HCCPARAMS1 ); - xhci->addr64 = XHCI_HCCPARAMS1_ADDR64 ( hccparams1 ); - xhci->csz_shift = XHCI_HCCPARAMS1_CSZ_SHIFT ( hccparams1 ); - xhci->xecp = XHCI_HCCPARAMS1_XECP ( hccparams1 ); - - /* Read page size */ - pagesize = readl ( xhci->op + XHCI_OP_PAGESIZE ); - xhci->pagesize = XHCI_PAGESIZE ( pagesize ); - assert ( xhci->pagesize != 0 ); - assert ( ( ( xhci->pagesize ) & ( xhci->pagesize - 1 ) ) == 0 ); - DBGC2 ( xhci, "XHCI %s page size %zd bytes\n", - xhci->name, xhci->pagesize ); -} - -/** - * Find extended capability - * - * @v xhci xHCI device - * @v id Capability ID - * @v offset Offset to previous extended capability instance, or zero - * @ret offset Offset to extended capability, or zero if not found - */ -static unsigned int xhci_extended_capability ( struct xhci_device *xhci, - unsigned int id, - unsigned int offset ) { - uint32_t xecp; - unsigned int next; - - /* Locate the extended capability */ - while ( 1 ) { - - /* Locate first or next capability as applicable */ - if ( offset ) { - xecp = readl ( xhci->cap + offset ); - next = XHCI_XECP_NEXT ( xecp ); - } else { - next = xhci->xecp; - } - if ( ! next ) - return 0; - offset += next; - - /* Check if this is the requested capability */ - xecp = readl ( xhci->cap + offset ); - if ( XHCI_XECP_ID ( xecp ) == id ) - return offset; - } -} - -/** - * Write potentially 64-bit register - * - * @v xhci xHCI device - * @v value Value - * @v reg Register address - * @ret rc Return status code - */ -static inline __attribute__ (( always_inline )) int -xhci_writeq ( struct xhci_device *xhci, physaddr_t value, void *reg ) { - - /* If this is a 32-bit build, then this can never fail - * (allowing the compiler to optimise out the error path). - */ - if ( sizeof ( value ) <= sizeof ( uint32_t ) ) { - writel ( value, reg ); - writel ( 0, ( reg + sizeof ( uint32_t ) ) ); - return 0; - } - - /* If the device does not support 64-bit addresses and this - * address is outside the 32-bit address space, then fail. - */ - if ( ( value & ~0xffffffffULL ) && ! xhci->addr64 ) { - DBGC ( xhci, "XHCI %s cannot access address %lx\n", - xhci->name, value ); - return -ENOTSUP; - } - - /* If this is a 64-bit build, then writeq() is available */ - writeq ( value, reg ); - return 0; -} - -/** - * Calculate buffer alignment - * - * @v len Length - * @ret align Buffer alignment - * - * Determine alignment required for a buffer which must be aligned to - * at least XHCI_MIN_ALIGN and which must not cross a page boundary. - */ -static inline size_t xhci_align ( size_t len ) { - size_t align; - - /* Align to own length (rounded up to a power of two) */ - align = ( 1 << fls ( len - 1 ) ); - - /* Round up to XHCI_MIN_ALIGN if needed */ - if ( align < XHCI_MIN_ALIGN ) - align = XHCI_MIN_ALIGN; - - return align; -} - -/** - * Calculate device context offset - * - * @v xhci xHCI device - * @v ctx Context index - */ -static inline size_t xhci_device_context_offset ( struct xhci_device *xhci, - unsigned int ctx ) { - - return ( XHCI_DCI ( ctx ) << xhci->csz_shift ); -} - -/** - * Calculate input context offset - * - * @v xhci xHCI device - * @v ctx Context index - */ -static inline size_t xhci_input_context_offset ( struct xhci_device *xhci, - unsigned int ctx ) { - - return ( XHCI_ICI ( ctx ) << xhci->csz_shift ); -} - -/****************************************************************************** - * - * Diagnostics - * - ****************************************************************************** - */ - -/** - * Dump host controller registers - * - * @v xhci xHCI device - */ -static inline void xhci_dump ( struct xhci_device *xhci ) { - uint32_t usbcmd; - uint32_t usbsts; - uint32_t pagesize; - uint32_t dnctrl; - uint32_t config; - - /* Do nothing unless debugging is enabled */ - if ( ! DBG_LOG ) - return; - - /* Dump USBCMD */ - usbcmd = readl ( xhci->op + XHCI_OP_USBCMD ); - DBGC ( xhci, "XHCI %s USBCMD %08x%s%s\n", xhci->name, usbcmd, - ( ( usbcmd & XHCI_USBCMD_RUN ) ? " run" : "" ), - ( ( usbcmd & XHCI_USBCMD_HCRST ) ? " hcrst" : "" ) ); - - /* Dump USBSTS */ - usbsts = readl ( xhci->op + XHCI_OP_USBSTS ); - DBGC ( xhci, "XHCI %s USBSTS %08x%s\n", xhci->name, usbsts, - ( ( usbsts & XHCI_USBSTS_HCH ) ? " hch" : "" ) ); - - /* Dump PAGESIZE */ - pagesize = readl ( xhci->op + XHCI_OP_PAGESIZE ); - DBGC ( xhci, "XHCI %s PAGESIZE %08x\n", xhci->name, pagesize ); - - /* Dump DNCTRL */ - dnctrl = readl ( xhci->op + XHCI_OP_DNCTRL ); - DBGC ( xhci, "XHCI %s DNCTRL %08x\n", xhci->name, dnctrl ); - - /* Dump CONFIG */ - config = readl ( xhci->op + XHCI_OP_CONFIG ); - DBGC ( xhci, "XHCI %s CONFIG %08x\n", xhci->name, config ); -} - -/** - * Dump port registers - * - * @v xhci xHCI device - * @v port Port number - */ -static inline void xhci_dump_port ( struct xhci_device *xhci, - unsigned int port ) { - uint32_t portsc; - uint32_t portpmsc; - uint32_t portli; - uint32_t porthlpmc; - - /* Do nothing unless debugging is enabled */ - if ( ! DBG_LOG ) - return; - - /* Dump PORTSC */ - portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port ) ); - DBGC ( xhci, "XHCI %s-%d PORTSC %08x%s%s%s%s psiv=%d\n", - xhci->name, port, portsc, - ( ( portsc & XHCI_PORTSC_CCS ) ? " ccs" : "" ), - ( ( portsc & XHCI_PORTSC_PED ) ? " ped" : "" ), - ( ( portsc & XHCI_PORTSC_PR ) ? " pr" : "" ), - ( ( portsc & XHCI_PORTSC_PP ) ? " pp" : "" ), - XHCI_PORTSC_PSIV ( portsc ) ); - - /* Dump PORTPMSC */ - portpmsc = readl ( xhci->op + XHCI_OP_PORTPMSC ( port ) ); - DBGC ( xhci, "XHCI %s-%d PORTPMSC %08x\n", xhci->name, port, portpmsc ); - - /* Dump PORTLI */ - portli = readl ( xhci->op + XHCI_OP_PORTLI ( port ) ); - DBGC ( xhci, "XHCI %s-%d PORTLI %08x\n", xhci->name, port, portli ); - - /* Dump PORTHLPMC */ - porthlpmc = readl ( xhci->op + XHCI_OP_PORTHLPMC ( port ) ); - DBGC ( xhci, "XHCI %s-%d PORTHLPMC %08x\n", - xhci->name, port, porthlpmc ); -} - -/****************************************************************************** - * - * USB legacy support - * - ****************************************************************************** - */ - -/** Prevent the release of ownership back to BIOS */ -static int xhci_legacy_prevent_release; - -/** - * Initialise USB legacy support - * - * @v xhci xHCI device - */ -static void xhci_legacy_init ( struct xhci_device *xhci ) { - unsigned int legacy; - uint8_t bios; - - /* Locate USB legacy support capability (if present) */ - legacy = xhci_extended_capability ( xhci, XHCI_XECP_ID_LEGACY, 0 ); - if ( ! legacy ) { - /* Not an error; capability may not be present */ - DBGC ( xhci, "XHCI %s has no USB legacy support capability\n", - xhci->name ); - return; - } - - /* Check if legacy USB support is enabled */ - bios = readb ( xhci->cap + legacy + XHCI_USBLEGSUP_BIOS ); - if ( ! ( bios & XHCI_USBLEGSUP_BIOS_OWNED ) ) { - /* Not an error; already owned by OS */ - DBGC ( xhci, "XHCI %s USB legacy support already disabled\n", - xhci->name ); - return; - } - - /* Record presence of USB legacy support capability */ - xhci->legacy = legacy; -} - -/** - * Claim ownership from BIOS - * - * @v xhci xHCI device - */ -static void xhci_legacy_claim ( struct xhci_device *xhci ) { - uint32_t ctlsts; - uint8_t bios; - unsigned int i; - - /* Do nothing unless legacy support capability is present */ - if ( ! xhci->legacy ) - return; - - /* Claim ownership */ - writeb ( XHCI_USBLEGSUP_OS_OWNED, - xhci->cap + xhci->legacy + XHCI_USBLEGSUP_OS ); - - /* Wait for BIOS to release ownership */ - for ( i = 0 ; i < XHCI_USBLEGSUP_MAX_WAIT_MS ; i++ ) { - - /* Check if BIOS has released ownership */ - bios = readb ( xhci->cap + xhci->legacy + XHCI_USBLEGSUP_BIOS ); - if ( ! ( bios & XHCI_USBLEGSUP_BIOS_OWNED ) ) { - DBGC ( xhci, "XHCI %s claimed ownership from BIOS\n", - xhci->name ); - ctlsts = readl ( xhci->cap + xhci->legacy + - XHCI_USBLEGSUP_CTLSTS ); - if ( ctlsts ) { - DBGC ( xhci, "XHCI %s warning: BIOS retained " - "SMIs: %08x\n", xhci->name, ctlsts ); - } - return; - } - - /* Delay */ - mdelay ( 1 ); - } - - /* BIOS did not release ownership. Claim it forcibly by - * disabling all SMIs. - */ - DBGC ( xhci, "XHCI %s could not claim ownership from BIOS: forcibly " - "disabling SMIs\n", xhci->name ); - writel ( 0, xhci->cap + xhci->legacy + XHCI_USBLEGSUP_CTLSTS ); -} - -/** - * Release ownership back to BIOS - * - * @v xhci xHCI device - */ -static void xhci_legacy_release ( struct xhci_device *xhci ) { - - /* Do nothing unless legacy support capability is present */ - if ( ! xhci->legacy ) - return; - - /* Do nothing if releasing ownership is prevented */ - if ( xhci_legacy_prevent_release ) { - DBGC ( xhci, "XHCI %s not releasing ownership to BIOS\n", - xhci->name ); - return; - } - - /* Release ownership */ - writeb ( 0, xhci->cap + xhci->legacy + XHCI_USBLEGSUP_OS ); - DBGC ( xhci, "XHCI %s released ownership to BIOS\n", xhci->name ); -} - -/****************************************************************************** - * - * Supported protocols - * - ****************************************************************************** - */ - -/** - * Transcribe port speed (for debugging) - * - * @v psi Protocol speed ID - * @ret speed Transcribed speed - */ -static inline const char * xhci_speed_name ( uint32_t psi ) { - static const char *exponents[4] = { "", "k", "M", "G" }; - static char buf[ 10 /* "xxxxxXbps" + NUL */ ]; - unsigned int mantissa; - unsigned int exponent; - - /* Extract mantissa and exponent */ - mantissa = XHCI_SUPPORTED_PSI_MANTISSA ( psi ); - exponent = XHCI_SUPPORTED_PSI_EXPONENT ( psi ); - - /* Transcribe speed */ - snprintf ( buf, sizeof ( buf ), "%d%sbps", - mantissa, exponents[exponent] ); - return buf; -} - -/** - * Find supported protocol extended capability for a port - * - * @v xhci xHCI device - * @v port Port number - * @ret supported Offset to extended capability, or zero if not found - */ -static unsigned int xhci_supported_protocol ( struct xhci_device *xhci, - unsigned int port ) { - unsigned int supported = 0; - unsigned int offset; - unsigned int count; - uint32_t ports; - - /* Iterate over all supported protocol structures */ - while ( ( supported = xhci_extended_capability ( xhci, - XHCI_XECP_ID_SUPPORTED, - supported ) ) ) { - - /* Determine port range */ - ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS ); - offset = XHCI_SUPPORTED_PORTS_OFFSET ( ports ); - count = XHCI_SUPPORTED_PORTS_COUNT ( ports ); - - /* Check if port lies within this range */ - if ( ( port - offset ) < count ) - return supported; - } - - DBGC ( xhci, "XHCI %s-%d has no supported protocol\n", - xhci->name, port ); - return 0; -} - -/** - * Find port protocol - * - * @v xhci xHCI device - * @v port Port number - * @ret protocol USB protocol, or zero if not found - */ -static unsigned int xhci_port_protocol ( struct xhci_device *xhci, - unsigned int port ) { - unsigned int supported = xhci_supported_protocol ( xhci, port ); - union { - uint32_t raw; - char text[5]; - } name; - unsigned int protocol; - unsigned int type; - unsigned int psic; - unsigned int psiv; - unsigned int i; - uint32_t revision; - uint32_t ports; - uint32_t slot; - uint32_t psi; - - /* Fail if there is no supported protocol */ - if ( ! supported ) - return 0; - - /* Determine protocol version */ - revision = readl ( xhci->cap + supported + XHCI_SUPPORTED_REVISION ); - protocol = XHCI_SUPPORTED_REVISION_VER ( revision ); - - /* Describe port protocol */ - if ( DBG_EXTRA ) { - name.raw = cpu_to_le32 ( readl ( xhci->cap + supported + - XHCI_SUPPORTED_NAME ) ); - name.text[4] = '\0'; - slot = readl ( xhci->cap + supported + XHCI_SUPPORTED_SLOT ); - type = XHCI_SUPPORTED_SLOT_TYPE ( slot ); - DBGC2 ( xhci, "XHCI %s-%d %sv%04x type %d", - xhci->name, port, name.text, protocol, type ); - ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS ); - psic = XHCI_SUPPORTED_PORTS_PSIC ( ports ); - if ( psic ) { - DBGC2 ( xhci, " speeds" ); - for ( i = 0 ; i < psic ; i++ ) { - psi = readl ( xhci->cap + supported + - XHCI_SUPPORTED_PSI ( i ) ); - psiv = XHCI_SUPPORTED_PSI_VALUE ( psi ); - DBGC2 ( xhci, " %d:%s", psiv, - xhci_speed_name ( psi ) ); - } - } - if ( xhci->quirks & XHCI_BAD_PSIV ) - DBGC2 ( xhci, " (ignored)" ); - DBGC2 ( xhci, "\n" ); - } - - return protocol; -} - -/** - * Find port slot type - * - * @v xhci xHCI device - * @v port Port number - * @ret type Slot type, or negative error - */ -static int xhci_port_slot_type ( struct xhci_device *xhci, unsigned int port ) { - unsigned int supported = xhci_supported_protocol ( xhci, port ); - unsigned int type; - uint32_t slot; - - /* Fail if there is no supported protocol */ - if ( ! supported ) - return -ENOTSUP; - - /* Get slot type */ - slot = readl ( xhci->cap + supported + XHCI_SUPPORTED_SLOT ); - type = XHCI_SUPPORTED_SLOT_TYPE ( slot ); - - return type; -} - -/** - * Find port speed - * - * @v xhci xHCI device - * @v port Port number - * @v psiv Protocol speed ID value - * @ret speed Port speed, or negative error - */ -static int xhci_port_speed ( struct xhci_device *xhci, unsigned int port, - unsigned int psiv ) { - unsigned int supported = xhci_supported_protocol ( xhci, port ); - unsigned int psic; - unsigned int mantissa; - unsigned int exponent; - unsigned int speed; - unsigned int i; - uint32_t ports; - uint32_t psi; - - /* Fail if there is no supported protocol */ - if ( ! supported ) - return -ENOTSUP; - - /* Get protocol speed ID count */ - ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS ); - psic = XHCI_SUPPORTED_PORTS_PSIC ( ports ); - - /* Use the default mappings if applicable */ - if ( ( psic == 0 ) || ( xhci->quirks & XHCI_BAD_PSIV ) ) { - switch ( psiv ) { - case XHCI_SPEED_LOW : return USB_SPEED_LOW; - case XHCI_SPEED_FULL : return USB_SPEED_FULL; - case XHCI_SPEED_HIGH : return USB_SPEED_HIGH; - case XHCI_SPEED_SUPER : return USB_SPEED_SUPER; - default: - DBGC ( xhci, "XHCI %s-%d non-standard PSI value %d\n", - xhci->name, port, psiv ); - return -ENOTSUP; - } - } - - /* Iterate over PSI dwords looking for a match */ - for ( i = 0 ; i < psic ; i++ ) { - psi = readl ( xhci->cap + supported + XHCI_SUPPORTED_PSI ( i )); - if ( psiv == XHCI_SUPPORTED_PSI_VALUE ( psi ) ) { - mantissa = XHCI_SUPPORTED_PSI_MANTISSA ( psi ); - exponent = XHCI_SUPPORTED_PSI_EXPONENT ( psi ); - speed = USB_SPEED ( mantissa, exponent ); - return speed; - } - } - - DBGC ( xhci, "XHCI %s-%d spurious PSI value %d\n", - xhci->name, port, psiv ); - return -ENOENT; -} - -/** - * Find protocol speed ID value - * - * @v xhci xHCI device - * @v port Port number - * @v speed USB speed - * @ret psiv Protocol speed ID value, or negative error - */ -static int xhci_port_psiv ( struct xhci_device *xhci, unsigned int port, - unsigned int speed ) { - unsigned int supported = xhci_supported_protocol ( xhci, port ); - unsigned int psic; - unsigned int mantissa; - unsigned int exponent; - unsigned int psiv; - unsigned int i; - uint32_t ports; - uint32_t psi; - - /* Fail if there is no supported protocol */ - if ( ! supported ) - return -ENOTSUP; - - /* Get protocol speed ID count */ - ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS ); - psic = XHCI_SUPPORTED_PORTS_PSIC ( ports ); - - /* Use the default mappings if applicable */ - if ( ( psic == 0 ) || ( xhci->quirks & XHCI_BAD_PSIV ) ) { - switch ( speed ) { - case USB_SPEED_LOW : return XHCI_SPEED_LOW; - case USB_SPEED_FULL : return XHCI_SPEED_FULL; - case USB_SPEED_HIGH : return XHCI_SPEED_HIGH; - case USB_SPEED_SUPER : return XHCI_SPEED_SUPER; - default: - DBGC ( xhci, "XHCI %s-%d non-standard speed %d\n", - xhci->name, port, speed ); - return -ENOTSUP; - } - } - - /* Iterate over PSI dwords looking for a match */ - for ( i = 0 ; i < psic ; i++ ) { - psi = readl ( xhci->cap + supported + XHCI_SUPPORTED_PSI ( i )); - mantissa = XHCI_SUPPORTED_PSI_MANTISSA ( psi ); - exponent = XHCI_SUPPORTED_PSI_EXPONENT ( psi ); - if ( speed == USB_SPEED ( mantissa, exponent ) ) { - psiv = XHCI_SUPPORTED_PSI_VALUE ( psi ); - return psiv; - } - } - - DBGC ( xhci, "XHCI %s-%d unrepresentable speed %#x\n", - xhci->name, port, speed ); - return -ENOENT; -} - -/****************************************************************************** - * - * Device context base address array - * - ****************************************************************************** - */ - -/** - * Allocate device context base address array - * - * @v xhci xHCI device - * @ret rc Return status code - */ -static int xhci_dcbaa_alloc ( struct xhci_device *xhci ) { - size_t len; - physaddr_t dcbaap; - int rc; - - /* Allocate and initialise structure. Must be at least - * 64-byte aligned and must not cross a page boundary, so - * align on its own size (rounded up to a power of two and - * with a minimum of 64 bytes). - */ - len = ( ( xhci->slots + 1 ) * sizeof ( xhci->dcbaa[0] ) ); - xhci->dcbaa = malloc_dma ( len, xhci_align ( len ) ); - if ( ! xhci->dcbaa ) { - DBGC ( xhci, "XHCI %s could not allocate DCBAA\n", xhci->name ); - rc = -ENOMEM; - goto err_alloc; - } - memset ( xhci->dcbaa, 0, len ); - - /* Program DCBAA pointer */ - dcbaap = virt_to_phys ( xhci->dcbaa ); - if ( ( rc = xhci_writeq ( xhci, dcbaap, - xhci->op + XHCI_OP_DCBAAP ) ) != 0 ) - goto err_writeq; - - DBGC2 ( xhci, "XHCI %s DCBAA at [%08lx,%08lx)\n", - xhci->name, dcbaap, ( dcbaap + len ) ); - return 0; - - err_writeq: - free_dma ( xhci->dcbaa, len ); - err_alloc: - return rc; -} - -/** - * Free device context base address array - * - * @v xhci xHCI device - */ -static void xhci_dcbaa_free ( struct xhci_device *xhci ) { - size_t len; - unsigned int i; - - /* Sanity check */ - for ( i = 0 ; i <= xhci->slots ; i++ ) - assert ( xhci->dcbaa[i] == 0 ); - - /* Clear DCBAA pointer */ - xhci_writeq ( xhci, 0, xhci->op + XHCI_OP_DCBAAP ); - - /* Free DCBAA */ - len = ( ( xhci->slots + 1 ) * sizeof ( xhci->dcbaa[0] ) ); - free_dma ( xhci->dcbaa, len ); -} - -/****************************************************************************** - * - * Scratchpad buffers - * - ****************************************************************************** - */ - -/** - * Allocate scratchpad buffers - * - * @v xhci xHCI device - * @ret rc Return status code - */ -static int xhci_scratchpad_alloc ( struct xhci_device *xhci ) { - size_t array_len; - size_t len; - physaddr_t phys; - unsigned int i; - int rc; - - /* Do nothing if no scratchpad buffers are used */ - if ( ! xhci->scratchpads ) - return 0; - - /* Allocate scratchpads */ - len = ( xhci->scratchpads * xhci->pagesize ); - xhci->scratchpad = umalloc ( len ); - if ( ! xhci->scratchpad ) { - DBGC ( xhci, "XHCI %s could not allocate scratchpad buffers\n", - xhci->name ); - rc = -ENOMEM; - goto err_alloc; - } - memset_user ( xhci->scratchpad, 0, 0, len ); - - /* Allocate scratchpad array */ - array_len = ( xhci->scratchpads * sizeof ( xhci->scratchpad_array[0] )); - xhci->scratchpad_array = - malloc_dma ( array_len, xhci_align ( array_len ) ); - if ( ! xhci->scratchpad_array ) { - DBGC ( xhci, "XHCI %s could not allocate scratchpad buffer " - "array\n", xhci->name ); - rc = -ENOMEM; - goto err_alloc_array; - } - - /* Populate scratchpad array */ - for ( i = 0 ; i < xhci->scratchpads ; i++ ) { - phys = user_to_phys ( xhci->scratchpad, ( i * xhci->pagesize )); - xhci->scratchpad_array[i] = phys; - } - - /* Set scratchpad array pointer */ - assert ( xhci->dcbaa != NULL ); - xhci->dcbaa[0] = cpu_to_le64 ( virt_to_phys ( xhci->scratchpad_array )); - - DBGC2 ( xhci, "XHCI %s scratchpad [%08lx,%08lx) array [%08lx,%08lx)\n", - xhci->name, user_to_phys ( xhci->scratchpad, 0 ), - user_to_phys ( xhci->scratchpad, len ), - virt_to_phys ( xhci->scratchpad_array ), - ( virt_to_phys ( xhci->scratchpad_array ) + array_len ) ); - return 0; - - free_dma ( xhci->scratchpad_array, array_len ); - err_alloc_array: - ufree ( xhci->scratchpad ); - err_alloc: - return rc; -} - -/** - * Free scratchpad buffers - * - * @v xhci xHCI device - */ -static void xhci_scratchpad_free ( struct xhci_device *xhci ) { - size_t array_len; - - /* Do nothing if no scratchpad buffers are used */ - if ( ! xhci->scratchpads ) - return; - - /* Clear scratchpad array pointer */ - assert ( xhci->dcbaa != NULL ); - xhci->dcbaa[0] = 0; - - /* Free scratchpad array */ - array_len = ( xhci->scratchpads * sizeof ( xhci->scratchpad_array[0] )); - free_dma ( xhci->scratchpad_array, array_len ); - - /* Free scratchpads */ - ufree ( xhci->scratchpad ); -} - -/****************************************************************************** - * - * Run / stop / reset - * - ****************************************************************************** - */ - -/** - * Start xHCI device - * - * @v xhci xHCI device - */ -static void xhci_run ( struct xhci_device *xhci ) { - uint32_t config; - uint32_t usbcmd; - - /* Configure number of device slots */ - config = readl ( xhci->op + XHCI_OP_CONFIG ); - config &= ~XHCI_CONFIG_MAX_SLOTS_EN_MASK; - config |= XHCI_CONFIG_MAX_SLOTS_EN ( xhci->slots ); - writel ( config, xhci->op + XHCI_OP_CONFIG ); - - /* Set run/stop bit */ - usbcmd = readl ( xhci->op + XHCI_OP_USBCMD ); - usbcmd |= XHCI_USBCMD_RUN; - writel ( usbcmd, xhci->op + XHCI_OP_USBCMD ); -} - -/** - * Stop xHCI device - * - * @v xhci xHCI device - * @ret rc Return status code - */ -static int xhci_stop ( struct xhci_device *xhci ) { - uint32_t usbcmd; - uint32_t usbsts; - unsigned int i; - - /* Clear run/stop bit */ - usbcmd = readl ( xhci->op + XHCI_OP_USBCMD ); - usbcmd &= ~XHCI_USBCMD_RUN; - writel ( usbcmd, xhci->op + XHCI_OP_USBCMD ); - - /* Wait for device to stop */ - for ( i = 0 ; i < XHCI_STOP_MAX_WAIT_MS ; i++ ) { - - /* Check if device is stopped */ - usbsts = readl ( xhci->op + XHCI_OP_USBSTS ); - if ( usbsts & XHCI_USBSTS_HCH ) - return 0; - - /* Delay */ - mdelay ( 1 ); - } - - DBGC ( xhci, "XHCI %s timed out waiting for stop\n", xhci->name ); - return -ETIMEDOUT; -} - -/** - * Reset xHCI device - * - * @v xhci xHCI device - * @ret rc Return status code - */ -static int xhci_reset ( struct xhci_device *xhci ) { - uint32_t usbcmd; - unsigned int i; - int rc; - - /* The xHCI specification states that resetting a running - * device may result in undefined behaviour, so try stopping - * it first. - */ - if ( ( rc = xhci_stop ( xhci ) ) != 0 ) { - /* Ignore errors and attempt to reset the device anyway */ - } - - /* Reset device */ - writel ( XHCI_USBCMD_HCRST, xhci->op + XHCI_OP_USBCMD ); - - /* Wait for reset to complete */ - for ( i = 0 ; i < XHCI_RESET_MAX_WAIT_MS ; i++ ) { - - /* Check if reset is complete */ - usbcmd = readl ( xhci->op + XHCI_OP_USBCMD ); - if ( ! ( usbcmd & XHCI_USBCMD_HCRST ) ) - return 0; - - /* Delay */ - mdelay ( 1 ); - } - - DBGC ( xhci, "XHCI %s timed out waiting for reset\n", xhci->name ); - return -ETIMEDOUT; -} - -/****************************************************************************** - * - * Transfer request blocks - * - ****************************************************************************** - */ - -/** - * Allocate transfer request block ring - * - * @v xhci xHCI device - * @v ring TRB ring - * @v shift Ring size (log2) - * @v slot Device slot - * @v target Doorbell target - * @v stream Doorbell stream ID - * @ret rc Return status code - */ -static int xhci_ring_alloc ( struct xhci_device *xhci, - struct xhci_trb_ring *ring, - unsigned int shift, unsigned int slot, - unsigned int target, unsigned int stream ) { - struct xhci_trb_link *link; - unsigned int count; - int rc; - - /* Sanity check */ - assert ( shift > 0 ); - - /* Initialise structure */ - memset ( ring, 0, sizeof ( *ring ) ); - ring->shift = shift; - count = ( 1U << shift ); - ring->mask = ( count - 1 ); - ring->len = ( ( count + 1 /* Link TRB */ ) * sizeof ( ring->trb[0] ) ); - ring->db = ( xhci->db + ( slot * sizeof ( ring->dbval ) ) ); - ring->dbval = XHCI_DBVAL ( target, stream ); - - /* Allocate I/O buffers */ - ring->iobuf = zalloc ( count * sizeof ( ring->iobuf[0] ) ); - if ( ! ring->iobuf ) { - rc = -ENOMEM; - goto err_alloc_iobuf; - } - - /* Allocate TRBs */ - ring->trb = malloc_dma ( ring->len, xhci_align ( ring->len ) ); - if ( ! ring->trb ) { - rc = -ENOMEM; - goto err_alloc_trb; - } - memset ( ring->trb, 0, ring->len ); - - /* Initialise Link TRB */ - link = &ring->trb[count].link; - link->next = cpu_to_le64 ( virt_to_phys ( ring->trb ) ); - link->flags = XHCI_TRB_TC; - link->type = XHCI_TRB_LINK; - ring->link = link; - - return 0; - - free_dma ( ring->trb, ring->len ); - err_alloc_trb: - free ( ring->iobuf ); - err_alloc_iobuf: - return rc; -} - -/** - * Reset transfer request block ring - * - * @v ring TRB ring - */ -static void xhci_ring_reset ( struct xhci_trb_ring *ring ) { - unsigned int count = ( 1U << ring->shift ); - - /* Reset producer and consumer counters */ - ring->prod = 0; - ring->cons = 0; - - /* Reset TRBs (except Link TRB) */ - memset ( ring->trb, 0, ( count * sizeof ( ring->trb[0] ) ) ); -} - -/** - * Free transfer request block ring - * - * @v ring TRB ring - */ -static void xhci_ring_free ( struct xhci_trb_ring *ring ) { - unsigned int count = ( 1U << ring->shift ); - unsigned int i; - - /* Sanity checks */ - assert ( ring->cons == ring->prod ); - for ( i = 0 ; i < count ; i++ ) - assert ( ring->iobuf[i] == NULL ); - - /* Free TRBs */ - free_dma ( ring->trb, ring->len ); - - /* Free I/O buffers */ - free ( ring->iobuf ); -} - -/** - * Enqueue a transfer request block - * - * @v ring TRB ring - * @v iobuf I/O buffer (if any) - * @v trb Transfer request block (with empty Cycle flag) - * @ret rc Return status code - * - * This operation does not implicitly ring the doorbell register. - */ -static int xhci_enqueue ( struct xhci_trb_ring *ring, struct io_buffer *iobuf, - const union xhci_trb *trb ) { - union xhci_trb *dest; - unsigned int prod; - unsigned int mask; - unsigned int index; - unsigned int cycle; - - /* Sanity check */ - assert ( ! ( trb->common.flags & XHCI_TRB_C ) ); - - /* Fail if ring is full */ - if ( ! xhci_ring_remaining ( ring ) ) - return -ENOBUFS; - - /* Update producer counter (and link TRB, if applicable) */ - prod = ring->prod++; - mask = ring->mask; - cycle = ( ( ~( prod >> ring->shift ) ) & XHCI_TRB_C ); - index = ( prod & mask ); - if ( index == 0 ) - ring->link->flags = ( XHCI_TRB_TC | ( cycle ^ XHCI_TRB_C ) ); - - /* Record I/O buffer */ - ring->iobuf[index] = iobuf; - - /* Enqueue TRB */ - dest = &ring->trb[index]; - dest->template.parameter = trb->template.parameter; - dest->template.status = trb->template.status; - wmb(); - dest->template.control = ( trb->template.control | - cpu_to_le32 ( cycle ) ); - - return 0; -} - -/** - * Dequeue a transfer request block - * - * @v ring TRB ring - * @ret iobuf I/O buffer - */ -static struct io_buffer * xhci_dequeue ( struct xhci_trb_ring *ring ) { - struct io_buffer *iobuf; - unsigned int cons; - unsigned int mask; - unsigned int index; - - /* Sanity check */ - assert ( xhci_ring_fill ( ring ) != 0 ); - - /* Update consumer counter */ - cons = ring->cons++; - mask = ring->mask; - index = ( cons & mask ); - - /* Retrieve I/O buffer */ - iobuf = ring->iobuf[index]; - ring->iobuf[index] = NULL; - - return iobuf; -} - -/** - * Enqueue multiple transfer request blocks - * - * @v ring TRB ring - * @v iobuf I/O buffer - * @v trbs Transfer request blocks (with empty Cycle flag) - * @v count Number of transfer request blocks - * @ret rc Return status code - * - * This operation does not implicitly ring the doorbell register. - */ -static int xhci_enqueue_multi ( struct xhci_trb_ring *ring, - struct io_buffer *iobuf, - const union xhci_trb *trbs, - unsigned int count ) { - const union xhci_trb *trb = trbs; - int rc; - - /* Sanity check */ - assert ( iobuf != NULL ); - - /* Fail if ring does not have sufficient space */ - if ( xhci_ring_remaining ( ring ) < count ) - return -ENOBUFS; - - /* Enqueue each TRB, recording the I/O buffer with the final TRB */ - while ( count-- ) { - rc = xhci_enqueue ( ring, ( count ? NULL : iobuf ), trb++ ); - assert ( rc == 0 ); /* Should never be able to fail */ - } - - return 0; -} - -/** - * Dequeue multiple transfer request blocks - * - * @v ring TRB ring - * @ret iobuf I/O buffer - */ -static struct io_buffer * xhci_dequeue_multi ( struct xhci_trb_ring *ring ) { - struct io_buffer *iobuf; - - /* Dequeue TRBs until we reach the final TRB for an I/O buffer */ - do { - iobuf = xhci_dequeue ( ring ); - } while ( iobuf == NULL ); - - return iobuf; -} - -/** - * Ring doorbell register - * - * @v ring TRB ring - */ -static inline __attribute__ (( always_inline )) void -xhci_doorbell ( struct xhci_trb_ring *ring ) { - - wmb(); - writel ( ring->dbval, ring->db ); -} - -/****************************************************************************** - * - * Command and event rings - * - ****************************************************************************** - */ - -/** - * Allocate command ring - * - * @v xhci xHCI device - * @ret rc Return status code - */ -static int xhci_command_alloc ( struct xhci_device *xhci ) { - physaddr_t crp; - int rc; - - /* Allocate TRB ring */ - if ( ( rc = xhci_ring_alloc ( xhci, &xhci->command, XHCI_CMD_TRBS_LOG2, - 0, 0, 0 ) ) != 0 ) - goto err_ring_alloc; - - /* Program command ring control register */ - crp = virt_to_phys ( xhci->command.trb ); - if ( ( rc = xhci_writeq ( xhci, ( crp | XHCI_CRCR_RCS ), - xhci->op + XHCI_OP_CRCR ) ) != 0 ) - goto err_writeq; - - DBGC2 ( xhci, "XHCI %s CRCR at [%08lx,%08lx)\n", - xhci->name, crp, ( crp + xhci->command.len ) ); - return 0; - - err_writeq: - xhci_ring_free ( &xhci->command ); - err_ring_alloc: - return rc; -} - -/** - * Free command ring - * - * @v xhci xHCI device - */ -static void xhci_command_free ( struct xhci_device *xhci ) { - - /* Sanity check */ - assert ( ( readl ( xhci->op + XHCI_OP_CRCR ) & XHCI_CRCR_CRR ) == 0 ); - - /* Clear command ring control register */ - xhci_writeq ( xhci, 0, xhci->op + XHCI_OP_CRCR ); - - /* Free TRB ring */ - xhci_ring_free ( &xhci->command ); -} - -/** - * Allocate event ring - * - * @v xhci xHCI device - * @ret rc Return status code - */ -static int xhci_event_alloc ( struct xhci_device *xhci ) { - struct xhci_event_ring *event = &xhci->event; - unsigned int count; - size_t len; - int rc; - - /* Allocate event ring */ - count = ( 1 << XHCI_EVENT_TRBS_LOG2 ); - len = ( count * sizeof ( event->trb[0] ) ); - event->trb = malloc_dma ( len, xhci_align ( len ) ); - if ( ! event->trb ) { - rc = -ENOMEM; - goto err_alloc_trb; - } - memset ( event->trb, 0, len ); - - /* Allocate event ring segment table */ - event->segment = malloc_dma ( sizeof ( event->segment[0] ), - xhci_align ( sizeof (event->segment[0]))); - if ( ! event->segment ) { - rc = -ENOMEM; - goto err_alloc_segment; - } - memset ( event->segment, 0, sizeof ( event->segment[0] ) ); - event->segment[0].base = cpu_to_le64 ( virt_to_phys ( event->trb ) ); - event->segment[0].count = cpu_to_le32 ( count ); - - /* Program event ring registers */ - writel ( 1, xhci->run + XHCI_RUN_ERSTSZ ( 0 ) ); - if ( ( rc = xhci_writeq ( xhci, virt_to_phys ( event->trb ), - xhci->run + XHCI_RUN_ERDP ( 0 ) ) ) != 0 ) - goto err_writeq_erdp; - if ( ( rc = xhci_writeq ( xhci, virt_to_phys ( event->segment ), - xhci->run + XHCI_RUN_ERSTBA ( 0 ) ) ) != 0 ) - goto err_writeq_erstba; - - DBGC2 ( xhci, "XHCI %s event ring [%08lx,%08lx) table [%08lx,%08lx)\n", - xhci->name, virt_to_phys ( event->trb ), - ( virt_to_phys ( event->trb ) + len ), - virt_to_phys ( event->segment ), - ( virt_to_phys ( event->segment ) + - sizeof (event->segment[0] ) ) ); - return 0; - - xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERSTBA ( 0 ) ); - err_writeq_erstba: - xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERDP ( 0 ) ); - err_writeq_erdp: - free_dma ( event->trb, len ); - err_alloc_segment: - free_dma ( event->segment, sizeof ( event->segment[0] ) ); - err_alloc_trb: - return rc; -} - -/** - * Free event ring - * - * @v xhci xHCI device - */ -static void xhci_event_free ( struct xhci_device *xhci ) { - struct xhci_event_ring *event = &xhci->event; - unsigned int count; - size_t len; - - /* Clear event ring registers */ - writel ( 0, xhci->run + XHCI_RUN_ERSTSZ ( 0 ) ); - xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERSTBA ( 0 ) ); - xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERDP ( 0 ) ); - - /* Free event ring segment table */ - free_dma ( event->segment, sizeof ( event->segment[0] ) ); - - /* Free event ring */ - count = ( 1 << XHCI_EVENT_TRBS_LOG2 ); - len = ( count * sizeof ( event->trb[0] ) ); - free_dma ( event->trb, len ); -} - -/** - * Handle transfer event - * - * @v xhci xHCI device - * @v trb Transfer event TRB - */ -static void xhci_transfer ( struct xhci_device *xhci, - struct xhci_trb_transfer *trb ) { - struct xhci_slot *slot; - struct xhci_endpoint *endpoint; - struct io_buffer *iobuf; - int rc; - - /* Profile transfer events */ - profile_start ( &xhci_transfer_profiler ); - - /* Identify slot */ - if ( ( trb->slot > xhci->slots ) || - ( ( slot = xhci->slot[trb->slot] ) == NULL ) ) { - DBGC ( xhci, "XHCI %s transfer event invalid slot %d:\n", - xhci->name, trb->slot ); - DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) ); - return; - } - - /* Identify endpoint */ - if ( ( trb->endpoint > XHCI_CTX_END ) || - ( ( endpoint = slot->endpoint[trb->endpoint] ) == NULL ) ) { - DBGC ( xhci, "XHCI %s slot %d transfer event invalid epid " - "%d:\n", xhci->name, slot->id, trb->endpoint ); - DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) ); - return; - } - - /* Dequeue TRB(s) */ - iobuf = xhci_dequeue_multi ( &endpoint->ring ); - assert ( iobuf != NULL ); - - /* Check for errors */ - if ( ! ( ( trb->code == XHCI_CMPLT_SUCCESS ) || - ( trb->code == XHCI_CMPLT_SHORT ) ) ) { - - /* Construct error */ - rc = -ECODE ( trb->code ); - DBGC ( xhci, "XHCI %s slot %d ctx %d failed (code %d): %s\n", - xhci->name, slot->id, endpoint->ctx, trb->code, - strerror ( rc ) ); - DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) ); - - /* Sanity check */ - assert ( ( endpoint->context->state & XHCI_ENDPOINT_STATE_MASK ) - != XHCI_ENDPOINT_RUNNING ); - - /* Report failure to USB core */ - usb_complete_err ( endpoint->ep, iobuf, rc ); - return; - } - - /* Record actual transfer size */ - iob_unput ( iobuf, le16_to_cpu ( trb->residual ) ); - - /* Sanity check (for successful completions only) */ - assert ( xhci_ring_consumed ( &endpoint->ring ) == - le64_to_cpu ( trb->transfer ) ); - - /* Report completion to USB core */ - usb_complete ( endpoint->ep, iobuf ); - profile_stop ( &xhci_transfer_profiler ); -} - -/** - * Handle command completion event - * - * @v xhci xHCI device - * @v trb Command completion event - */ -static void xhci_complete ( struct xhci_device *xhci, - struct xhci_trb_complete *trb ) { - int rc; - - /* Ignore "command ring stopped" notifications */ - if ( trb->code == XHCI_CMPLT_CMD_STOPPED ) { - DBGC2 ( xhci, "XHCI %s command ring stopped\n", xhci->name ); - return; - } - - /* Ignore unexpected completions */ - if ( ! xhci->pending ) { - rc = -ECODE ( trb->code ); - DBGC ( xhci, "XHCI %s unexpected completion (code %d): %s\n", - xhci->name, trb->code, strerror ( rc ) ); - DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) ); - return; - } - - /* Dequeue command TRB */ - xhci_dequeue ( &xhci->command ); - - /* Sanity check */ - assert ( xhci_ring_consumed ( &xhci->command ) == - le64_to_cpu ( trb->command ) ); - - /* Record completion */ - memcpy ( xhci->pending, trb, sizeof ( *xhci->pending ) ); - xhci->pending = NULL; -} - -/** - * Handle port status event - * - * @v xhci xHCI device - * @v trb Port status event - */ -static void xhci_port_status ( struct xhci_device *xhci, - struct xhci_trb_port_status *trb ) { - struct usb_port *port = usb_port ( xhci->bus->hub, trb->port ); - uint32_t portsc; - - /* Sanity check */ - assert ( ( trb->port > 0 ) && ( trb->port <= xhci->ports ) ); - - /* Record disconnections and clear changes */ - portsc = readl ( xhci->op + XHCI_OP_PORTSC ( trb->port ) ); - port->disconnected |= ( portsc & XHCI_PORTSC_CSC ); - portsc &= ( XHCI_PORTSC_PRESERVE | XHCI_PORTSC_CHANGE ); - writel ( portsc, xhci->op + XHCI_OP_PORTSC ( trb->port ) ); - - /* Report port status change */ - usb_port_changed ( port ); -} - -/** - * Handle host controller event - * - * @v xhci xHCI device - * @v trb Host controller event - */ -static void xhci_host_controller ( struct xhci_device *xhci, - struct xhci_trb_host_controller *trb ) { - int rc; - - /* Construct error */ - rc = -ECODE ( trb->code ); - DBGC ( xhci, "XHCI %s host controller event (code %d): %s\n", - xhci->name, trb->code, strerror ( rc ) ); -} - -/** - * Poll event ring - * - * @v xhci xHCI device - */ -static void xhci_event_poll ( struct xhci_device *xhci ) { - struct xhci_event_ring *event = &xhci->event; - union xhci_trb *trb; - unsigned int shift = XHCI_EVENT_TRBS_LOG2; - unsigned int count = ( 1 << shift ); - unsigned int mask = ( count - 1 ); - unsigned int consumed; - unsigned int type; - - /* Poll for events */ - profile_start ( &xhci_event_profiler ); - for ( consumed = 0 ; ; consumed++ ) { - - /* Stop if we reach an empty TRB */ - rmb(); - trb = &event->trb[ event->cons & mask ]; - if ( ! ( ( trb->common.flags ^ - ( event->cons >> shift ) ) & XHCI_TRB_C ) ) - break; - - /* Handle TRB */ - type = ( trb->common.type & XHCI_TRB_TYPE_MASK ); - switch ( type ) { - - case XHCI_TRB_TRANSFER : - xhci_transfer ( xhci, &trb->transfer ); - break; - - case XHCI_TRB_COMPLETE : - xhci_complete ( xhci, &trb->complete ); - break; - - case XHCI_TRB_PORT_STATUS: - xhci_port_status ( xhci, &trb->port ); - break; - - case XHCI_TRB_HOST_CONTROLLER: - xhci_host_controller ( xhci, &trb->host ); - break; - - default: - DBGC ( xhci, "XHCI %s unrecognised event %#x\n:", - xhci->name, event->cons ); - DBGC_HDA ( xhci, virt_to_phys ( trb ), - trb, sizeof ( *trb ) ); - break; - } - - /* Consume this TRB */ - event->cons++; - } - - /* Update dequeue pointer if applicable */ - if ( consumed ) { - xhci_writeq ( xhci, virt_to_phys ( trb ), - xhci->run + XHCI_RUN_ERDP ( 0 ) ); - profile_stop ( &xhci_event_profiler ); - } -} - -/** - * Abort command - * - * @v xhci xHCI device - */ -static void xhci_abort ( struct xhci_device *xhci ) { - physaddr_t crp; - - /* Abort the command */ - DBGC2 ( xhci, "XHCI %s aborting command\n", xhci->name ); - xhci_writeq ( xhci, XHCI_CRCR_CA, xhci->op + XHCI_OP_CRCR ); - - /* Allow time for command to abort */ - mdelay ( XHCI_COMMAND_ABORT_DELAY_MS ); - - /* Sanity check */ - assert ( ( readl ( xhci->op + XHCI_OP_CRCR ) & XHCI_CRCR_CRR ) == 0 ); - - /* Consume (and ignore) any final command status */ - xhci_event_poll ( xhci ); - - /* Reset the command ring control register */ - xhci_ring_reset ( &xhci->command ); - crp = virt_to_phys ( xhci->command.trb ); - xhci_writeq ( xhci, ( crp | XHCI_CRCR_RCS ), xhci->op + XHCI_OP_CRCR ); -} - -/** - * Issue command and wait for completion - * - * @v xhci xHCI device - * @v trb Transfer request block (with empty Cycle flag) - * @ret rc Return status code - * - * On a successful completion, the TRB will be overwritten with the - * completion. - */ -static int xhci_command ( struct xhci_device *xhci, union xhci_trb *trb ) { - struct xhci_trb_complete *complete = &trb->complete; - unsigned int i; - int rc; - - /* Record the pending command */ - xhci->pending = trb; - - /* Enqueue the command */ - if ( ( rc = xhci_enqueue ( &xhci->command, NULL, trb ) ) != 0 ) - goto err_enqueue; - - /* Ring the command doorbell */ - xhci_doorbell ( &xhci->command ); - - /* Wait for the command to complete */ - for ( i = 0 ; i < XHCI_COMMAND_MAX_WAIT_MS ; i++ ) { - - /* Poll event ring */ - xhci_event_poll ( xhci ); - - /* Check for completion */ - if ( ! xhci->pending ) { - if ( complete->code != XHCI_CMPLT_SUCCESS ) { - rc = -ECODE ( complete->code ); - DBGC ( xhci, "XHCI %s command failed (code " - "%d): %s\n", xhci->name, complete->code, - strerror ( rc ) ); - DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) ); - return rc; - } - return 0; - } - - /* Delay */ - mdelay ( 1 ); - } - - /* Timeout */ - DBGC ( xhci, "XHCI %s timed out waiting for completion\n", xhci->name ); - rc = -ETIMEDOUT; - - /* Abort command */ - xhci_abort ( xhci ); - - err_enqueue: - xhci->pending = NULL; - return rc; -} - -/** - * Issue NOP and wait for completion - * - * @v xhci xHCI device - * @ret rc Return status code - */ -static inline int xhci_nop ( struct xhci_device *xhci ) { - union xhci_trb trb; - struct xhci_trb_common *nop = &trb.common; - int rc; - - /* Construct command */ - memset ( nop, 0, sizeof ( *nop ) ); - nop->flags = XHCI_TRB_IOC; - nop->type = XHCI_TRB_NOP_CMD; - - /* Issue command and wait for completion */ - if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) - return rc; - - return 0; -} - -/** - * Enable slot - * - * @v xhci xHCI device - * @v type Slot type - * @ret slot Device slot ID, or negative error - */ -static inline int xhci_enable_slot ( struct xhci_device *xhci, - unsigned int type ) { - union xhci_trb trb; - struct xhci_trb_enable_slot *enable = &trb.enable; - struct xhci_trb_complete *enabled = &trb.complete; - unsigned int slot; - int rc; - - /* Construct command */ - memset ( enable, 0, sizeof ( *enable ) ); - enable->slot = type; - enable->type = XHCI_TRB_ENABLE_SLOT; - - /* Issue command and wait for completion */ - if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) { - DBGC ( xhci, "XHCI %s could not enable new slot: %s\n", - xhci->name, strerror ( rc ) ); - return rc; - } - - /* Extract slot number */ - slot = enabled->slot; - - DBGC2 ( xhci, "XHCI %s slot %d enabled\n", xhci->name, slot ); - return slot; -} - -/** - * Disable slot - * - * @v xhci xHCI device - * @v slot Device slot - * @ret rc Return status code - */ -static inline int xhci_disable_slot ( struct xhci_device *xhci, - unsigned int slot ) { - union xhci_trb trb; - struct xhci_trb_disable_slot *disable = &trb.disable; - int rc; - - /* Construct command */ - memset ( disable, 0, sizeof ( *disable ) ); - disable->type = XHCI_TRB_DISABLE_SLOT; - disable->slot = slot; - - /* Issue command and wait for completion */ - if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) { - DBGC ( xhci, "XHCI %s could not disable slot %d: %s\n", - xhci->name, slot, strerror ( rc ) ); - return rc; - } - - DBGC2 ( xhci, "XHCI %s slot %d disabled\n", xhci->name, slot ); - return 0; -} - -/** - * Issue context-based command and wait for completion - * - * @v xhci xHCI device - * @v slot Device slot - * @v endpoint Endpoint - * @v type TRB type - * @v populate Input context populater - * @ret rc Return status code - */ -static int xhci_context ( struct xhci_device *xhci, struct xhci_slot *slot, - struct xhci_endpoint *endpoint, unsigned int type, - void ( * populate ) ( struct xhci_device *xhci, - struct xhci_slot *slot, - struct xhci_endpoint *endpoint, - void *input ) ) { - union xhci_trb trb; - struct xhci_trb_context *context = &trb.context; - size_t len; - void *input; - int rc; - - /* Allocate an input context */ - len = xhci_input_context_offset ( xhci, XHCI_CTX_END ); - input = malloc_dma ( len, xhci_align ( len ) ); - if ( ! input ) { - rc = -ENOMEM; - goto err_alloc; - } - memset ( input, 0, len ); - - /* Populate input context */ - populate ( xhci, slot, endpoint, input ); - - /* Construct command */ - memset ( context, 0, sizeof ( *context ) ); - context->type = type; - context->input = cpu_to_le64 ( virt_to_phys ( input ) ); - context->slot = slot->id; - - /* Issue command and wait for completion */ - if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) - goto err_command; - - err_command: - free_dma ( input, len ); - err_alloc: - return rc; -} - -/** - * Populate address device input context - * - * @v xhci xHCI device - * @v slot Device slot - * @v endpoint Endpoint - * @v input Input context - */ -static void xhci_address_device_input ( struct xhci_device *xhci, - struct xhci_slot *slot, - struct xhci_endpoint *endpoint, - void *input ) { - struct xhci_control_context *control_ctx; - struct xhci_slot_context *slot_ctx; - struct xhci_endpoint_context *ep_ctx; - - /* Sanity checks */ - assert ( endpoint->ctx == XHCI_CTX_EP0 ); - - /* Populate control context */ - control_ctx = input; - control_ctx->add = cpu_to_le32 ( ( 1 << XHCI_CTX_SLOT ) | - ( 1 << XHCI_CTX_EP0 ) ); - - /* Populate slot context */ - slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT )); - slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( 1, 0, slot->psiv, - slot->route ) ); - slot_ctx->port = slot->port; - slot_ctx->tt_id = slot->tt_id; - slot_ctx->tt_port = slot->tt_port; - - /* Populate control endpoint context */ - ep_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_EP0 ) ); - ep_ctx->type = XHCI_EP_TYPE_CONTROL; - ep_ctx->burst = endpoint->ep->burst; - ep_ctx->mtu = cpu_to_le16 ( endpoint->ep->mtu ); - ep_ctx->dequeue = cpu_to_le64 ( virt_to_phys ( endpoint->ring.trb ) | - XHCI_EP_DCS ); - ep_ctx->trb_len = cpu_to_le16 ( XHCI_EP0_TRB_LEN ); -} - -/** - * Address device - * - * @v xhci xHCI device - * @v slot Device slot - * @ret rc Return status code - */ -static inline int xhci_address_device ( struct xhci_device *xhci, - struct xhci_slot *slot ) { - struct usb_device *usb = slot->usb; - struct xhci_slot_context *slot_ctx; - int rc; - - /* Assign device address */ - if ( ( rc = xhci_context ( xhci, slot, slot->endpoint[XHCI_CTX_EP0], - XHCI_TRB_ADDRESS_DEVICE, - xhci_address_device_input ) ) != 0 ) - return rc; - - /* Get assigned address */ - slot_ctx = ( slot->context + - xhci_device_context_offset ( xhci, XHCI_CTX_SLOT ) ); - usb->address = slot_ctx->address; - DBGC2 ( xhci, "XHCI %s assigned address %d to %s\n", - xhci->name, usb->address, usb->name ); - - return 0; -} - -/** - * Populate configure endpoint input context - * - * @v xhci xHCI device - * @v slot Device slot - * @v endpoint Endpoint - * @v input Input context - */ -static void xhci_configure_endpoint_input ( struct xhci_device *xhci, - struct xhci_slot *slot, - struct xhci_endpoint *endpoint, - void *input ) { - struct xhci_control_context *control_ctx; - struct xhci_slot_context *slot_ctx; - struct xhci_endpoint_context *ep_ctx; - - /* Populate control context */ - control_ctx = input; - control_ctx->add = cpu_to_le32 ( ( 1 << XHCI_CTX_SLOT ) | - ( 1 << endpoint->ctx ) ); - - /* Populate slot context */ - slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT )); - slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( ( XHCI_CTX_END - 1 ), - ( slot->ports ? 1 : 0 ), - slot->psiv, 0 ) ); - slot_ctx->ports = slot->ports; - - /* Populate endpoint context */ - ep_ctx = ( input + xhci_input_context_offset ( xhci, endpoint->ctx ) ); - ep_ctx->interval = endpoint->interval; - ep_ctx->type = endpoint->type; - ep_ctx->burst = endpoint->ep->burst; - ep_ctx->mtu = cpu_to_le16 ( endpoint->ep->mtu ); - ep_ctx->dequeue = cpu_to_le64 ( virt_to_phys ( endpoint->ring.trb ) | - XHCI_EP_DCS ); - ep_ctx->trb_len = cpu_to_le16 ( endpoint->ep->mtu ); /* best guess */ -} - -/** - * Configure endpoint - * - * @v xhci xHCI device - * @v slot Device slot - * @v endpoint Endpoint - * @ret rc Return status code - */ -static inline int xhci_configure_endpoint ( struct xhci_device *xhci, - struct xhci_slot *slot, - struct xhci_endpoint *endpoint ) { - int rc; - - /* Configure endpoint */ - if ( ( rc = xhci_context ( xhci, slot, endpoint, - XHCI_TRB_CONFIGURE_ENDPOINT, - xhci_configure_endpoint_input ) ) != 0 ) - return rc; - - DBGC2 ( xhci, "XHCI %s slot %d ctx %d configured\n", - xhci->name, slot->id, endpoint->ctx ); - return 0; -} - -/** - * Populate deconfigure endpoint input context - * - * @v xhci xHCI device - * @v slot Device slot - * @v endpoint Endpoint - * @v input Input context - */ -static void -xhci_deconfigure_endpoint_input ( struct xhci_device *xhci __unused, - struct xhci_slot *slot __unused, - struct xhci_endpoint *endpoint, - void *input ) { - struct xhci_control_context *control_ctx; - struct xhci_slot_context *slot_ctx; - - /* Populate control context */ - control_ctx = input; - control_ctx->add = cpu_to_le32 ( 1 << XHCI_CTX_SLOT ); - control_ctx->drop = cpu_to_le32 ( 1 << endpoint->ctx ); - - /* Populate slot context */ - slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT )); - slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( ( XHCI_CTX_END - 1 ), - 0, 0, 0 ) ); -} - -/** - * Deconfigure endpoint - * - * @v xhci xHCI device - * @v slot Device slot - * @v endpoint Endpoint - * @ret rc Return status code - */ -static inline int xhci_deconfigure_endpoint ( struct xhci_device *xhci, - struct xhci_slot *slot, - struct xhci_endpoint *endpoint ) { - int rc; - - /* Deconfigure endpoint */ - if ( ( rc = xhci_context ( xhci, slot, endpoint, - XHCI_TRB_CONFIGURE_ENDPOINT, - xhci_deconfigure_endpoint_input ) ) != 0 ) - return rc; - - DBGC2 ( xhci, "XHCI %s slot %d ctx %d deconfigured\n", - xhci->name, slot->id, endpoint->ctx ); - return 0; -} - -/** - * Populate evaluate context input context - * - * @v xhci xHCI device - * @v slot Device slot - * @v endpoint Endpoint - * @v input Input context - */ -static void xhci_evaluate_context_input ( struct xhci_device *xhci, - struct xhci_slot *slot __unused, - struct xhci_endpoint *endpoint, - void *input ) { - struct xhci_control_context *control_ctx; - struct xhci_slot_context *slot_ctx; - struct xhci_endpoint_context *ep_ctx; - - /* Populate control context */ - control_ctx = input; - control_ctx->add = cpu_to_le32 ( ( 1 << XHCI_CTX_SLOT ) | - ( 1 << endpoint->ctx ) ); - - /* Populate slot context */ - slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT )); - slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( ( XHCI_CTX_END - 1 ), - 0, 0, 0 ) ); - - /* Populate endpoint context */ - ep_ctx = ( input + xhci_input_context_offset ( xhci, endpoint->ctx ) ); - ep_ctx->mtu = cpu_to_le16 ( endpoint->ep->mtu ); -} - -/** - * Evaluate context - * - * @v xhci xHCI device - * @v slot Device slot - * @v endpoint Endpoint - * @ret rc Return status code - */ -static inline int xhci_evaluate_context ( struct xhci_device *xhci, - struct xhci_slot *slot, - struct xhci_endpoint *endpoint ) { - int rc; - - /* Configure endpoint */ - if ( ( rc = xhci_context ( xhci, slot, endpoint, - XHCI_TRB_EVALUATE_CONTEXT, - xhci_evaluate_context_input ) ) != 0 ) - return rc; - - DBGC2 ( xhci, "XHCI %s slot %d ctx %d (re-)evaluated\n", - xhci->name, slot->id, endpoint->ctx ); - return 0; -} - -/** - * Reset endpoint - * - * @v xhci xHCI device - * @v slot Device slot - * @v endpoint Endpoint - * @ret rc Return status code - */ -static inline int xhci_reset_endpoint ( struct xhci_device *xhci, - struct xhci_slot *slot, - struct xhci_endpoint *endpoint ) { - union xhci_trb trb; - struct xhci_trb_reset_endpoint *reset = &trb.reset; - int rc; - - /* Construct command */ - memset ( reset, 0, sizeof ( *reset ) ); - reset->slot = slot->id; - reset->endpoint = endpoint->ctx; - reset->type = XHCI_TRB_RESET_ENDPOINT; - - /* Issue command and wait for completion */ - if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) { - DBGC ( xhci, "XHCI %s slot %d ctx %d could not reset endpoint " - "in state %d: %s\n", xhci->name, slot->id, endpoint->ctx, - endpoint->context->state, strerror ( rc ) ); - return rc; - } - - return 0; -} - -/** - * Stop endpoint - * - * @v xhci xHCI device - * @v slot Device slot - * @v endpoint Endpoint - * @ret rc Return status code - */ -static inline int xhci_stop_endpoint ( struct xhci_device *xhci, - struct xhci_slot *slot, - struct xhci_endpoint *endpoint ) { - union xhci_trb trb; - struct xhci_trb_stop_endpoint *stop = &trb.stop; - int rc; - - /* Construct command */ - memset ( stop, 0, sizeof ( *stop ) ); - stop->slot = slot->id; - stop->endpoint = endpoint->ctx; - stop->type = XHCI_TRB_STOP_ENDPOINT; - - /* Issue command and wait for completion */ - if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) { - DBGC ( xhci, "XHCI %s slot %d ctx %d could not stop endpoint " - "in state %d: %s\n", xhci->name, slot->id, endpoint->ctx, - endpoint->context->state, strerror ( rc ) ); - return rc; - } - - return 0; -} - -/** - * Set transfer ring dequeue pointer - * - * @v xhci xHCI device - * @v slot Device slot - * @v endpoint Endpoint - * @ret rc Return status code - */ -static inline int -xhci_set_tr_dequeue_pointer ( struct xhci_device *xhci, - struct xhci_slot *slot, - struct xhci_endpoint *endpoint ) { - union xhci_trb trb; - struct xhci_trb_set_tr_dequeue_pointer *dequeue = &trb.dequeue; - struct xhci_trb_ring *ring = &endpoint->ring; - unsigned int cons; - unsigned int mask; - unsigned int index; - unsigned int dcs; - int rc; - - /* Construct command */ - memset ( dequeue, 0, sizeof ( *dequeue ) ); - cons = ring->cons; - mask = ring->mask; - dcs = ( ( ~( cons >> ring->shift ) ) & XHCI_EP_DCS ); - index = ( cons & mask ); - dequeue->dequeue = - cpu_to_le64 ( virt_to_phys ( &ring->trb[index] ) | dcs ); - dequeue->slot = slot->id; - dequeue->endpoint = endpoint->ctx; - dequeue->type = XHCI_TRB_SET_TR_DEQUEUE_POINTER; - - /* Issue command and wait for completion */ - if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) { - DBGC ( xhci, "XHCI %s slot %d ctx %d could not set TR dequeue " - "pointer in state %d: %s\n", xhci->name, slot->id, - endpoint->ctx, endpoint->context->state, strerror ( rc)); - return rc; - } - - return 0; -} - -/****************************************************************************** - * - * Endpoint operations - * - ****************************************************************************** - */ - -/** - * Open endpoint - * - * @v ep USB endpoint - * @ret rc Return status code - */ -static int xhci_endpoint_open ( struct usb_endpoint *ep ) { - struct usb_device *usb = ep->usb; - struct xhci_slot *slot = usb_get_hostdata ( usb ); - struct xhci_device *xhci = slot->xhci; - struct xhci_endpoint *endpoint; - unsigned int ctx; - unsigned int type; - unsigned int interval; - int rc; - - /* Calculate context index */ - ctx = XHCI_CTX ( ep->address ); - assert ( slot->endpoint[ctx] == NULL ); - - /* Calculate endpoint type */ - type = XHCI_EP_TYPE ( ep->attributes & USB_ENDPOINT_ATTR_TYPE_MASK ); - if ( type == XHCI_EP_TYPE ( USB_ENDPOINT_ATTR_CONTROL ) ) - type = XHCI_EP_TYPE_CONTROL; - if ( ep->address & USB_DIR_IN ) - type |= XHCI_EP_TYPE_IN; - - /* Calculate interval */ - if ( type & XHCI_EP_TYPE_PERIODIC ) { - interval = ( fls ( ep->interval ) - 1 ); - } else { - interval = ep->interval; - } - - /* Allocate and initialise structure */ - endpoint = zalloc ( sizeof ( *endpoint ) ); - if ( ! endpoint ) { - rc = -ENOMEM; - goto err_alloc; - } - usb_endpoint_set_hostdata ( ep, endpoint ); - slot->endpoint[ctx] = endpoint; - endpoint->xhci = xhci; - endpoint->slot = slot; - endpoint->ep = ep; - endpoint->ctx = ctx; - endpoint->type = type; - endpoint->interval = interval; - endpoint->context = ( ( ( void * ) slot->context ) + - xhci_device_context_offset ( xhci, ctx ) ); - - /* Allocate transfer ring */ - if ( ( rc = xhci_ring_alloc ( xhci, &endpoint->ring, - XHCI_TRANSFER_TRBS_LOG2, - slot->id, ctx, 0 ) ) != 0 ) - goto err_ring_alloc; - - /* Configure endpoint, if applicable */ - if ( ( ctx != XHCI_CTX_EP0 ) && - ( ( rc = xhci_configure_endpoint ( xhci, slot, endpoint ) ) != 0 )) - goto err_configure_endpoint; - - DBGC2 ( xhci, "XHCI %s slot %d ctx %d ring [%08lx,%08lx)\n", - xhci->name, slot->id, ctx, virt_to_phys ( endpoint->ring.trb ), - ( virt_to_phys ( endpoint->ring.trb ) + endpoint->ring.len ) ); - return 0; - - xhci_deconfigure_endpoint ( xhci, slot, endpoint ); - err_configure_endpoint: - xhci_ring_free ( &endpoint->ring ); - err_ring_alloc: - slot->endpoint[ctx] = NULL; - free ( endpoint ); - err_alloc: - return rc; -} - -/** - * Close endpoint - * - * @v ep USB endpoint - */ -static void xhci_endpoint_close ( struct usb_endpoint *ep ) { - struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep ); - struct xhci_slot *slot = endpoint->slot; - struct xhci_device *xhci = slot->xhci; - struct io_buffer *iobuf; - unsigned int ctx = endpoint->ctx; - - /* Deconfigure endpoint, if applicable */ - if ( ctx != XHCI_CTX_EP0 ) - xhci_deconfigure_endpoint ( xhci, slot, endpoint ); - - /* Cancel any incomplete transfers */ - while ( xhci_ring_fill ( &endpoint->ring ) ) { - iobuf = xhci_dequeue_multi ( &endpoint->ring ); - usb_complete_err ( ep, iobuf, -ECANCELED ); - } - - /* Free endpoint */ - xhci_ring_free ( &endpoint->ring ); - slot->endpoint[ctx] = NULL; - free ( endpoint ); -} - -/** - * Reset endpoint - * - * @v ep USB endpoint - * @ret rc Return status code - */ -static int xhci_endpoint_reset ( struct usb_endpoint *ep ) { - struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep ); - struct xhci_slot *slot = endpoint->slot; - struct xhci_device *xhci = slot->xhci; - int rc; - - /* Reset endpoint context */ - if ( ( rc = xhci_reset_endpoint ( xhci, slot, endpoint ) ) != 0 ) - return rc; - - /* Set transfer ring dequeue pointer */ - if ( ( rc = xhci_set_tr_dequeue_pointer ( xhci, slot, endpoint ) ) != 0) - return rc; - - /* Ring doorbell to resume processing */ - xhci_doorbell ( &endpoint->ring ); - - DBGC ( xhci, "XHCI %s slot %d ctx %d reset\n", - xhci->name, slot->id, endpoint->ctx ); - return 0; -} - -/** - * Update MTU - * - * @v ep USB endpoint - * @ret rc Return status code - */ -static int xhci_endpoint_mtu ( struct usb_endpoint *ep ) { - struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep ); - struct xhci_slot *slot = endpoint->slot; - struct xhci_device *xhci = slot->xhci; - int rc; - - /* Evalulate context */ - if ( ( rc = xhci_evaluate_context ( xhci, slot, endpoint ) ) != 0 ) - return rc; - - return 0; -} - -/** - * Enqueue message transfer - * - * @v ep USB endpoint - * @v iobuf I/O buffer - * @ret rc Return status code - */ -static int xhci_endpoint_message ( struct usb_endpoint *ep, - struct io_buffer *iobuf ) { - struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep ); - struct usb_setup_packet *packet; - unsigned int input; - size_t len; - union xhci_trb trbs[ 1 /* setup */ + 1 /* possible data */ + - 1 /* status */ ]; - union xhci_trb *trb = trbs; - struct xhci_trb_setup *setup; - struct xhci_trb_data *data; - struct xhci_trb_status *status; - int rc; - - /* Profile message transfers */ - profile_start ( &xhci_message_profiler ); - - /* Construct setup stage TRB */ - memset ( trbs, 0, sizeof ( trbs ) ); - assert ( iob_len ( iobuf ) >= sizeof ( *packet ) ); - packet = iobuf->data; - iob_pull ( iobuf, sizeof ( *packet ) ); - setup = &(trb++)->setup; - memcpy ( &setup->packet, packet, sizeof ( setup->packet ) ); - setup->len = cpu_to_le32 ( sizeof ( *packet ) ); - setup->flags = XHCI_TRB_IDT; - setup->type = XHCI_TRB_SETUP; - len = iob_len ( iobuf ); - input = ( packet->request & cpu_to_le16 ( USB_DIR_IN ) ); - if ( len ) - setup->direction = ( input ? XHCI_SETUP_IN : XHCI_SETUP_OUT ); - - /* Construct data stage TRB, if applicable */ - if ( len ) { - data = &(trb++)->data; - data->data = cpu_to_le64 ( virt_to_phys ( iobuf->data ) ); - data->len = cpu_to_le32 ( len ); - data->type = XHCI_TRB_DATA; - data->direction = ( input ? XHCI_DATA_IN : XHCI_DATA_OUT ); - } - - /* Construct status stage TRB */ - status = &(trb++)->status; - status->flags = XHCI_TRB_IOC; - status->type = XHCI_TRB_STATUS; - status->direction = - ( ( len && input ) ? XHCI_STATUS_OUT : XHCI_STATUS_IN ); - - /* Enqueue TRBs */ - if ( ( rc = xhci_enqueue_multi ( &endpoint->ring, iobuf, trbs, - ( trb - trbs ) ) ) != 0 ) - return rc; - - /* Ring the doorbell */ - xhci_doorbell ( &endpoint->ring ); - - profile_stop ( &xhci_message_profiler ); - return 0; -} - -/** - * Enqueue stream transfer - * - * @v ep USB endpoint - * @v iobuf I/O buffer - * @v terminate Terminate using a short packet - * @ret rc Return status code - */ -static int xhci_endpoint_stream ( struct usb_endpoint *ep, - struct io_buffer *iobuf, int terminate ) { - struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep ); - union xhci_trb trbs[ 1 /* Normal */ + 1 /* Possible zero-length */ ]; - union xhci_trb *trb = trbs; - struct xhci_trb_normal *normal; - size_t len = iob_len ( iobuf ); - int rc; - - /* Profile stream transfers */ - profile_start ( &xhci_stream_profiler ); - - /* Construct normal TRBs */ - memset ( &trbs, 0, sizeof ( trbs ) ); - normal = &(trb++)->normal; - normal->data = cpu_to_le64 ( virt_to_phys ( iobuf->data ) ); - normal->len = cpu_to_le32 ( len ); - normal->type = XHCI_TRB_NORMAL; - if ( terminate && ( ( len & ( ep->mtu - 1 ) ) == 0 ) ) { - normal->flags = XHCI_TRB_CH; - normal = &(trb++)->normal; - normal->type = XHCI_TRB_NORMAL; - } - normal->flags = XHCI_TRB_IOC; - - /* Enqueue TRBs */ - if ( ( rc = xhci_enqueue_multi ( &endpoint->ring, iobuf, trbs, - ( trb - trbs ) ) ) != 0 ) - return rc; - - /* Ring the doorbell */ - xhci_doorbell ( &endpoint->ring ); - - profile_stop ( &xhci_stream_profiler ); - return 0; -} - -/****************************************************************************** - * - * Device operations - * - ****************************************************************************** - */ - -/** - * Open device - * - * @v usb USB device - * @ret rc Return status code - */ -static int xhci_device_open ( struct usb_device *usb ) { - struct xhci_device *xhci = usb_bus_get_hostdata ( usb->port->hub->bus ); - struct usb_port *tt = usb_transaction_translator ( usb ); - struct xhci_slot *slot; - struct xhci_slot *tt_slot; - size_t len; - int type; - int id; - int rc; - - /* Determine applicable slot type */ - type = xhci_port_slot_type ( xhci, usb->port->address ); - if ( type < 0 ) { - rc = type; - DBGC ( xhci, "XHCI %s-%d has no slot type\n", - xhci->name, usb->port->address ); - goto err_type; - } - - /* Allocate a device slot number */ - id = xhci_enable_slot ( xhci, type ); - if ( id < 0 ) { - rc = id; - goto err_enable_slot; - } - assert ( ( id > 0 ) && ( ( unsigned int ) id <= xhci->slots ) ); - assert ( xhci->slot[id] == NULL ); - - /* Allocate and initialise structure */ - slot = zalloc ( sizeof ( *slot ) ); - if ( ! slot ) { - rc = -ENOMEM; - goto err_alloc; - } - usb_set_hostdata ( usb, slot ); - xhci->slot[id] = slot; - slot->xhci = xhci; - slot->usb = usb; - slot->id = id; - if ( tt ) { - tt_slot = usb_get_hostdata ( tt->hub->usb ); - slot->tt_id = tt_slot->id; - slot->tt_port = tt->address; - } - - /* Allocate a device context */ - len = xhci_device_context_offset ( xhci, XHCI_CTX_END ); - slot->context = malloc_dma ( len, xhci_align ( len ) ); - if ( ! slot->context ) { - rc = -ENOMEM; - goto err_alloc_context; - } - memset ( slot->context, 0, len ); - - /* Set device context base address */ - assert ( xhci->dcbaa[id] == 0 ); - xhci->dcbaa[id] = cpu_to_le64 ( virt_to_phys ( slot->context ) ); - - DBGC2 ( xhci, "XHCI %s slot %d device context [%08lx,%08lx) for %s\n", - xhci->name, slot->id, virt_to_phys ( slot->context ), - ( virt_to_phys ( slot->context ) + len ), usb->name ); - return 0; - - xhci->dcbaa[id] = 0; - free_dma ( slot->context, len ); - err_alloc_context: - xhci->slot[id] = NULL; - free ( slot ); - err_alloc: - xhci_disable_slot ( xhci, id ); - err_enable_slot: - err_type: - return rc; -} - -/** - * Close device - * - * @v usb USB device - */ -static void xhci_device_close ( struct usb_device *usb ) { - struct xhci_slot *slot = usb_get_hostdata ( usb ); - struct xhci_device *xhci = slot->xhci; - size_t len = xhci_device_context_offset ( xhci, XHCI_CTX_END ); - unsigned int id = slot->id; - int rc; - - /* Disable slot */ - if ( ( rc = xhci_disable_slot ( xhci, id ) ) != 0 ) { - /* Slot is still enabled. Leak the slot context, - * since the controller may still write to this - * memory, and leave the DCBAA entry intact. - * - * If the controller later reports that this same slot - * has been re-enabled, then some assertions will be - * triggered. - */ - DBGC ( xhci, "XHCI %s slot %d leaking context memory\n", - xhci->name, slot->id ); - slot->context = NULL; - } - - /* Free slot */ - if ( slot->context ) { - free_dma ( slot->context, len ); - xhci->dcbaa[id] = 0; - } - xhci->slot[id] = NULL; - free ( slot ); -} - -/** - * Assign device address - * - * @v usb USB device - * @ret rc Return status code - */ -static int xhci_device_address ( struct usb_device *usb ) { - struct xhci_slot *slot = usb_get_hostdata ( usb ); - struct xhci_device *xhci = slot->xhci; - struct usb_port *port = usb->port; - struct usb_port *root_port; - int psiv; - int rc; - - /* Calculate route string */ - slot->route = usb_route_string ( usb ); - - /* Calculate root hub port number */ - root_port = usb_root_hub_port ( usb ); - slot->port = root_port->address; - - /* Calculate protocol speed ID */ - psiv = xhci_port_psiv ( xhci, slot->port, port->speed ); - if ( psiv < 0 ) { - rc = psiv; - return rc; - } - slot->psiv = psiv; - - /* Address device */ - if ( ( rc = xhci_address_device ( xhci, slot ) ) != 0 ) - return rc; - - return 0; -} - -/****************************************************************************** - * - * Bus operations - * - ****************************************************************************** - */ - -/** - * Open USB bus - * - * @v bus USB bus - * @ret rc Return status code - */ -static int xhci_bus_open ( struct usb_bus *bus ) { - struct xhci_device *xhci = usb_bus_get_hostdata ( bus ); - int rc; - - /* Allocate device slot array */ - xhci->slot = zalloc ( ( xhci->slots + 1 ) * sizeof ( xhci->slot[0] ) ); - if ( ! xhci->slot ) { - rc = -ENOMEM; - goto err_slot_alloc; - } - - /* Allocate device context base address array */ - if ( ( rc = xhci_dcbaa_alloc ( xhci ) ) != 0 ) - goto err_dcbaa_alloc; - - /* Allocate scratchpad buffers */ - if ( ( rc = xhci_scratchpad_alloc ( xhci ) ) != 0 ) - goto err_scratchpad_alloc; - - /* Allocate command ring */ - if ( ( rc = xhci_command_alloc ( xhci ) ) != 0 ) - goto err_command_alloc; - - /* Allocate event ring */ - if ( ( rc = xhci_event_alloc ( xhci ) ) != 0 ) - goto err_event_alloc; - - /* Start controller */ - xhci_run ( xhci ); - - return 0; - - xhci_stop ( xhci ); - xhci_event_free ( xhci ); - err_event_alloc: - xhci_command_free ( xhci ); - err_command_alloc: - xhci_scratchpad_free ( xhci ); - err_scratchpad_alloc: - xhci_dcbaa_free ( xhci ); - err_dcbaa_alloc: - free ( xhci->slot ); - err_slot_alloc: - return rc; -} - -/** - * Close USB bus - * - * @v bus USB bus - */ -static void xhci_bus_close ( struct usb_bus *bus ) { - struct xhci_device *xhci = usb_bus_get_hostdata ( bus ); - unsigned int i; - - /* Sanity checks */ - assert ( xhci->slot != NULL ); - for ( i = 0 ; i <= xhci->slots ; i++ ) - assert ( xhci->slot[i] == NULL ); - - xhci_stop ( xhci ); - xhci_event_free ( xhci ); - xhci_command_free ( xhci ); - xhci_scratchpad_free ( xhci ); - xhci_dcbaa_free ( xhci ); - free ( xhci->slot ); -} - -/** - * Poll USB bus - * - * @v bus USB bus - */ -static void xhci_bus_poll ( struct usb_bus *bus ) { - struct xhci_device *xhci = usb_bus_get_hostdata ( bus ); - - /* Poll event ring */ - xhci_event_poll ( xhci ); -} - -/****************************************************************************** - * - * Hub operations - * - ****************************************************************************** - */ - -/** - * Open hub - * - * @v hub USB hub - * @ret rc Return status code - */ -static int xhci_hub_open ( struct usb_hub *hub ) { - struct xhci_slot *slot; - - /* Do nothing if this is the root hub */ - if ( ! hub->usb ) - return 0; - - /* Get device slot */ - slot = usb_get_hostdata ( hub->usb ); - - /* Update device slot hub parameters. We don't inform the - * hardware of this information until the hub's interrupt - * endpoint is opened, since the only mechanism for so doing - * provided by the xHCI specification is a Configure Endpoint - * command, and we can't issue that command until we have a - * non-EP0 endpoint to configure. - */ - slot->ports = hub->ports; - - return 0; -} - -/** - * Close hub - * - * @v hub USB hub - */ -static void xhci_hub_close ( struct usb_hub *hub __unused ) { - - /* Nothing to do */ -} - -/****************************************************************************** - * - * Root hub operations - * - ****************************************************************************** - */ - -/** - * Open root hub - * - * @v hub USB hub - * @ret rc Return status code - */ -static int xhci_root_open ( struct usb_hub *hub ) { - struct usb_bus *bus = hub->bus; - struct xhci_device *xhci = usb_bus_get_hostdata ( bus ); - struct usb_port *port; - uint32_t portsc; - unsigned int i; - - /* Enable power to all ports */ - for ( i = 1 ; i <= xhci->ports ; i++ ) { - portsc = readl ( xhci->op + XHCI_OP_PORTSC ( i ) ); - portsc &= XHCI_PORTSC_PRESERVE; - portsc |= XHCI_PORTSC_PP; - writel ( portsc, xhci->op + XHCI_OP_PORTSC ( i ) ); - } - - /* xHCI spec requires us to potentially wait 20ms after - * enabling power to a port. - */ - mdelay ( XHCI_PORT_POWER_DELAY_MS ); - - /* USB3 ports may power up as Disabled */ - for ( i = 1 ; i <= xhci->ports ; i++ ) { - portsc = readl ( xhci->op + XHCI_OP_PORTSC ( i ) ); - port = usb_port ( hub, i ); - if ( ( port->protocol >= USB_PROTO_3_0 ) && - ( ( portsc & XHCI_PORTSC_PLS_MASK ) == - XHCI_PORTSC_PLS_DISABLED ) ) { - /* Force link state to RxDetect */ - portsc &= XHCI_PORTSC_PRESERVE; - portsc |= ( XHCI_PORTSC_PLS_RXDETECT | XHCI_PORTSC_LWS); - writel ( portsc, xhci->op + XHCI_OP_PORTSC ( i ) ); - } - } - - /* Some xHCI cards seem to require an additional delay after - * setting the link state to RxDetect. - */ - mdelay ( XHCI_LINK_STATE_DELAY_MS ); - - /* Record hub driver private data */ - usb_hub_set_drvdata ( hub, xhci ); - - return 0; -} - -/** - * Close root hub - * - * @v hub USB hub - */ -static void xhci_root_close ( struct usb_hub *hub ) { - - /* Clear hub driver private data */ - usb_hub_set_drvdata ( hub, NULL ); -} - -/** - * Enable port - * - * @v hub USB hub - * @v port USB port - * @ret rc Return status code - */ -static int xhci_root_enable ( struct usb_hub *hub, struct usb_port *port ) { - struct xhci_device *xhci = usb_hub_get_drvdata ( hub ); - uint32_t portsc; - unsigned int i; - - /* Reset port */ - portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) ); - portsc &= XHCI_PORTSC_PRESERVE; - portsc |= XHCI_PORTSC_PR; - writel ( portsc, xhci->op + XHCI_OP_PORTSC ( port->address ) ); - - /* Wait for port to become enabled */ - for ( i = 0 ; i < XHCI_PORT_RESET_MAX_WAIT_MS ; i++ ) { - - /* Check port status */ - portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) ); - if ( portsc & XHCI_PORTSC_PED ) - return 0; - - /* Delay */ - mdelay ( 1 ); - } - - DBGC ( xhci, "XHCI %s-%d timed out waiting for port to enable\n", - xhci->name, port->address ); - return -ETIMEDOUT; -} - -/** - * Disable port - * - * @v hub USB hub - * @v port USB port - * @ret rc Return status code - */ -static int xhci_root_disable ( struct usb_hub *hub, struct usb_port *port ) { - struct xhci_device *xhci = usb_hub_get_drvdata ( hub ); - uint32_t portsc; - - /* Disable port */ - portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) ); - portsc &= XHCI_PORTSC_PRESERVE; - portsc |= XHCI_PORTSC_PED; - writel ( portsc, xhci->op + XHCI_OP_PORTSC ( port->address ) ); - - return 0; -} - -/** - * Update root hub port speed - * - * @v hub USB hub - * @v port USB port - * @ret rc Return status code - */ -static int xhci_root_speed ( struct usb_hub *hub, struct usb_port *port ) { - struct xhci_device *xhci = usb_hub_get_drvdata ( hub ); - uint32_t portsc; - unsigned int psiv; - int ccs; - int ped; - int csc; - int speed; - int rc; - - /* Read port status */ - portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) ); - DBGC2 ( xhci, "XHCI %s-%d status is %08x\n", - xhci->name, port->address, portsc ); - ccs = ( portsc & XHCI_PORTSC_CCS ); - ped = ( portsc & XHCI_PORTSC_PED ); - csc = ( portsc & XHCI_PORTSC_CSC ); - psiv = XHCI_PORTSC_PSIV ( portsc ); - - /* Record disconnections and clear changes */ - port->disconnected |= csc; - portsc &= ( XHCI_PORTSC_PRESERVE | XHCI_PORTSC_CHANGE ); - writel ( portsc, xhci->op + XHCI_OP_PORTSC ( port->address ) ); - - /* Port speed is not valid unless port is connected */ - if ( ! ccs ) { - port->speed = USB_SPEED_NONE; - return 0; - } - - /* For USB2 ports, the PSIV field is not valid until the port - * completes reset and becomes enabled. - */ - if ( ( port->protocol < USB_PROTO_3_0 ) && ! ped ) { - port->speed = USB_SPEED_FULL; - return 0; - } - - /* Get port speed and map to generic USB speed */ - speed = xhci_port_speed ( xhci, port->address, psiv ); - if ( speed < 0 ) { - rc = speed; - return rc; - } - - port->speed = speed; - return 0; -} - -/** - * Clear transaction translator buffer - * - * @v hub USB hub - * @v port USB port - * @v ep USB endpoint - * @ret rc Return status code - */ -static int xhci_root_clear_tt ( struct usb_hub *hub, struct usb_port *port, - struct usb_endpoint *ep ) { - struct xhci_device *xhci = usb_hub_get_drvdata ( hub ); - - /* Should never be called; this is a root hub */ - DBGC ( xhci, "XHCI %s-%d nonsensical CLEAR_TT for %s %s\n", xhci->name, - port->address, ep->usb->name, usb_endpoint_name ( ep ) ); - - return -ENOTSUP; -} - -/****************************************************************************** - * - * PCI interface - * - ****************************************************************************** - */ - -/** USB host controller operations */ -static struct usb_host_operations xhci_operations = { - .endpoint = { - .open = xhci_endpoint_open, - .close = xhci_endpoint_close, - .reset = xhci_endpoint_reset, - .mtu = xhci_endpoint_mtu, - .message = xhci_endpoint_message, - .stream = xhci_endpoint_stream, - }, - .device = { - .open = xhci_device_open, - .close = xhci_device_close, - .address = xhci_device_address, - }, - .bus = { - .open = xhci_bus_open, - .close = xhci_bus_close, - .poll = xhci_bus_poll, - }, - .hub = { - .open = xhci_hub_open, - .close = xhci_hub_close, - }, - .root = { - .open = xhci_root_open, - .close = xhci_root_close, - .enable = xhci_root_enable, - .disable = xhci_root_disable, - .speed = xhci_root_speed, - .clear_tt = xhci_root_clear_tt, - }, -}; - -/** - * Fix Intel PCH-specific quirks - * - * @v xhci xHCI device - * @v pci PCI device - */ -static void xhci_pch_fix ( struct xhci_device *xhci, struct pci_device *pci ) { - struct xhci_pch *pch = &xhci->pch; - uint32_t xusb2pr; - uint32_t xusb2prm; - uint32_t usb3pssen; - uint32_t usb3prm; - - /* Enable SuperSpeed capability. Do this before rerouting - * USB2 ports, so that USB3 devices connect at SuperSpeed. - */ - pci_read_config_dword ( pci, XHCI_PCH_USB3PSSEN, &usb3pssen ); - pci_read_config_dword ( pci, XHCI_PCH_USB3PRM, &usb3prm ); - if ( usb3prm & ~usb3pssen ) { - DBGC ( xhci, "XHCI %s enabling SuperSpeed on ports %08x\n", - xhci->name, ( usb3prm & ~usb3pssen ) ); - } - pch->usb3pssen = usb3pssen; - usb3pssen |= usb3prm; - pci_write_config_dword ( pci, XHCI_PCH_USB3PSSEN, usb3pssen ); - - /* Route USB2 ports from EHCI to xHCI */ - pci_read_config_dword ( pci, XHCI_PCH_XUSB2PR, &xusb2pr ); - pci_read_config_dword ( pci, XHCI_PCH_XUSB2PRM, &xusb2prm ); - if ( xusb2prm & ~xusb2pr ) { - DBGC ( xhci, "XHCI %s routing ports %08x from EHCI to xHCI\n", - xhci->name, ( xusb2prm & ~xusb2pr ) ); - } - pch->xusb2pr = xusb2pr; - xusb2pr |= xusb2prm; - pci_write_config_dword ( pci, XHCI_PCH_XUSB2PR, xusb2pr ); -} - -/** - * Undo Intel PCH-specific quirk fixes - * - * @v xhci xHCI device - * @v pci PCI device - */ -static void xhci_pch_undo ( struct xhci_device *xhci, struct pci_device *pci ) { - struct xhci_pch *pch = &xhci->pch; - - /* Restore USB2 port routing to original state */ - pci_write_config_dword ( pci, XHCI_PCH_XUSB2PR, pch->xusb2pr ); - - /* Restore SuperSpeed capability to original state */ - pci_write_config_dword ( pci, XHCI_PCH_USB3PSSEN, pch->usb3pssen ); -} - -/** - * Probe PCI device - * - * @v pci PCI device - * @ret rc Return status code - */ -static int xhci_probe ( struct pci_device *pci ) { - struct xhci_device *xhci; - struct usb_port *port; - unsigned long bar_start; - size_t bar_size; - unsigned int i; - int rc; - - /* Allocate and initialise structure */ - xhci = zalloc ( sizeof ( *xhci ) ); - if ( ! xhci ) { - rc = -ENOMEM; - goto err_alloc; - } - xhci->name = pci->dev.name; - xhci->quirks = pci->id->driver_data; - - /* Fix up PCI device */ - adjust_pci_device ( pci ); - - /* Map registers */ - bar_start = pci_bar_start ( pci, XHCI_BAR ); - bar_size = pci_bar_size ( pci, XHCI_BAR ); - xhci->regs = ioremap ( bar_start, bar_size ); - if ( ! xhci->regs ) { - rc = -ENODEV; - goto err_ioremap; - } - - /* Initialise xHCI device */ - xhci_init ( xhci, xhci->regs ); - - /* Initialise USB legacy support and claim ownership */ - xhci_legacy_init ( xhci ); - xhci_legacy_claim ( xhci ); - - /* Fix Intel PCH-specific quirks, if applicable */ - if ( xhci->quirks & XHCI_PCH ) - xhci_pch_fix ( xhci, pci ); - - /* Reset device */ - if ( ( rc = xhci_reset ( xhci ) ) != 0 ) - goto err_reset; - - /* Allocate USB bus */ - xhci->bus = alloc_usb_bus ( &pci->dev, xhci->ports, XHCI_MTU, - &xhci_operations ); - if ( ! xhci->bus ) { - rc = -ENOMEM; - goto err_alloc_bus; - } - usb_bus_set_hostdata ( xhci->bus, xhci ); - usb_hub_set_drvdata ( xhci->bus->hub, xhci ); - - /* Set port protocols */ - for ( i = 1 ; i <= xhci->ports ; i++ ) { - port = usb_port ( xhci->bus->hub, i ); - port->protocol = xhci_port_protocol ( xhci, i ); - } - - /* Register USB bus */ - if ( ( rc = register_usb_bus ( xhci->bus ) ) != 0 ) - goto err_register; - - pci_set_drvdata ( pci, xhci ); - return 0; - - unregister_usb_bus ( xhci->bus ); - err_register: - free_usb_bus ( xhci->bus ); - err_alloc_bus: - xhci_reset ( xhci ); - err_reset: - if ( xhci->quirks & XHCI_PCH ) - xhci_pch_undo ( xhci, pci ); - xhci_legacy_release ( xhci ); - iounmap ( xhci->regs ); - err_ioremap: - free ( xhci ); - err_alloc: - return rc; -} - -/** - * Remove PCI device - * - * @v pci PCI device - */ -static void xhci_remove ( struct pci_device *pci ) { - struct xhci_device *xhci = pci_get_drvdata ( pci ); - struct usb_bus *bus = xhci->bus; - - unregister_usb_bus ( bus ); - free_usb_bus ( bus ); - xhci_reset ( xhci ); - if ( xhci->quirks & XHCI_PCH ) - xhci_pch_undo ( xhci, pci ); - xhci_legacy_release ( xhci ); - iounmap ( xhci->regs ); - free ( xhci ); -} - -/** XHCI PCI device IDs */ -static struct pci_device_id xhci_ids[] = { - PCI_ROM ( 0x8086, 0x9d2f, "xhci-skylake", "xHCI (Skylake)", ( XHCI_PCH | XHCI_BAD_PSIV ) ), - PCI_ROM ( 0x8086, 0xffff, "xhci-pch", "xHCI (Intel PCH)", XHCI_PCH ), - PCI_ROM ( 0xffff, 0xffff, "xhci", "xHCI", 0 ), -}; - -/** XHCI PCI driver */ -struct pci_driver xhci_driver __pci_driver = { - .ids = xhci_ids, - .id_count = ( sizeof ( xhci_ids ) / sizeof ( xhci_ids[0] ) ), - .class = PCI_CLASS_ID ( PCI_CLASS_SERIAL, PCI_CLASS_SERIAL_USB, - PCI_CLASS_SERIAL_USB_XHCI ), - .probe = xhci_probe, - .remove = xhci_remove, -}; - -/** - * Prepare for exit - * - * @v booting System is shutting down for OS boot - */ -static void xhci_shutdown ( int booting ) { - /* If we are shutting down to boot an OS, then prevent the - * release of ownership back to BIOS. - */ - xhci_legacy_prevent_release = booting; -} - -/** Startup/shutdown function */ -struct startup_fn xhci_startup __startup_fn ( STARTUP_LATE ) = { - .shutdown = xhci_shutdown, -}; |