Add qemu 2.4.0

Change-Id: Ic99cbad4b61f8b127b7dc74d04576c0bcbaaf4f5
Signed-off-by: Yang Zhang <yang.z.zhang@intel.com>

1 files changed, 2435 insertions, 0 deletions

diff --git a/qemu/roms/ipxe/src/drivers/infiniband/qib7322.c b/qemu/roms/ipxe/src/drivers/infiniband/qib7322.c
new file mode 100644
index 000000000..9979b346e
--- /dev/null
+++ b/qemu/roms/ipxe/src/drivers/infiniband/qib7322.c
@@ -0,0 +1,2435 @@
+/*
+ * Copyright (C) 2009 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER );
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <unistd.h>
+#include <assert.h>
+#include <ipxe/io.h>
+#include <ipxe/pci.h>
+#include <ipxe/infiniband.h>
+#include <ipxe/i2c.h>
+#include <ipxe/bitbash.h>
+#include <ipxe/malloc.h>
+#include <ipxe/iobuf.h>
+#include <ipxe/pcibackup.h>
+#include "qib7322.h"
+
+/**
+ * @file
+ *
+ * QLogic QIB7322 Infiniband HCA
+ *
+ */
+
+/** A QIB7322 send buffer set */
+struct qib7322_send_buffers {
+	/** Offset within register space of the first send buffer */
+	unsigned long base;
+	/** Send buffer size */
+	unsigned int size;
+	/** Index of first send buffer */
+	unsigned int start;
+	/** Number of send buffers
+	 *
+	 * Must be a power of two.
+	 */
+	unsigned int count;
+	/** Send buffer availability producer counter */
+	unsigned int prod;
+	/** Send buffer availability consumer counter */
+	unsigned int cons;
+	/** Send buffer availability */
+	uint16_t avail[0];
+};
+
+/** A QIB7322 send work queue */
+struct qib7322_send_work_queue {
+	/** Send buffer set */
+	struct qib7322_send_buffers *send_bufs;
+	/** Send buffer usage */
+	uint16_t *used;
+	/** Producer index */
+	unsigned int prod;
+	/** Consumer index */
+	unsigned int cons;
+};
+
+/** A QIB7322 receive work queue */
+struct qib7322_recv_work_queue {
+	/** Receive header ring */
+	void *header;
+	/** Receive header producer offset (written by hardware) */
+	struct QIB_7322_scalar header_prod;
+	/** Receive header consumer offset */
+	unsigned int header_cons;
+	/** Offset within register space of the eager array */
+	unsigned long eager_array;
+	/** Number of entries in eager array */
+	unsigned int eager_entries;
+	/** Eager array producer index */
+	unsigned int eager_prod;
+	/** Eager array consumer index */
+	unsigned int eager_cons;
+};
+
+/** A QIB7322 HCA */
+struct qib7322 {
+	/** Registers */
+	void *regs;
+
+	/** In-use contexts */
+	uint8_t used_ctx[QIB7322_NUM_CONTEXTS];
+	/** Send work queues */
+	struct qib7322_send_work_queue send_wq[QIB7322_NUM_CONTEXTS];
+	/** Receive work queues */
+	struct qib7322_recv_work_queue recv_wq[QIB7322_NUM_CONTEXTS];
+
+	/** Send buffer availability (reported by hardware) */
+	struct QIB_7322_SendBufAvail *sendbufavail;
+	/** Small send buffers */
+	struct qib7322_send_buffers *send_bufs_small;
+	/** VL15 port 0 send buffers */
+	struct qib7322_send_buffers *send_bufs_vl15_port0;
+	/** VL15 port 1 send buffers */
+	struct qib7322_send_buffers *send_bufs_vl15_port1;
+
+	/** I2C bit-bashing interface */
+	struct i2c_bit_basher i2c;
+	/** I2C serial EEPROM */
+	struct i2c_device eeprom;
+
+	/** Base GUID */
+	union ib_guid guid;
+	/** Infiniband devices */
+	struct ib_device *ibdev[QIB7322_MAX_PORTS];
+};
+
+/***************************************************************************
+ *
+ * QIB7322 register access
+ *
+ ***************************************************************************
+ *
+ * This card requires atomic 64-bit accesses.  Strange things happen
+ * if you try to use 32-bit accesses; sometimes they work, sometimes
+ * they don't, sometimes you get random data.
+ *
+ * These accessors use the "movq" MMX instruction, and so won't work
+ * on really old Pentiums (which won't have PCIe anyway, so this is
+ * something of a moot point).
+ */
+
+/**
+ * Read QIB7322 qword register
+ *
+ * @v qib7322		QIB7322 device
+ * @v dwords		Register buffer to read into
+ * @v offset		Register offset
+ */
+static void qib7322_readq ( struct qib7322 *qib7322, uint32_t *dwords,
+			    unsigned long offset ) {
+	void *addr = ( qib7322->regs + offset );
+
+	__asm__ __volatile__ ( "movq (%1), %%mm0\n\t"
+			       "movq %%mm0, (%0)\n\t"
+			       : : "r" ( dwords ), "r" ( addr ) : "memory" );
+
+	DBGIO ( "[%08lx] => %08x%08x\n",
+		virt_to_phys ( addr ), dwords[1], dwords[0] );
+}
+#define qib7322_readq( _qib7322, _ptr, _offset ) \
+	qib7322_readq ( (_qib7322), (_ptr)->u.dwords, (_offset) )
+#define qib7322_readq_array8b( _qib7322, _ptr, _offset, _idx ) \
+	qib7322_readq ( (_qib7322), (_ptr), ( (_offset) + ( (_idx) * 8 ) ) )
+#define qib7322_readq_array64k( _qib7322, _ptr, _offset, _idx ) \
+	qib7322_readq ( (_qib7322), (_ptr), ( (_offset) + ( (_idx) * 65536 ) ) )
+#define qib7322_readq_port( _qib7322, _ptr, _offset, _port ) \
+	qib7322_readq ( (_qib7322), (_ptr), ( (_offset) + ( (_port) * 4096 ) ) )
+
+/**
+ * Write QIB7322 qword register
+ *
+ * @v qib7322		QIB7322 device
+ * @v dwords		Register buffer to write
+ * @v offset		Register offset
+ */
+static void qib7322_writeq ( struct qib7322 *qib7322, const uint32_t *dwords,
+			     unsigned long offset ) {
+	void *addr = ( qib7322->regs + offset );
+
+	DBGIO ( "[%08lx] <= %08x%08x\n",
+		virt_to_phys ( addr ), dwords[1], dwords[0] );
+
+	__asm__ __volatile__ ( "movq (%0), %%mm0\n\t"
+			       "movq %%mm0, (%1)\n\t"
+			       : : "r" ( dwords ), "r" ( addr ) : "memory" );
+}
+#define qib7322_writeq( _qib7322, _ptr, _offset ) \
+	qib7322_writeq ( (_qib7322), (_ptr)->u.dwords, (_offset) )
+#define qib7322_writeq_array8b( _qib7322, _ptr, _offset, _idx ) \
+	qib7322_writeq ( (_qib7322), (_ptr), ( (_offset) + ( (_idx) * 8 ) ) )
+#define qib7322_writeq_array64k( _qib7322, _ptr, _offset, _idx ) \
+	qib7322_writeq ( (_qib7322), (_ptr), ( (_offset) + ( (_idx) * 65536 ) ))
+#define qib7322_writeq_port( _qib7322, _ptr, _offset, _port ) \
+	qib7322_writeq ( (_qib7322), (_ptr), ( (_offset) + ( (_port) * 4096 ) ))
+
+/**
+ * Write QIB7322 dword register
+ *
+ * @v qib7322		QIB7322 device
+ * @v dword		Value to write
+ * @v offset		Register offset
+ */
+static void qib7322_writel ( struct qib7322 *qib7322, uint32_t dword,
+			     unsigned long offset ) {
+	writel ( dword, ( qib7322->regs + offset ) );
+}
+
+/***************************************************************************
+ *
+ * Link state management
+ *
+ ***************************************************************************
+ */
+
+/**
+ * Textual representation of link state
+ *
+ * @v link_state	Link state
+ * @ret link_text	Link state text
+ */
+static const char * qib7322_link_state_text ( unsigned int link_state ) {
+	switch ( link_state ) {
+	case QIB7322_LINK_STATE_DOWN:		return "DOWN";
+	case QIB7322_LINK_STATE_INIT:		return "INIT";
+	case QIB7322_LINK_STATE_ARM:		return "ARM";
+	case QIB7322_LINK_STATE_ACTIVE:		return "ACTIVE";
+	case QIB7322_LINK_STATE_ACT_DEFER:	return "ACT_DEFER";
+	default:				return "UNKNOWN";
+	}
+}
+
+/**
+ * Handle link state change
+ *
+ * @v qib7322		QIB7322 device
+ */
+static void qib7322_link_state_changed ( struct ib_device *ibdev ) {
+	struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
+	struct QIB_7322_IBCStatusA_0 ibcstatusa;
+	struct QIB_7322_EXTCtrl extctrl;
+	unsigned int port = ( ibdev->port - QIB7322_PORT_BASE );
+	unsigned int link_training_state;
+	unsigned int link_state;
+	unsigned int link_width;
+	unsigned int link_speed;
+	unsigned int link_speed_qdr;
+	unsigned int green;
+	unsigned int yellow;
+
+	/* Read link state */
+	qib7322_readq_port ( qib7322, &ibcstatusa,
+			     QIB_7322_IBCStatusA_0_offset, port );
+	link_training_state = BIT_GET ( &ibcstatusa, LinkTrainingState );
+	link_state = BIT_GET ( &ibcstatusa, LinkState );
+	link_width = BIT_GET ( &ibcstatusa, LinkWidthActive );
+	link_speed = BIT_GET ( &ibcstatusa, LinkSpeedActive );
+	link_speed_qdr = BIT_GET ( &ibcstatusa, LinkSpeedQDR );
+	DBGC ( qib7322, "QIB7322 %p port %d training state %#x link state %s "
+	       "(%s %s)\n", qib7322, port, link_training_state,
+	       qib7322_link_state_text ( link_state ),
+	       ( link_speed_qdr ? "QDR" : ( link_speed ? "DDR" : "SDR" ) ),
+	       ( link_width ? "x4" : "x1" ) );
+
+	/* Set LEDs according to link state */
+	qib7322_readq ( qib7322, &extctrl, QIB_7322_EXTCtrl_offset );
+	green = ( ( link_state >= QIB7322_LINK_STATE_INIT ) ? 1 : 0 );
+	yellow = ( ( link_state >= QIB7322_LINK_STATE_ACTIVE ) ? 1 : 0 );
+	if ( port == 0 ) {
+		BIT_SET ( &extctrl, LEDPort0GreenOn, green );
+		BIT_SET ( &extctrl, LEDPort0YellowOn, yellow );
+	} else {
+		BIT_SET ( &extctrl, LEDPort1GreenOn, green );
+		BIT_SET ( &extctrl, LEDPort1YellowOn, yellow );
+	}
+	qib7322_writeq ( qib7322, &extctrl, QIB_7322_EXTCtrl_offset );
+
+	/* Notify Infiniband core of link state change */
+	ibdev->port_state = ( link_state + 1 );
+	ibdev->link_width_active =
+		( link_width ? IB_LINK_WIDTH_4X : IB_LINK_WIDTH_1X );
+	ibdev->link_speed_active =
+		( link_speed ? IB_LINK_SPEED_DDR : IB_LINK_SPEED_SDR );
+	ib_link_state_changed ( ibdev );
+}
+
+/**
+ * Wait for link state change to take effect
+ *
+ * @v ibdev		Infiniband device
+ * @v new_link_state	Expected link state
+ * @ret rc		Return status code
+ */
+static int qib7322_link_state_check ( struct ib_device *ibdev,
+				      unsigned int new_link_state ) {
+	struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
+	struct QIB_7322_IBCStatusA_0 ibcstatusa;
+	unsigned int port = ( ibdev->port - QIB7322_PORT_BASE );
+	unsigned int link_state;
+	unsigned int i;
+
+	for ( i = 0 ; i < QIB7322_LINK_STATE_MAX_WAIT_US ; i++ ) {
+		qib7322_readq_port ( qib7322, &ibcstatusa,
+				     QIB_7322_IBCStatusA_0_offset, port );
+		link_state = BIT_GET ( &ibcstatusa, LinkState );
+		if ( link_state == new_link_state )
+			return 0;
+		udelay ( 1 );
+	}
+
+	DBGC ( qib7322, "QIB7322 %p port %d timed out waiting for link state "
+	       "%s\n", qib7322, port, qib7322_link_state_text ( link_state ) );
+	return -ETIMEDOUT;
+}
+
+/**
+ * Set port information
+ *
+ * @v ibdev		Infiniband device
+ * @v mad		Set port information MAD
+ */
+static int qib7322_set_port_info ( struct ib_device *ibdev,
+				   union ib_mad *mad ) {
+	struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
+	struct ib_port_info *port_info = &mad->smp.smp_data.port_info;
+	struct QIB_7322_IBCCtrlA_0 ibcctrla;
+	unsigned int port = ( ibdev->port - QIB7322_PORT_BASE );
+	unsigned int port_state;
+	unsigned int link_state;
+
+	/* Set new link state */
+	port_state = ( port_info->link_speed_supported__port_state & 0xf );
+	if ( port_state ) {
+		link_state = ( port_state - 1 );
+		DBGC ( qib7322, "QIB7322 %p set link state to %s (%x)\n",
+		       qib7322, qib7322_link_state_text ( link_state ),
+		       link_state );
+		qib7322_readq_port ( qib7322, &ibcctrla,
+				     QIB_7322_IBCCtrlA_0_offset, port );
+		BIT_SET ( &ibcctrla, LinkCmd, link_state );
+		qib7322_writeq_port ( qib7322, &ibcctrla,
+				      QIB_7322_IBCCtrlA_0_offset, port );
+
+		/* Wait for link state change to take effect.  Ignore
+		 * errors; the current link state will be returned via
+		 * the GetResponse MAD.
+		 */
+		qib7322_link_state_check ( ibdev, link_state );
+	}
+
+	/* Detect and report link state change */
+	qib7322_link_state_changed ( ibdev );
+
+	return 0;
+}
+
+/**
+ * Set partition key table
+ *
+ * @v ibdev		Infiniband device
+ * @v mad		Set partition key table MAD
+ */
+static int qib7322_set_pkey_table ( struct ib_device *ibdev __unused,
+				    union ib_mad *mad __unused ) {
+	/* Nothing to do */
+	return 0;
+}
+
+/***************************************************************************
+ *
+ * Context allocation
+ *
+ ***************************************************************************
+ */
+
+/**
+ * Allocate a context and set queue pair number
+ *
+ * @v ibdev		Infiniband device
+ * @v qp		Queue pair
+ * @ret rc		Return status code
+ */
+static int qib7322_alloc_ctx ( struct ib_device *ibdev,
+			       struct ib_queue_pair *qp ) {
+	struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
+	unsigned int port = ( ibdev->port - QIB7322_PORT_BASE );
+	unsigned int ctx;
+
+	for ( ctx = port ; ctx < QIB7322_NUM_CONTEXTS ; ctx += 2 ) {
+
+		if ( ! qib7322->used_ctx[ctx] ) {
+			qib7322->used_ctx[ctx] = 1;
+			qp->qpn = ( ctx & ~0x01 );
+			DBGC2 ( qib7322, "QIB7322 %p port %d QPN %ld is CTX "
+				"%d\n", qib7322, port, qp->qpn, ctx );
+			return 0;
+		}
+	}
+
+	DBGC ( qib7322, "QIB7322 %p port %d out of available contexts\n",
+	       qib7322, port );
+	return -ENOENT;
+}
+
+/**
+ * Get queue pair context number
+ *
+ * @v ibdev		Infiniband device
+ * @v qp		Queue pair
+ * @ret ctx		Context index
+ */
+static unsigned int qib7322_ctx ( struct ib_device *ibdev,
+				  struct ib_queue_pair *qp ) {
+	return ( qp->qpn + ( ibdev->port - QIB7322_PORT_BASE ) );
+}
+
+/**
+ * Free a context
+ *
+ * @v qib7322		QIB7322 device
+ * @v ctx		Context index
+ */
+static void qib7322_free_ctx ( struct ib_device *ibdev,
+			       struct ib_queue_pair *qp ) {
+	struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
+	unsigned int port = ( ibdev->port - QIB7322_PORT_BASE );
+	unsigned int ctx = qib7322_ctx ( ibdev, qp );
+
+	qib7322->used_ctx[ctx] = 0;
+	DBGC2 ( qib7322, "QIB7322 %p port %d CTX %d freed\n",
+		qib7322, port, ctx );
+}
+
+/***************************************************************************
+ *
+ * Send datapath
+ *
+ ***************************************************************************
+ */
+
+/** Send buffer toggle bit
+ *
+ * We encode send buffers as 15 bits of send buffer index plus a
+ * single bit which should match the "check" bit in the SendBufAvail
+ * array.
+ */
+#define QIB7322_SEND_BUF_TOGGLE 0x8000
+
+/**
+ * Create send buffer set
+ *
+ * @v qib7322		QIB7322 device
+ * @v base		Send buffer base offset
+ * @v size		Send buffer size
+ * @v start		Index of first send buffer
+ * @v count		Number of send buffers
+ * @ret send_bufs	Send buffer set
+ */
+static struct qib7322_send_buffers *
+qib7322_create_send_bufs ( struct qib7322 *qib7322, unsigned long base,
+			   unsigned int size, unsigned int start,
+			   unsigned int count ) {
+	struct qib7322_send_buffers *send_bufs;
+	unsigned int i;
+
+	/* Allocate send buffer set */
+	send_bufs = zalloc ( sizeof ( *send_bufs ) +
+			     ( count * sizeof ( send_bufs->avail[0] ) ) );
+	if ( ! send_bufs )
+		return NULL;
+
+	/* Populate send buffer set */
+	send_bufs->base = base;
+	send_bufs->size = size;
+	send_bufs->start = start;
+	send_bufs->count = count;
+	for ( i = 0 ; i < count ; i++ )
+		send_bufs->avail[i] = ( start + i );
+
+	DBGC2 ( qib7322, "QIB7322 %p send buffer set %p [%d,%d] at %lx\n",
+		qib7322, send_bufs, start, ( start + count - 1 ),
+		send_bufs->base );
+
+	return send_bufs;
+}
+
+/**
+ * Destroy send buffer set
+ *
+ * @v qib7322		QIB7322 device
+ * @v send_bufs		Send buffer set
+ */
+static void
+qib7322_destroy_send_bufs ( struct qib7322 *qib7322 __unused,
+			    struct qib7322_send_buffers *send_bufs ) {
+	free ( send_bufs );
+}
+
+/**
+ * Allocate a send buffer
+ *
+ * @v qib7322		QIB7322 device
+ * @v send_bufs		Send buffer set
+ * @ret send_buf	Send buffer, or negative error
+ */
+static int qib7322_alloc_send_buf ( struct qib7322 *qib7322,
+				    struct qib7322_send_buffers *send_bufs ) {
+	unsigned int used;
+	unsigned int mask;
+	unsigned int send_buf;
+
+	used = ( send_bufs->cons - send_bufs->prod );
+	if ( used >= send_bufs->count ) {
+		DBGC ( qib7322, "QIB7322 %p send buffer set %p out of "
+		       "buffers\n", qib7322, send_bufs );
+		return -ENOBUFS;
+	}
+
+	mask = ( send_bufs->count - 1 );
+	send_buf = send_bufs->avail[ send_bufs->cons++ & mask ];
+	send_buf ^= QIB7322_SEND_BUF_TOGGLE;
+	return send_buf;
+}
+
+/**
+ * Free a send buffer
+ *
+ * @v qib7322		QIB7322 device
+ * @v send_bufs		Send buffer set
+ * @v send_buf		Send buffer
+ */
+static void qib7322_free_send_buf ( struct qib7322 *qib7322 __unused,
+				    struct qib7322_send_buffers *send_bufs,
+				    unsigned int send_buf ) {
+	unsigned int mask;
+
+	mask = ( send_bufs->count - 1 );
+	send_bufs->avail[ send_bufs->prod++ & mask ] = send_buf;
+}
+
+/**
+ * Check to see if send buffer is in use
+ *
+ * @v qib7322		QIB7322 device
+ * @v send_buf		Send buffer
+ * @ret in_use		Send buffer is in use
+ */
+static int qib7322_send_buf_in_use ( struct qib7322 *qib7322,
+				     unsigned int send_buf ) {
+	unsigned int send_idx;
+	unsigned int send_check;
+	unsigned int inusecheck;
+	unsigned int inuse;
+	unsigned int check;
+
+	send_idx = ( send_buf & ~QIB7322_SEND_BUF_TOGGLE );
+	send_check = ( !! ( send_buf & QIB7322_SEND_BUF_TOGGLE ) );
+	inusecheck = BIT_GET ( qib7322->sendbufavail, InUseCheck[send_idx] );
+	inuse = ( !! ( inusecheck & 0x02 ) );
+	check = ( !! ( inusecheck & 0x01 ) );
+	return ( inuse || ( check != send_check ) );
+}
+
+/**
+ * Calculate starting offset for send buffer
+ *
+ * @v qib7322		QIB7322 device
+ * @v send_buf		Send buffer
+ * @ret offset		Starting offset
+ */
+static unsigned long
+qib7322_send_buffer_offset ( struct qib7322 *qib7322 __unused,
+			     struct qib7322_send_buffers *send_bufs,
+			     unsigned int send_buf ) {
+	unsigned int index;
+
+	index = ( ( send_buf & ~QIB7322_SEND_BUF_TOGGLE ) - send_bufs->start );
+	return ( send_bufs->base + ( index * send_bufs->size ) );
+}
+
+/**
+ * Create send work queue
+ *
+ * @v ibdev		Infiniband device
+ * @v qp		Queue pair
+ */
+static int qib7322_create_send_wq ( struct ib_device *ibdev,
+				    struct ib_queue_pair *qp ) {
+	struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
+	struct ib_work_queue *wq = &qp->send;
+	struct qib7322_send_work_queue *qib7322_wq = ib_wq_get_drvdata ( wq );
+	unsigned int port = ( ibdev->port - QIB7322_PORT_BASE );
+
+	/* Select send buffer set */
+	if ( qp->type == IB_QPT_SMI ) {
+		if ( port == 0 ) {
+			qib7322_wq->send_bufs = qib7322->send_bufs_vl15_port0;
+		} else {
+			qib7322_wq->send_bufs = qib7322->send_bufs_vl15_port1;
+		}
+	} else {
+		qib7322_wq->send_bufs = qib7322->send_bufs_small;
+	}
+
+	/* Allocate space for send buffer usage list */
+	qib7322_wq->used = zalloc ( qp->send.num_wqes *
+				    sizeof ( qib7322_wq->used[0] ) );
+	if ( ! qib7322_wq->used )
+		return -ENOMEM;
+
+	/* Reset work queue */
+	qib7322_wq->prod = 0;
+	qib7322_wq->cons = 0;
+
+	return 0;
+}
+
+/**
+ * Destroy send work queue
+ *
+ * @v ibdev		Infiniband device
+ * @v qp		Queue pair
+ */
+static void qib7322_destroy_send_wq ( struct ib_device *ibdev __unused,
+				      struct ib_queue_pair *qp ) {
+	struct ib_work_queue *wq = &qp->send;
+	struct qib7322_send_work_queue *qib7322_wq = ib_wq_get_drvdata ( wq );
+
+	free ( qib7322_wq->used );
+}
+
+/**
+ * Initialise send datapath
+ *
+ * @v qib7322		QIB7322 device
+ * @ret rc		Return status code
+ */
+static int qib7322_init_send ( struct qib7322 *qib7322 ) {
+	struct QIB_7322_SendBufBase sendbufbase;
+	struct QIB_7322_SendBufAvailAddr sendbufavailaddr;
+	struct QIB_7322_SendCtrl sendctrl;
+	struct QIB_7322_SendCtrl_0 sendctrlp;
+	unsigned long baseaddr_smallpio;
+	unsigned long baseaddr_largepio;
+	unsigned long baseaddr_vl15_port0;
+	unsigned long baseaddr_vl15_port1;
+	int rc;
+
+	/* Create send buffer sets */
+	qib7322_readq ( qib7322, &sendbufbase, QIB_7322_SendBufBase_offset );
+	baseaddr_smallpio = BIT_GET ( &sendbufbase, BaseAddr_SmallPIO );
+	baseaddr_largepio = BIT_GET ( &sendbufbase, BaseAddr_LargePIO );
+	baseaddr_vl15_port0 = ( baseaddr_largepio +
+				( QIB7322_LARGE_SEND_BUF_SIZE *
+				  QIB7322_LARGE_SEND_BUF_COUNT ) );
+	baseaddr_vl15_port1 = ( baseaddr_vl15_port0 +
+				QIB7322_VL15_PORT0_SEND_BUF_SIZE );
+	qib7322->send_bufs_small =
+		qib7322_create_send_bufs ( qib7322, baseaddr_smallpio,
+					   QIB7322_SMALL_SEND_BUF_SIZE,
+					   QIB7322_SMALL_SEND_BUF_START,
+					   QIB7322_SMALL_SEND_BUF_USED );
+	if ( ! qib7322->send_bufs_small ) {
+		rc = -ENOMEM;
+		goto err_create_send_bufs_small;
+	}
+	qib7322->send_bufs_vl15_port0 =
+		qib7322_create_send_bufs ( qib7322, baseaddr_vl15_port0,
+					   QIB7322_VL15_PORT0_SEND_BUF_SIZE,
+					   QIB7322_VL15_PORT0_SEND_BUF_START,
+					   QIB7322_VL15_PORT0_SEND_BUF_COUNT );
+	if ( ! qib7322->send_bufs_vl15_port0 ) {
+		rc = -ENOMEM;
+		goto err_create_send_bufs_vl15_port0;
+	}
+	qib7322->send_bufs_vl15_port1 =
+		qib7322_create_send_bufs ( qib7322, baseaddr_vl15_port1,
+					   QIB7322_VL15_PORT1_SEND_BUF_SIZE,
+					   QIB7322_VL15_PORT1_SEND_BUF_START,
+					   QIB7322_VL15_PORT1_SEND_BUF_COUNT );
+	if ( ! qib7322->send_bufs_vl15_port1 ) {
+		rc = -ENOMEM;
+		goto err_create_send_bufs_vl15_port1;
+	}
+
+	/* Allocate space for the SendBufAvail array */
+	qib7322->sendbufavail = malloc_dma ( sizeof ( *qib7322->sendbufavail ),
+					     QIB7322_SENDBUFAVAIL_ALIGN );
+	if ( ! qib7322->sendbufavail ) {
+		rc = -ENOMEM;
+		goto err_alloc_sendbufavail;
+	}
+	memset ( qib7322->sendbufavail, 0, sizeof ( qib7322->sendbufavail ) );
+
+	/* Program SendBufAvailAddr into the hardware */
+	memset ( &sendbufavailaddr, 0, sizeof ( sendbufavailaddr ) );
+	BIT_FILL_1 ( &sendbufavailaddr, SendBufAvailAddr,
+		     ( virt_to_bus ( qib7322->sendbufavail ) >> 6 ) );
+	qib7322_writeq ( qib7322, &sendbufavailaddr,
+			 QIB_7322_SendBufAvailAddr_offset );
+
+	/* Enable sending */
+	memset ( &sendctrlp, 0, sizeof ( sendctrlp ) );
+	BIT_FILL_1 ( &sendctrlp, SendEnable, 1 );
+	qib7322_writeq ( qib7322, &sendctrlp, QIB_7322_SendCtrl_0_offset );
+	qib7322_writeq ( qib7322, &sendctrlp, QIB_7322_SendCtrl_1_offset );
+
+	/* Enable DMA of SendBufAvail */
+	memset ( &sendctrl, 0, sizeof ( sendctrl ) );
+	BIT_FILL_1 ( &sendctrl, SendBufAvailUpd, 1 );
+	qib7322_writeq ( qib7322, &sendctrl, QIB_7322_SendCtrl_offset );
+
+	return 0;
+
+	free_dma ( qib7322->sendbufavail, sizeof ( *qib7322->sendbufavail ) );
+ err_alloc_sendbufavail:
+	qib7322_destroy_send_bufs ( qib7322, qib7322->send_bufs_vl15_port1 );
+ err_create_send_bufs_vl15_port1:
+	qib7322_destroy_send_bufs ( qib7322, qib7322->send_bufs_vl15_port0 );
+ err_create_send_bufs_vl15_port0:
+	qib7322_destroy_send_bufs ( qib7322, qib7322->send_bufs_small );
+ err_create_send_bufs_small:
+	return rc;
+}
+
+/**
+ * Shut down send datapath
+ *
+ * @v qib7322		QIB7322 device
+ */
+static void qib7322_fini_send ( struct qib7322 *qib7322 ) {
+	struct QIB_7322_SendCtrl sendctrl;
+
+	/* Disable sending and DMA of SendBufAvail */
+	memset ( &sendctrl, 0, sizeof ( sendctrl ) );
+	qib7322_writeq ( qib7322, &sendctrl, QIB_7322_SendCtrl_offset );
+	mb();
+
+	/* Ensure hardware has seen this disable */
+	qib7322_readq ( qib7322, &sendctrl, QIB_7322_SendCtrl_offset );
+
+	free_dma ( qib7322->sendbufavail, sizeof ( *qib7322->sendbufavail ) );
+	qib7322_destroy_send_bufs ( qib7322, qib7322->send_bufs_vl15_port1 );
+	qib7322_destroy_send_bufs ( qib7322, qib7322->send_bufs_vl15_port0 );
+	qib7322_destroy_send_bufs ( qib7322, qib7322->send_bufs_small );
+}
+
+/***************************************************************************
+ *
+ * Receive datapath
+ *
+ ***************************************************************************
+ */
+
+/**
+ * Create receive work queue
+ *
+ * @v ibdev		Infiniband device
+ * @v qp		Queue pair
+ * @ret rc		Return status code
+ */
+static int qib7322_create_recv_wq ( struct ib_device *ibdev,
+				    struct ib_queue_pair *qp ) {
+	struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
+	struct ib_work_queue *wq = &qp->recv;
+	struct qib7322_recv_work_queue *qib7322_wq = ib_wq_get_drvdata ( wq );
+	struct QIB_7322_RcvHdrAddr0 rcvhdraddr;
+	struct QIB_7322_RcvHdrTailAddr0 rcvhdrtailaddr;
+	struct QIB_7322_RcvHdrHead0 rcvhdrhead;
+	struct QIB_7322_scalar rcvegrindexhead;
+	struct QIB_7322_RcvCtrl rcvctrl;
+	struct QIB_7322_RcvCtrl_P rcvctrlp;
+	unsigned int port = ( ibdev->port - QIB7322_PORT_BASE );
+	unsigned int ctx = qib7322_ctx ( ibdev, qp );
+	int rc;
+
+	/* Reset context information */
+	memset ( &qib7322_wq->header_prod, 0,
+		 sizeof ( qib7322_wq->header_prod ) );
+	qib7322_wq->header_cons = 0;
+	qib7322_wq->eager_prod = 0;
+	qib7322_wq->eager_cons = 0;
+
+	/* Allocate receive header buffer */
+	qib7322_wq->header = malloc_dma ( QIB7322_RECV_HEADERS_SIZE,
+					  QIB7322_RECV_HEADERS_ALIGN );
+	if ( ! qib7322_wq->header ) {
+		rc = -ENOMEM;
+		goto err_alloc_header;
+	}
+
+	/* Enable context in hardware */
+	memset ( &rcvhdraddr, 0, sizeof ( rcvhdraddr ) );
+	BIT_FILL_1 ( &rcvhdraddr, RcvHdrAddr,
+		     ( virt_to_bus ( qib7322_wq->header ) >> 2 ) );
+	qib7322_writeq_array8b ( qib7322, &rcvhdraddr,
+				 QIB_7322_RcvHdrAddr0_offset, ctx );
+	memset ( &rcvhdrtailaddr, 0, sizeof ( rcvhdrtailaddr ) );
+	BIT_FILL_1 ( &rcvhdrtailaddr, RcvHdrTailAddr,
+		     ( virt_to_bus ( &qib7322_wq->header_prod ) >> 2 ) );
+	qib7322_writeq_array8b ( qib7322, &rcvhdrtailaddr,
+				 QIB_7322_RcvHdrTailAddr0_offset, ctx );
+	memset ( &rcvhdrhead, 0, sizeof ( rcvhdrhead ) );
+	BIT_FILL_1 ( &rcvhdrhead, counter, 1 );
+	qib7322_writeq_array64k ( qib7322, &rcvhdrhead,
+				  QIB_7322_RcvHdrHead0_offset, ctx );
+	memset ( &rcvegrindexhead, 0, sizeof ( rcvegrindexhead ) );
+	BIT_FILL_1 ( &rcvegrindexhead, Value, 1 );
+	qib7322_writeq_array64k ( qib7322, &rcvegrindexhead,
+				  QIB_7322_RcvEgrIndexHead0_offset, ctx );
+	qib7322_readq_port ( qib7322, &rcvctrlp,
+			     QIB_7322_RcvCtrl_0_offset, port );
+	BIT_SET ( &rcvctrlp, ContextEnable[ctx], 1 );
+	qib7322_writeq_port ( qib7322, &rcvctrlp,
+			      QIB_7322_RcvCtrl_0_offset, port );
+	qib7322_readq ( qib7322, &rcvctrl, QIB_7322_RcvCtrl_offset );
+	BIT_SET ( &rcvctrl, IntrAvail[ctx], 1 );
+	qib7322_writeq ( qib7322, &rcvctrl, QIB_7322_RcvCtrl_offset );
+
+	DBGC ( qib7322, "QIB7322 %p port %d QPN %ld CTX %d hdrs [%lx,%lx) prod "
+	       "%lx\n", qib7322, port, qp->qpn, ctx,
+	       virt_to_bus ( qib7322_wq->header ),
+	       ( virt_to_bus ( qib7322_wq->header )
+		 + QIB7322_RECV_HEADERS_SIZE ),
+	       virt_to_bus ( &qib7322_wq->header_prod ) );
+	return 0;
+
+	free_dma ( qib7322_wq->header, QIB7322_RECV_HEADERS_SIZE );
+ err_alloc_header:
+	return rc;
+}
+
+/**
+ * Destroy receive work queue
+ *
+ * @v ibdev		Infiniband device
+ * @v qp		Queue pair
+ */
+static void qib7322_destroy_recv_wq ( struct ib_device *ibdev,
+				      struct ib_queue_pair *qp ) {
+	struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
+	struct ib_work_queue *wq = &qp->recv;
+	struct qib7322_recv_work_queue *qib7322_wq = ib_wq_get_drvdata ( wq );
+	struct QIB_7322_RcvCtrl rcvctrl;
+	struct QIB_7322_RcvCtrl_P rcvctrlp;
+	unsigned int port = ( ibdev->port - QIB7322_PORT_BASE );
+	unsigned int ctx = qib7322_ctx ( ibdev, qp );
+
+	/* Disable context in hardware */
+	qib7322_readq_port ( qib7322, &rcvctrlp,
+			     QIB_7322_RcvCtrl_0_offset, port );
+	BIT_SET ( &rcvctrlp, ContextEnable[ctx], 0 );
+	qib7322_writeq_port ( qib7322, &rcvctrlp,
+			      QIB_7322_RcvCtrl_0_offset, port );
+	qib7322_readq ( qib7322, &rcvctrl, QIB_7322_RcvCtrl_offset );
+	BIT_SET ( &rcvctrl, IntrAvail[ctx], 0 );
+	qib7322_writeq ( qib7322, &rcvctrl, QIB_7322_RcvCtrl_offset );
+
+	/* Make sure the hardware has seen that the context is disabled */
+	qib7322_readq ( qib7322, &rcvctrl, QIB_7322_RcvCtrl_offset );
+	mb();
+
+	/* Free headers ring */
+	free_dma ( qib7322_wq->header, QIB7322_RECV_HEADERS_SIZE );
+}
+
+/**
+ * Initialise receive datapath
+ *
+ * @v qib7322		QIB7322 device
+ * @ret rc		Return status code
+ */
+static int qib7322_init_recv ( struct qib7322 *qib7322 ) {
+	struct QIB_7322_RcvCtrl rcvctrl;
+	struct QIB_7322_RcvCtrl_0 rcvctrlp;
+	struct QIB_7322_RcvQPMapTableA_0 rcvqpmaptablea0;
+	struct QIB_7322_RcvQPMapTableB_0 rcvqpmaptableb0;
+	struct QIB_7322_RcvQPMapTableA_1 rcvqpmaptablea1;
+	struct QIB_7322_RcvQPMapTableB_1 rcvqpmaptableb1;
+	struct QIB_7322_RcvQPMulticastContext_0 rcvqpmcastctx0;
+	struct QIB_7322_RcvQPMulticastContext_1 rcvqpmcastctx1;
+	struct QIB_7322_scalar rcvegrbase;
+	struct QIB_7322_scalar rcvhdrentsize;
+	struct QIB_7322_scalar rcvhdrcnt;
+	struct QIB_7322_RcvBTHQP_0 rcvbthqp;
+	struct QIB_7322_RxCreditVL0_0 rxcreditvl;
+	unsigned int contextcfg;
+	unsigned long egrbase;
+	unsigned int eager_array_size_kernel;
+	unsigned int eager_array_size_user;
+	unsigned int ctx;
+
+	/* Select configuration based on number of contexts */
+	switch ( QIB7322_NUM_CONTEXTS ) {
+	case 6:
+		contextcfg = QIB7322_CONTEXTCFG_6CTX;
+		eager_array_size_kernel = QIB7322_EAGER_ARRAY_SIZE_6CTX_KERNEL;
+		eager_array_size_user = QIB7322_EAGER_ARRAY_SIZE_6CTX_USER;
+		break;
+	case 10:
+		contextcfg = QIB7322_CONTEXTCFG_10CTX;
+		eager_array_size_kernel = QIB7322_EAGER_ARRAY_SIZE_10CTX_KERNEL;
+		eager_array_size_user = QIB7322_EAGER_ARRAY_SIZE_10CTX_USER;
+		break;
+	case 18:
+		contextcfg = QIB7322_CONTEXTCFG_18CTX;
+		eager_array_size_kernel = QIB7322_EAGER_ARRAY_SIZE_18CTX_KERNEL;
+		eager_array_size_user = QIB7322_EAGER_ARRAY_SIZE_18CTX_USER;
+		break;
+	default:
+		linker_assert ( 0, invalid_QIB7322_NUM_CONTEXTS );
+		return -EINVAL;
+	}
+
+	/* Configure number of contexts */
+	memset ( &rcvctrl, 0, sizeof ( rcvctrl ) );
+	BIT_FILL_2 ( &rcvctrl,
+		     TailUpd, 1,
+		     ContextCfg, contextcfg );
+	qib7322_writeq ( qib7322, &rcvctrl, QIB_7322_RcvCtrl_offset );
+
+	/* Map QPNs to contexts */
+	memset ( &rcvctrlp, 0, sizeof ( rcvctrlp ) );
+	BIT_FILL_3 ( &rcvctrlp,
+		     RcvIBPortEnable, 1,
+		     RcvQPMapEnable, 1,
+		     RcvPartitionKeyDisable, 1 );
+	qib7322_writeq ( qib7322, &rcvctrlp, QIB_7322_RcvCtrl_0_offset );
+	qib7322_writeq ( qib7322, &rcvctrlp, QIB_7322_RcvCtrl_1_offset );
+	memset ( &rcvqpmaptablea0, 0, sizeof ( rcvqpmaptablea0 ) );
+	BIT_FILL_6 ( &rcvqpmaptablea0,
+		     RcvQPMapContext0, 0,
+		     RcvQPMapContext1, 2,
+		     RcvQPMapContext2, 4,
+		     RcvQPMapContext3, 6,
+		     RcvQPMapContext4, 8,
+		     RcvQPMapContext5, 10 );
+	qib7322_writeq ( qib7322, &rcvqpmaptablea0,
+			 QIB_7322_RcvQPMapTableA_0_offset );
+	memset ( &rcvqpmaptableb0, 0, sizeof ( rcvqpmaptableb0 ) );
+	BIT_FILL_3 ( &rcvqpmaptableb0,
+		     RcvQPMapContext6, 12,
+		     RcvQPMapContext7, 14,
+		     RcvQPMapContext8, 16 );
+	qib7322_writeq ( qib7322, &rcvqpmaptableb0,
+			 QIB_7322_RcvQPMapTableB_0_offset );
+	memset ( &rcvqpmaptablea1, 0, sizeof ( rcvqpmaptablea1 ) );
+	BIT_FILL_6 ( &rcvqpmaptablea1,
+		     RcvQPMapContext0, 1,
+		     RcvQPMapContext1, 3,
+		     RcvQPMapContext2, 5,
+		     RcvQPMapContext3, 7,
+		     RcvQPMapContext4, 9,
+		     RcvQPMapContext5, 11 );
+	qib7322_writeq ( qib7322, &rcvqpmaptablea1,
+			 QIB_7322_RcvQPMapTableA_1_offset );
+	memset ( &rcvqpmaptableb1, 0, sizeof ( rcvqpmaptableb1 ) );
+	BIT_FILL_3 ( &rcvqpmaptableb1,
+		     RcvQPMapContext6, 13,
+		     RcvQPMapContext7, 15,
+		     RcvQPMapContext8, 17 );
+	qib7322_writeq ( qib7322, &rcvqpmaptableb1,
+			 QIB_7322_RcvQPMapTableB_1_offset );
+
+	/* Map multicast QPNs to contexts */
+	memset ( &rcvqpmcastctx0, 0, sizeof ( rcvqpmcastctx0 ) );
+	BIT_FILL_1 ( &rcvqpmcastctx0, RcvQpMcContext, 0 );
+	qib7322_writeq ( qib7322, &rcvqpmcastctx0,
+			 QIB_7322_RcvQPMulticastContext_0_offset );
+	memset ( &rcvqpmcastctx1, 0, sizeof ( rcvqpmcastctx1 ) );
+	BIT_FILL_1 ( &rcvqpmcastctx1, RcvQpMcContext, 1 );
+	qib7322_writeq ( qib7322, &rcvqpmcastctx1,
+			 QIB_7322_RcvQPMulticastContext_1_offset );
+
+	/* Configure receive header buffer sizes */
+	memset ( &rcvhdrcnt, 0, sizeof ( rcvhdrcnt ) );
+	BIT_FILL_1 ( &rcvhdrcnt, Value, QIB7322_RECV_HEADER_COUNT );
+	qib7322_writeq ( qib7322, &rcvhdrcnt, QIB_7322_RcvHdrCnt_offset );
+	memset ( &rcvhdrentsize, 0, sizeof ( rcvhdrentsize ) );
+	BIT_FILL_1 ( &rcvhdrentsize, Value, ( QIB7322_RECV_HEADER_SIZE >> 2 ) );
+	qib7322_writeq ( qib7322, &rcvhdrentsize,
+			 QIB_7322_RcvHdrEntSize_offset );
+
+	/* Calculate eager array start addresses for each context */
+	qib7322_readq ( qib7322, &rcvegrbase, QIB_7322_RcvEgrBase_offset );
+	egrbase = BIT_GET ( &rcvegrbase, Value );
+	for ( ctx = 0 ; ctx < QIB7322_MAX_PORTS ; ctx++ ) {
+		qib7322->recv_wq[ctx].eager_array = egrbase;
+		qib7322->recv_wq[ctx].eager_entries = eager_array_size_kernel;
+		egrbase += ( eager_array_size_kernel *
+			     sizeof ( struct QIB_7322_RcvEgr ) );
+	}
+	for ( ; ctx < QIB7322_NUM_CONTEXTS ; ctx++ ) {
+		qib7322->recv_wq[ctx].eager_array = egrbase;
+		qib7322->recv_wq[ctx].eager_entries = eager_array_size_user;
+		egrbase += ( eager_array_size_user *
+			     sizeof ( struct QIB_7322_RcvEgr ) );
+	}
+	for ( ctx = 0 ; ctx < QIB7322_NUM_CONTEXTS ; ctx++ ) {
+		DBGC ( qib7322, "QIB7322 %p CTX %d eager array at %lx (%d "
+		       "entries)\n", qib7322, ctx,
+		       qib7322->recv_wq[ctx].eager_array,
+		       qib7322->recv_wq[ctx].eager_entries );
+	}
+
+	/* Set the BTH QP for Infinipath packets to an unused value */
+	memset ( &rcvbthqp, 0, sizeof ( rcvbthqp ) );
+	BIT_FILL_1 ( &rcvbthqp, RcvBTHQP, QIB7322_QP_IDETH );
+	qib7322_writeq ( qib7322, &rcvbthqp, QIB_7322_RcvBTHQP_0_offset );
+	qib7322_writeq ( qib7322, &rcvbthqp, QIB_7322_RcvBTHQP_1_offset );
+
+	/* Assign initial credits */
+	memset ( &rxcreditvl, 0, sizeof ( rxcreditvl ) );
+	BIT_FILL_1 ( &rxcreditvl, RxMaxCreditVL, QIB7322_MAX_CREDITS_VL0 );
+	qib7322_writeq_array8b ( qib7322, &rxcreditvl,
+				 QIB_7322_RxCreditVL0_0_offset, 0 );
+	qib7322_writeq_array8b ( qib7322, &rxcreditvl,
+				 QIB_7322_RxCreditVL0_1_offset, 0 );
+	BIT_FILL_1 ( &rxcreditvl, RxMaxCreditVL, QIB7322_MAX_CREDITS_VL15 );
+	qib7322_writeq_array8b ( qib7322, &rxcreditvl,
+				 QIB_7322_RxCreditVL0_0_offset, 15 );
+	qib7322_writeq_array8b ( qib7322, &rxcreditvl,
+				 QIB_7322_RxCreditVL0_1_offset, 15 );
+
+	return 0;
+}
+
+/**
+ * Shut down receive datapath
+ *
+ * @v qib7322		QIB7322 device
+ */
+static void qib7322_fini_recv ( struct qib7322 *qib7322 __unused ) {
+	/* Nothing to do; all contexts were already disabled when the
+	 * queue pairs were destroyed
+	 */
+}
+
+/***************************************************************************
+ *
+ * Completion queue operations
+ *
+ ***************************************************************************
+ */
+
+/**
+ * Create completion queue
+ *
+ * @v ibdev		Infiniband device
+ * @v cq		Completion queue
+ * @ret rc		Return status code
+ */
+static int qib7322_create_cq ( struct ib_device *ibdev,
+			       struct ib_completion_queue *cq ) {
+	struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
+	static int cqn;
+
+	/* The hardware has no concept of completion queues.  We
+	 * simply use the association between CQs and WQs (already
+	 * handled by the IB core) to decide which WQs to poll.
+	 *
+	 * We do set a CQN, just to avoid confusing debug messages
+	 * from the IB core.
+	 */
+	cq->cqn = ++cqn;
+	DBGC ( qib7322, "QIB7322 %p CQN %ld created\n", qib7322, cq->cqn );
+
+	return 0;
+}
+
+/**
+ * Destroy completion queue
+ *
+ * @v ibdev		Infiniband device
+ * @v cq		Completion queue
+ */
+static void qib7322_destroy_cq ( struct ib_device *ibdev,
+				 struct ib_completion_queue *cq ) {
+	struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
+
+	/* Nothing to do */
+	DBGC ( qib7322, "QIB7322 %p CQN %ld destroyed\n", qib7322, cq->cqn );
+}
+
+/***************************************************************************
+ *
+ * Queue pair operations
+ *
+ ***************************************************************************
+ */
+
+/**
+ * Create queue pair
+ *
+ * @v ibdev		Infiniband device
+ * @v qp		Queue pair
+ * @ret rc		Return status code
+ */
+static int qib7322_create_qp ( struct ib_device *ibdev,
+			       struct ib_queue_pair *qp ) {
+	struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
+	unsigned int ctx;
+	int rc;
+
+	/* Allocate a context and QPN */
+	if ( ( rc = qib7322_alloc_ctx ( ibdev, qp ) ) != 0 )
+		goto err_alloc_ctx;
+	ctx = qib7322_ctx ( ibdev, qp );
+
+	/* Set work-queue private data pointers */
+	ib_wq_set_drvdata ( &qp->send, &qib7322->send_wq[ctx] );
+	ib_wq_set_drvdata ( &qp->recv, &qib7322->recv_wq[ctx] );
+
+	/* Create receive work queue */
+	if ( ( rc = qib7322_create_recv_wq ( ibdev, qp ) ) != 0 )
+		goto err_create_recv_wq;
+
+	/* Create send work queue */
+	if ( ( rc = qib7322_create_send_wq ( ibdev, qp ) ) != 0 )
+		goto err_create_send_wq;
+
+	return 0;
+
+	qib7322_destroy_send_wq ( ibdev, qp );
+ err_create_send_wq:
+	qib7322_destroy_recv_wq ( ibdev, qp );
+ err_create_recv_wq:
+	qib7322_free_ctx ( ibdev, qp );
+ err_alloc_ctx:
+	return rc;
+}
+
+/**
+ * Modify queue pair
+ *
+ * @v ibdev		Infiniband device
+ * @v qp		Queue pair
+ * @ret rc		Return status code
+ */
+static int qib7322_modify_qp ( struct ib_device *ibdev,
+			       struct ib_queue_pair *qp ) {
+	struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
+
+	/* Nothing to do; the hardware doesn't have a notion of queue
+	 * keys
+	 */
+	DBGC2 ( qib7322, "QIB7322 %p QPN %ld modified\n", qib7322, qp->qpn );
+	return 0;
+}
+
+/**
+ * Destroy queue pair
+ *
+ * @v ibdev		Infiniband device
+ * @v qp		Queue pair
+ */
+static void qib7322_destroy_qp ( struct ib_device *ibdev,
+				 struct ib_queue_pair *qp ) {
+
+	qib7322_destroy_send_wq ( ibdev, qp );
+	qib7322_destroy_recv_wq ( ibdev, qp );
+	qib7322_free_ctx ( ibdev, qp );
+}
+
+/***************************************************************************
+ *
+ * Work request operations
+ *
+ ***************************************************************************
+ */
+
+/**
+ * Post send work queue entry
+ *
+ * @v ibdev		Infiniband device
+ * @v qp		Queue pair
+ * @v dest		Destination address vector
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ */
+static int qib7322_post_send ( struct ib_device *ibdev,
+			       struct ib_queue_pair *qp,
+			       struct ib_address_vector *dest,
+			       struct io_buffer *iobuf ) {
+	struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
+	struct ib_work_queue *wq = &qp->send;
+	struct qib7322_send_work_queue *qib7322_wq = ib_wq_get_drvdata ( wq );
+	struct QIB_7322_SendPbc sendpbc;
+	unsigned int port = ( ibdev->port - QIB7322_PORT_BASE );
+	uint8_t header_buf[IB_MAX_HEADER_SIZE];
+	struct io_buffer headers;
+	int send_buf;
+	unsigned long start_offset;
+	unsigned long offset;
+	size_t len;
+	ssize_t frag_len;
+	uint32_t *data;
+
+	/* Allocate send buffer and calculate offset */
+	send_buf = qib7322_alloc_send_buf ( qib7322, qib7322_wq->send_bufs );
+	if ( send_buf < 0 )
+		return send_buf;
+	start_offset = offset =
+		qib7322_send_buffer_offset ( qib7322, qib7322_wq->send_bufs,
+					     send_buf );
+
+	/* Store I/O buffer and send buffer index */
+	assert ( wq->iobufs[qib7322_wq->prod] == NULL );
+	wq->iobufs[qib7322_wq->prod] = iobuf;
+	qib7322_wq->used[qib7322_wq->prod] = send_buf;
+
+	/* Construct headers */
+	iob_populate ( &headers, header_buf, 0, sizeof ( header_buf ) );
+	iob_reserve ( &headers, sizeof ( header_buf ) );
+	ib_push ( ibdev, &headers, qp, iob_len ( iobuf ), dest );
+
+	/* Calculate packet length */
+	len = ( ( sizeof ( sendpbc ) + iob_len ( &headers ) +
+		  iob_len ( iobuf ) + 3 ) & ~3 );
+
+	/* Construct send per-buffer control word */
+	memset ( &sendpbc, 0, sizeof ( sendpbc ) );
+	BIT_FILL_3 ( &sendpbc,
+		     LengthP1_toibc, ( ( len >> 2 ) - 1 ),
+		     Port, port,
+		     VL15, ( ( qp->type == IB_QPT_SMI ) ? 1 : 0 ) );
+
+	/* Write SendPbc */
+	DBG_DISABLE ( DBGLVL_IO );
+	qib7322_writeq ( qib7322, &sendpbc, offset );
+	offset += sizeof ( sendpbc );
+
+	/* Write headers */
+	for ( data = headers.data, frag_len = iob_len ( &headers ) ;
+	      frag_len > 0 ; data++, offset += 4, frag_len -= 4 ) {
+		qib7322_writel ( qib7322, *data, offset );
+	}
+
+	/* Write data */
+	for ( data = iobuf->data, frag_len = iob_len ( iobuf ) ;
+	      frag_len > 0 ; data++, offset += 4, frag_len -= 4 ) {
+		qib7322_writel ( qib7322, *data, offset );
+	}
+	DBG_ENABLE ( DBGLVL_IO );
+
+	assert ( ( start_offset + len ) == offset );
+	DBGC2 ( qib7322, "QIB7322 %p QPN %ld TX %04x(%04x) posted [%lx,%lx)\n",
+		qib7322, qp->qpn, send_buf, qib7322_wq->prod,
+		start_offset, offset );
+
+	/* Increment producer counter */
+	qib7322_wq->prod = ( ( qib7322_wq->prod + 1 ) & ( wq->num_wqes - 1 ) );
+
+	return 0;
+}
+
+/**
+ * Complete send work queue entry
+ *
+ * @v ibdev		Infiniband device
+ * @v qp		Queue pair
+ * @v wqe_idx		Work queue entry index
+ */
+static void qib7322_complete_send ( struct ib_device *ibdev,
+				    struct ib_queue_pair *qp,
+				    unsigned int wqe_idx ) {
+	struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
+	struct ib_work_queue *wq = &qp->send;
+	struct qib7322_send_work_queue *qib7322_wq = ib_wq_get_drvdata ( wq );
+	struct io_buffer *iobuf;
+	unsigned int send_buf;
+
+	/* Parse completion */
+	send_buf = qib7322_wq->used[wqe_idx];
+	DBGC2 ( qib7322, "QIB7322 %p QPN %ld TX %04x(%04x) complete\n",
+		qib7322, qp->qpn, send_buf, wqe_idx );
+
+	/* Complete work queue entry */
+	iobuf = wq->iobufs[wqe_idx];
+	assert ( iobuf != NULL );
+	ib_complete_send ( ibdev, qp, iobuf, 0 );
+	wq->iobufs[wqe_idx] = NULL;
+
+	/* Free send buffer */
+	qib7322_free_send_buf ( qib7322, qib7322_wq->send_bufs, send_buf );
+}
+
+/**
+ * Poll send work queue
+ *
+ * @v ibdev		Infiniband device
+ * @v qp		Queue pair
+ */
+static void qib7322_poll_send_wq ( struct ib_device *ibdev,
+				   struct ib_queue_pair *qp ) {
+	struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
+	struct ib_work_queue *wq = &qp->send;
+	struct qib7322_send_work_queue *qib7322_wq = ib_wq_get_drvdata ( wq );
+	unsigned int send_buf;
+
+	/* Look for completions */
+	while ( wq->fill ) {
+
+		/* Check to see if send buffer has completed */
+		send_buf = qib7322_wq->used[qib7322_wq->cons];
+		if ( qib7322_send_buf_in_use ( qib7322, send_buf ) )
+			break;
+
+		/* Complete this buffer */
+		qib7322_complete_send ( ibdev, qp, qib7322_wq->cons );
+
+		/* Increment consumer counter */
+		qib7322_wq->cons = ( ( qib7322_wq->cons + 1 ) &
+				     ( wq->num_wqes - 1 ) );
+	}
+}
+
+/**
+ * Post receive work queue entry
+ *
+ * @v ibdev		Infiniband device
+ * @v qp		Queue pair
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ */
+static int qib7322_post_recv ( struct ib_device *ibdev,
+			       struct ib_queue_pair *qp,
+			       struct io_buffer *iobuf ) {
+	struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
+	struct ib_work_queue *wq = &qp->recv;
+	struct qib7322_recv_work_queue *qib7322_wq = ib_wq_get_drvdata ( wq );
+	struct QIB_7322_RcvEgr rcvegr;
+	struct QIB_7322_scalar rcvegrindexhead;
+	unsigned int ctx = qib7322_ctx ( ibdev, qp );
+	physaddr_t addr;
+	size_t len;
+	unsigned int wqe_idx;
+	unsigned int bufsize;
+
+	/* Sanity checks */
+	addr = virt_to_bus ( iobuf->data );
+	len = iob_tailroom ( iobuf );
+	if ( addr & ( QIB7322_EAGER_BUFFER_ALIGN - 1 ) ) {
+		DBGC ( qib7322, "QIB7322 %p QPN %ld misaligned RX buffer "
+		       "(%08lx)\n", qib7322, qp->qpn, addr );
+		return -EINVAL;
+	}
+	if ( len != QIB7322_RECV_PAYLOAD_SIZE ) {
+		DBGC ( qib7322, "QIB7322 %p QPN %ld wrong RX buffer size "
+		       "(%zd)\n", qib7322, qp->qpn, len );
+		return -EINVAL;
+	}
+
+	/* Calculate eager producer index and WQE index */
+	wqe_idx = ( qib7322_wq->eager_prod & ( wq->num_wqes - 1 ) );
+	assert ( wq->iobufs[wqe_idx] == NULL );
+
+	/* Store I/O buffer */
+	wq->iobufs[wqe_idx] = iobuf;
+
+	/* Calculate buffer size */
+	switch ( QIB7322_RECV_PAYLOAD_SIZE ) {
+	case 2048:  bufsize = QIB7322_EAGER_BUFFER_2K;  break;
+	case 4096:  bufsize = QIB7322_EAGER_BUFFER_4K;  break;
+	case 8192:  bufsize = QIB7322_EAGER_BUFFER_8K;  break;
+	case 16384: bufsize = QIB7322_EAGER_BUFFER_16K; break;
+	case 32768: bufsize = QIB7322_EAGER_BUFFER_32K; break;
+	case 65536: bufsize = QIB7322_EAGER_BUFFER_64K; break;
+	default:    linker_assert ( 0, invalid_rx_payload_size );
+		    bufsize = QIB7322_EAGER_BUFFER_NONE;
+	}
+
+	/* Post eager buffer */
+	memset ( &rcvegr, 0, sizeof ( rcvegr ) );
+	BIT_FILL_2 ( &rcvegr,
+		     Addr, ( addr >> 11 ),
+		     BufSize, bufsize );
+	qib7322_writeq_array8b ( qib7322, &rcvegr, qib7322_wq->eager_array,
+				 qib7322_wq->eager_prod );
+	DBGC2 ( qib7322, "QIB7322 %p QPN %ld RX egr %04x(%04x) posted "
+		"[%lx,%lx)\n", qib7322, qp->qpn, qib7322_wq->eager_prod,
+		wqe_idx, addr, ( addr + len ) );
+
+	/* Increment producer index */
+	qib7322_wq->eager_prod = ( ( qib7322_wq->eager_prod + 1 ) &
+				   ( qib7322_wq->eager_entries - 1 ) );
+
+	/* Update head index */
+	memset ( &rcvegrindexhead, 0, sizeof ( rcvegrindexhead ) );
+	BIT_FILL_1 ( &rcvegrindexhead,
+		     Value, ( ( qib7322_wq->eager_prod + 1 ) &
+			      ( qib7322_wq->eager_entries - 1 ) ) );
+	qib7322_writeq_array64k ( qib7322, &rcvegrindexhead,
+				  QIB_7322_RcvEgrIndexHead0_offset, ctx );
+
+	return 0;
+}
+
+/**
+ * Complete receive work queue entry
+ *
+ * @v ibdev		Infiniband device
+ * @v qp		Queue pair
+ * @v header_offs	Header offset
+ */
+static void qib7322_complete_recv ( struct ib_device *ibdev,
+				    struct ib_queue_pair *qp,
+				    unsigned int header_offs ) {
+	struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
+	struct ib_work_queue *wq = &qp->recv;
+	struct qib7322_recv_work_queue *qib7322_wq = ib_wq_get_drvdata ( wq );
+	struct QIB_7322_RcvHdrFlags *rcvhdrflags;
+	struct QIB_7322_RcvEgr rcvegr;
+	struct io_buffer headers;
+	struct io_buffer *iobuf;
+	struct ib_queue_pair *intended_qp;
+	struct ib_address_vector dest;
+	struct ib_address_vector source;
+	unsigned int rcvtype;
+	unsigned int pktlen;
+	unsigned int egrindex;
+	unsigned int useegrbfr;
+	unsigned int iberr, mkerr, tiderr, khdrerr, mtuerr;
+	unsigned int lenerr, parityerr, vcrcerr, icrcerr;
+	unsigned int err;
+	unsigned int hdrqoffset;
+	unsigned int header_len;
+	unsigned int padded_payload_len;
+	unsigned int wqe_idx;
+	size_t payload_len;
+	int qp0;
+	int rc;
+
+	/* RcvHdrFlags are at the end of the header entry */
+	rcvhdrflags = ( qib7322_wq->header + header_offs +
+			QIB7322_RECV_HEADER_SIZE - sizeof ( *rcvhdrflags ) );
+	rcvtype = BIT_GET ( rcvhdrflags, RcvType );
+	pktlen = ( BIT_GET ( rcvhdrflags, PktLen ) << 2 );
+	egrindex = BIT_GET ( rcvhdrflags, EgrIndex );
+	useegrbfr = BIT_GET ( rcvhdrflags, UseEgrBfr );
+	hdrqoffset = ( BIT_GET ( rcvhdrflags, HdrqOffset ) << 2 );
+	iberr = BIT_GET ( rcvhdrflags, IBErr );
+	mkerr = BIT_GET ( rcvhdrflags, MKErr );
+	tiderr = BIT_GET ( rcvhdrflags, TIDErr );
+	khdrerr = BIT_GET ( rcvhdrflags, KHdrErr );
+	mtuerr = BIT_GET ( rcvhdrflags, MTUErr );
+	lenerr = BIT_GET ( rcvhdrflags, LenErr );
+	parityerr = BIT_GET ( rcvhdrflags, ParityErr );
+	vcrcerr = BIT_GET ( rcvhdrflags, VCRCErr );
+	icrcerr = BIT_GET ( rcvhdrflags, ICRCErr );
+	header_len = ( QIB7322_RECV_HEADER_SIZE - hdrqoffset -
+		       sizeof ( *rcvhdrflags ) );
+	padded_payload_len = ( pktlen - header_len - 4 /* ICRC */ );
+	err = ( iberr | mkerr | tiderr | khdrerr | mtuerr |
+		lenerr | parityerr | vcrcerr | icrcerr );
+	/* IB header is placed immediately before RcvHdrFlags */
+	iob_populate ( &headers, ( ( ( void * ) rcvhdrflags ) - header_len ),
+		       header_len, header_len );
+
+	/* Dump diagnostic information */
+	DBGC2 ( qib7322, "QIB7322 %p QPN %ld RX egr %04x%s hdr %d type %d len "
+		"%d(%d+%d+4)%s%s%s%s%s%s%s%s%s%s%s\n", qib7322, qp->qpn,
+		egrindex, ( useegrbfr ? "" : "(unused)" ),
+		( header_offs / QIB7322_RECV_HEADER_SIZE ),
+		rcvtype, pktlen, header_len, padded_payload_len,
+		( err ? " [Err" : "" ), ( iberr ? " IB" : "" ),
+		( mkerr ? " MK" : "" ), ( tiderr ? " TID" : "" ),
+		( khdrerr ? " KHdr" : "" ), ( mtuerr ? " MTU" : "" ),
+		( lenerr ? " Len" : "" ), ( parityerr ? " Parity" : ""),
+		( vcrcerr ? " VCRC" : "" ), ( icrcerr ? " ICRC" : "" ),
+		( err ? "]" : "" ) );
+	DBGCP_HDA ( qib7322, hdrqoffset, headers.data,
+		    ( header_len + sizeof ( *rcvhdrflags ) ) );
+
+	/* Parse header to generate address vector */
+	qp0 = ( qp->qpn == 0 );
+	intended_qp = NULL;
+	if ( ( rc = ib_pull ( ibdev, &headers, ( qp0 ? &intended_qp : NULL ),
+			      &payload_len, &dest, &source ) ) != 0 ) {
+		DBGC ( qib7322, "QIB7322 %p could not parse headers: %s\n",
+		       qib7322, strerror ( rc ) );
+		err = 1;
+	}
+	if ( ! intended_qp )
+		intended_qp = qp;
+
+	/* Complete this buffer and any skipped buffers.  Note that
+	 * when the hardware runs out of buffers, it will repeatedly
+	 * report the same buffer (the tail) as a TID error, and that
+	 * it also has a habit of sometimes skipping over several
+	 * buffers at once.
+	 */
+	while ( 1 ) {
+
+		/* If we have caught up to the producer counter, stop.
+		 * This will happen when the hardware first runs out
+		 * of buffers and starts reporting TID errors against
+		 * the eager buffer it wants to use next.
+		 */
+		if ( qib7322_wq->eager_cons == qib7322_wq->eager_prod )
+			break;
+
+		/* If we have caught up to where we should be after
+		 * completing this egrindex, stop.  We phrase the test
+		 * this way to avoid completing the entire ring when
+		 * we receive the same egrindex twice in a row.
+		 */
+		if ( ( qib7322_wq->eager_cons ==
+		       ( ( egrindex + 1 ) & ( qib7322_wq->eager_entries - 1 ))))
+			break;
+
+		/* Identify work queue entry and corresponding I/O
+		 * buffer.
+		 */
+		wqe_idx = ( qib7322_wq->eager_cons & ( wq->num_wqes - 1 ) );
+		iobuf = wq->iobufs[wqe_idx];
+		assert ( iobuf != NULL );
+		wq->iobufs[wqe_idx] = NULL;
+
+		/* Complete the eager buffer */
+		if ( qib7322_wq->eager_cons == egrindex ) {
+			/* Completing the eager buffer described in
+			 * this header entry.
+			 */
+			iob_put ( iobuf, payload_len );
+			rc = ( err ? -EIO : ( useegrbfr ? 0 : -ECANCELED ) );
+			/* Redirect to target QP if necessary */
+			if ( qp != intended_qp ) {
+				DBGC2 ( qib7322, "QIB7322 %p redirecting QPN "
+					"%ld => %ld\n",
+					qib7322, qp->qpn, intended_qp->qpn );
+				/* Compensate for incorrect fill levels */
+				qp->recv.fill--;
+				intended_qp->recv.fill++;
+			}
+			ib_complete_recv ( ibdev, intended_qp, &dest, &source,
+					   iobuf, rc);
+		} else {
+			/* Completing on a skipped-over eager buffer */
+			ib_complete_recv ( ibdev, qp, &dest, &source, iobuf,
+					   -ECANCELED );
+		}
+
+		/* Clear eager buffer */
+		memset ( &rcvegr, 0, sizeof ( rcvegr ) );
+		qib7322_writeq_array8b ( qib7322, &rcvegr,
+					 qib7322_wq->eager_array,
+					 qib7322_wq->eager_cons );
+
+		/* Increment consumer index */
+		qib7322_wq->eager_cons = ( ( qib7322_wq->eager_cons + 1 ) &
+					   ( qib7322_wq->eager_entries - 1 ) );
+	}
+}
+
+/**
+ * Poll receive work queue
+ *
+ * @v ibdev		Infiniband device
+ * @v qp		Queue pair
+ */
+static void qib7322_poll_recv_wq ( struct ib_device *ibdev,
+				   struct ib_queue_pair *qp ) {
+	struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
+	struct ib_work_queue *wq = &qp->recv;
+	struct qib7322_recv_work_queue *qib7322_wq = ib_wq_get_drvdata ( wq );
+	struct QIB_7322_RcvHdrHead0 rcvhdrhead;
+	unsigned int ctx = qib7322_ctx ( ibdev, qp );
+	unsigned int header_prod;
+
+	/* Check for received packets */
+	header_prod = ( BIT_GET ( &qib7322_wq->header_prod, Value ) << 2 );
+	if ( header_prod == qib7322_wq->header_cons )
+		return;
+
+	/* Process all received packets */
+	while ( qib7322_wq->header_cons != header_prod ) {
+
+		/* Complete the receive */
+		qib7322_complete_recv ( ibdev, qp, qib7322_wq->header_cons );
+
+		/* Increment the consumer offset */
+		qib7322_wq->header_cons += QIB7322_RECV_HEADER_SIZE;
+		qib7322_wq->header_cons %= QIB7322_RECV_HEADERS_SIZE;
+
+		/* QIB7322 has only one send buffer per port for VL15,
+		 * which almost always leads to send buffer exhaustion
+		 * and dropped MADs.  Mitigate this by refusing to
+		 * process more than one VL15 MAD per poll, which will
+		 * enforce interleaved TX/RX polls.
+		 */
+		if ( qp->type == IB_QPT_SMI )
+			break;
+	}
+
+	/* Update consumer offset */
+	memset ( &rcvhdrhead, 0, sizeof ( rcvhdrhead ) );
+	BIT_FILL_2 ( &rcvhdrhead,
+		     RcvHeadPointer, ( qib7322_wq->header_cons >> 2 ),
+		     counter, 1 );
+	qib7322_writeq_array64k ( qib7322, &rcvhdrhead,
+				  QIB_7322_RcvHdrHead0_offset, ctx );
+}
+
+/**
+ * Poll completion queue
+ *
+ * @v ibdev		Infiniband device
+ * @v cq		Completion queue
+ */
+static void qib7322_poll_cq ( struct ib_device *ibdev,
+			      struct ib_completion_queue *cq ) {
+	struct ib_work_queue *wq;
+
+	/* Poll associated send and receive queues */
+	list_for_each_entry ( wq, &cq->work_queues, list ) {
+		if ( wq->is_send ) {
+			qib7322_poll_send_wq ( ibdev, wq->qp );
+		} else {
+			qib7322_poll_recv_wq ( ibdev, wq->qp );
+		}
+	}
+}
+
+/***************************************************************************
+ *
+ * Event queues
+ *
+ ***************************************************************************
+ */
+
+/**
+ * Poll event queue
+ *
+ * @v ibdev		Infiniband device
+ */
+static void qib7322_poll_eq ( struct ib_device *ibdev ) {
+	struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
+	struct QIB_7322_ErrStatus_0 errstatus;
+	unsigned int port = ( ibdev->port - QIB7322_PORT_BASE );
+
+	/* Check for and clear status bits */
+	DBG_DISABLE ( DBGLVL_IO );
+	qib7322_readq_port ( qib7322, &errstatus,
+			     QIB_7322_ErrStatus_0_offset, port );
+	if ( errstatus.u.qwords[0] ) {
+		DBGC ( qib7322, "QIB7322 %p port %d status %08x%08x\n", qib7322,
+		       port, errstatus.u.dwords[1],  errstatus.u.dwords[0] );
+		qib7322_writeq_port ( qib7322, &errstatus,
+				      QIB_7322_ErrClear_0_offset, port );
+	}
+	DBG_ENABLE ( DBGLVL_IO );
+
+	/* Check for link status changes */
+	if ( BIT_GET ( &errstatus, IBStatusChanged ) )
+		qib7322_link_state_changed ( ibdev );
+}
+
+/***************************************************************************
+ *
+ * Infiniband link-layer operations
+ *
+ ***************************************************************************
+ */
+
+/**
+ * Determine supported link speeds
+ *
+ * @v qib7322		QIB7322 device
+ * @ret supported	Supported link speeds
+ */
+static unsigned int qib7322_link_speed_supported ( struct qib7322 *qib7322,
+						   unsigned int port ) {
+	struct QIB_7322_feature_mask features;
+	struct QIB_7322_Revision revision;
+	unsigned int supported;
+	unsigned int boardid;
+
+	/* Read the active feature mask */
+	qib7322_readq ( qib7322, &features,
+			QIB_7322_active_feature_mask_offset );
+	switch ( port ) {
+	case 0 :
+		supported = BIT_GET ( &features, Port0_Link_Speed_Supported );
+		break;
+	case 1 :
+		supported = BIT_GET ( &features, Port1_Link_Speed_Supported );
+		break;
+	default:
+		DBGC ( qib7322, "QIB7322 %p port %d is invalid\n",
+		       qib7322, port );
+		supported = 0;
+		break;
+	}
+
+	/* Apply hacks for specific board IDs */
+	qib7322_readq ( qib7322, &revision, QIB_7322_Revision_offset );
+	boardid = BIT_GET ( &revision, BoardID );
+	switch ( boardid ) {
+	case QIB7322_BOARD_QMH7342 :
+		DBGC2 ( qib7322, "QIB7322 %p is a QMH7342; forcing QDR-only\n",
+			qib7322 );
+		supported = IB_LINK_SPEED_QDR;
+		break;
+	default:
+		/* Do nothing */
+		break;
+	}
+
+	DBGC2 ( qib7322, "QIB7322 %p port %d %s%s%s%s\n", qib7322, port,
+		( supported ? "supports" : "disabled" ),
+		( ( supported & IB_LINK_SPEED_SDR ) ? " SDR" : "" ),
+		( ( supported & IB_LINK_SPEED_DDR ) ? " DDR" : "" ),
+		( ( supported & IB_LINK_SPEED_QDR ) ? " QDR" : "" ) );
+	return supported;
+}
+
+/**
+ * Initialise Infiniband link
+ *
+ * @v ibdev		Infiniband device
+ * @ret rc		Return status code
+ */
+static int qib7322_open ( struct ib_device *ibdev ) {
+	struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
+	struct QIB_7322_IBCCtrlA_0 ibcctrla;
+	unsigned int port = ( ibdev->port - QIB7322_PORT_BASE );
+
+	/* Enable link */
+	qib7322_readq_port ( qib7322, &ibcctrla,
+			     QIB_7322_IBCCtrlA_0_offset, port );
+	BIT_SET ( &ibcctrla, IBLinkEn, 1 );
+	qib7322_writeq_port ( qib7322, &ibcctrla,
+			      QIB_7322_IBCCtrlA_0_offset, port );
+
+	return 0;
+}
+
+/**
+ * Close Infiniband link
+ *
+ * @v ibdev		Infiniband device
+ */
+static void qib7322_close ( struct ib_device *ibdev ) {
+	struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
+	struct QIB_7322_IBCCtrlA_0 ibcctrla;
+	unsigned int port = ( ibdev->port - QIB7322_PORT_BASE );
+
+	/* Disable link */
+	qib7322_readq_port ( qib7322, &ibcctrla,
+			     QIB_7322_IBCCtrlA_0_offset, port );
+	BIT_SET ( &ibcctrla, IBLinkEn, 0 );
+	qib7322_writeq_port ( qib7322, &ibcctrla,
+			      QIB_7322_IBCCtrlA_0_offset, port );
+}
+
+/***************************************************************************
+ *
+ * Multicast group operations
+ *
+ ***************************************************************************
+ */
+
+/**
+ * Attach to multicast group
+ *
+ * @v ibdev		Infiniband device
+ * @v qp		Queue pair
+ * @v gid		Multicast GID
+ * @ret rc		Return status code
+ */
+static int qib7322_mcast_attach ( struct ib_device *ibdev,
+				  struct ib_queue_pair *qp,
+				  union ib_gid *gid ) {
+	struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
+
+	( void ) qib7322;
+	( void ) qp;
+	( void ) gid;
+	return 0;
+}
+
+/**
+ * Detach from multicast group
+ *
+ * @v ibdev		Infiniband device
+ * @v qp		Queue pair
+ * @v gid		Multicast GID
+ */
+static void qib7322_mcast_detach ( struct ib_device *ibdev,
+				   struct ib_queue_pair *qp,
+				   union ib_gid *gid ) {
+	struct qib7322 *qib7322 = ib_get_drvdata ( ibdev );
+
+	( void ) qib7322;
+	( void ) qp;
+	( void ) gid;
+ }
+
+/** QIB7322 Infiniband operations */
+static struct ib_device_operations qib7322_ib_operations = {
+	.create_cq	= qib7322_create_cq,
+	.destroy_cq	= qib7322_destroy_cq,
+	.create_qp	= qib7322_create_qp,
+	.modify_qp	= qib7322_modify_qp,
+	.destroy_qp	= qib7322_destroy_qp,
+	.post_send	= qib7322_post_send,
+	.post_recv	= qib7322_post_recv,
+	.poll_cq	= qib7322_poll_cq,
+	.poll_eq	= qib7322_poll_eq,
+	.open		= qib7322_open,
+	.close		= qib7322_close,
+	.mcast_attach	= qib7322_mcast_attach,
+	.mcast_detach	= qib7322_mcast_detach,
+	.set_port_info	= qib7322_set_port_info,
+	.set_pkey_table	= qib7322_set_pkey_table,
+};
+
+/***************************************************************************
+ *
+ * I2C bus operations
+ *
+ ***************************************************************************
+ */
+
+/** QIB7322 I2C bit to GPIO mappings */
+static unsigned int qib7322_i2c_bits[] = {
+	[I2C_BIT_SCL] = ( 1 << QIB7322_GPIO_SCL ),
+	[I2C_BIT_SDA] = ( 1 << QIB7322_GPIO_SDA ),
+};
+
+/**
+ * Read QIB7322 I2C line status
+ *
+ * @v basher		Bit-bashing interface
+ * @v bit_id		Bit number
+ * @ret zero		Input is a logic 0
+ * @ret non-zero	Input is a logic 1
+ */
+static int qib7322_i2c_read_bit ( struct bit_basher *basher,
+				  unsigned int bit_id ) {
+	struct qib7322 *qib7322 =
+		container_of ( basher, struct qib7322, i2c.basher );
+	struct QIB_7322_EXTStatus extstatus;
+	unsigned int status;
+
+	DBG_DISABLE ( DBGLVL_IO );
+
+	qib7322_readq ( qib7322, &extstatus, QIB_7322_EXTStatus_offset );
+	status = ( BIT_GET ( &extstatus, GPIOIn ) & qib7322_i2c_bits[bit_id] );
+
+	DBG_ENABLE ( DBGLVL_IO );
+
+	return status;
+}
+
+/**
+ * Write QIB7322 I2C line status
+ *
+ * @v basher		Bit-bashing interface
+ * @v bit_id		Bit number
+ * @v data		Value to write
+ */
+static void qib7322_i2c_write_bit ( struct bit_basher *basher,
+				    unsigned int bit_id, unsigned long data ) {
+	struct qib7322 *qib7322 =
+		container_of ( basher, struct qib7322, i2c.basher );
+	struct QIB_7322_EXTCtrl extctrl;
+	struct QIB_7322_GPIO gpioout;
+	unsigned int bit = qib7322_i2c_bits[bit_id];
+	unsigned int outputs = 0;
+	unsigned int output_enables = 0;
+
+	DBG_DISABLE ( DBGLVL_IO );
+
+	/* Read current GPIO mask and outputs */
+	qib7322_readq ( qib7322, &extctrl, QIB_7322_EXTCtrl_offset );
+	qib7322_readq ( qib7322, &gpioout, QIB_7322_GPIOOut_offset );
+
+	/* Update outputs and output enables.  I2C lines are tied
+	 * high, so we always set the output to 0 and use the output
+	 * enable to control the line.
+	 */
+	output_enables = BIT_GET ( &extctrl, GPIOOe );
+	output_enables = ( ( output_enables & ~bit ) | ( ~data & bit ) );
+	outputs = BIT_GET ( &gpioout, GPIO );
+	outputs = ( outputs & ~bit );
+	BIT_SET ( &extctrl, GPIOOe, output_enables );
+	BIT_SET ( &gpioout, GPIO, outputs );
+
+	/* Write the output enable first; that way we avoid logic
+	 * hazards.
+	 */
+	qib7322_writeq ( qib7322, &extctrl, QIB_7322_EXTCtrl_offset );
+	qib7322_writeq ( qib7322, &gpioout, QIB_7322_GPIOOut_offset );
+	mb();
+
+	DBG_ENABLE ( DBGLVL_IO );
+}
+
+/** QIB7322 I2C bit-bashing interface operations */
+static struct bit_basher_operations qib7322_i2c_basher_ops = {
+	.read	= qib7322_i2c_read_bit,
+	.write	= qib7322_i2c_write_bit,
+};
+
+/**
+ * Initialise QIB7322 I2C subsystem
+ *
+ * @v qib7322		QIB7322 device
+ * @ret rc		Return status code
+ */
+static int qib7322_init_i2c ( struct qib7322 *qib7322 ) {
+	static int try_eeprom_address[] = { 0x51, 0x50 };
+	unsigned int i;
+	int rc;
+
+	/* Initialise bus */
+	if ( ( rc = init_i2c_bit_basher ( &qib7322->i2c,
+					  &qib7322_i2c_basher_ops ) ) != 0 ) {
+		DBGC ( qib7322, "QIB7322 %p could not initialise I2C bus: %s\n",
+		       qib7322, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Probe for devices */
+	for ( i = 0 ; i < ( sizeof ( try_eeprom_address ) /
+			    sizeof ( try_eeprom_address[0] ) ) ; i++ ) {
+		init_i2c_eeprom ( &qib7322->eeprom, try_eeprom_address[i] );
+		if ( ( rc = i2c_check_presence ( &qib7322->i2c.i2c,
+						 &qib7322->eeprom ) ) == 0 ) {
+			DBGC2 ( qib7322, "QIB7322 %p found EEPROM at %02x\n",
+				qib7322, try_eeprom_address[i] );
+			return 0;
+		}
+	}
+
+	DBGC ( qib7322, "QIB7322 %p could not find EEPROM\n", qib7322 );
+	return -ENODEV;
+}
+
+/**
+ * Read EEPROM parameters
+ *
+ * @v qib7322		QIB7322 device
+ * @ret rc		Return status code
+ */
+static int qib7322_read_eeprom ( struct qib7322 *qib7322 ) {
+	struct i2c_interface *i2c = &qib7322->i2c.i2c;
+	union ib_guid *guid = &qib7322->guid;
+	int rc;
+
+	/* Read GUID */
+	if ( ( rc = i2c->read ( i2c, &qib7322->eeprom,
+				QIB7322_EEPROM_GUID_OFFSET, guid->bytes,
+				sizeof ( *guid ) ) ) != 0 ) {
+		DBGC ( qib7322, "QIB7322 %p could not read GUID: %s\n",
+		       qib7322, strerror ( rc ) );
+		return rc;
+	}
+	DBGC2 ( qib7322, "QIB7322 %p has GUID " IB_GUID_FMT "\n",
+		qib7322, IB_GUID_ARGS ( guid ) );
+
+	/* Read serial number (debug only) */
+	if ( DBG_LOG ) {
+		uint8_t serial[QIB7322_EEPROM_SERIAL_SIZE + 1];
+
+		serial[ sizeof ( serial ) - 1 ] = '\0';
+		if ( ( rc = i2c->read ( i2c, &qib7322->eeprom,
+					QIB7322_EEPROM_SERIAL_OFFSET, serial,
+					( sizeof ( serial ) - 1 ) ) ) != 0 ) {
+			DBGC ( qib7322, "QIB7322 %p could not read serial: "
+			       "%s\n", qib7322, strerror ( rc ) );
+			return rc;
+		}
+		DBGC2 ( qib7322, "QIB7322 %p has serial number \"%s\"\n",
+			qib7322, serial );
+	}
+
+	return 0;
+}
+
+/***************************************************************************
+ *
+ * Advanced High-performance Bus (AHB) access
+ *
+ ***************************************************************************
+ */
+
+/**
+ * Wait for AHB transaction to complete
+ *
+ * @v qib7322		QIB7322 device
+ * @ret rc		Return status code
+ */
+static int qib7322_ahb_wait ( struct qib7322 *qib7322 ) {
+	struct QIB_7322_ahb_transaction_reg transaction;
+	unsigned int i;
+
+	/* Wait for Ready bit to be asserted */
+	for ( i = 0 ; i < QIB7322_AHB_MAX_WAIT_US ; i++ ) {
+		qib7322_readq ( qib7322, &transaction,
+				QIB_7322_ahb_transaction_reg_offset );
+		if ( BIT_GET ( &transaction, ahb_rdy ) )
+			return 0;
+		udelay ( 1 );
+	}
+
+	DBGC ( qib7322, "QIB7322 %p timed out waiting for AHB transaction\n",
+	       qib7322 );
+	return -ETIMEDOUT;
+}
+
+/**
+ * Request ownership of the AHB
+ *
+ * @v qib7322		QIB7322 device
+ * @v location		AHB location
+ * @ret rc		Return status code
+ */
+static int qib7322_ahb_request ( struct qib7322 *qib7322,
+				 unsigned int location ) {
+	struct QIB_7322_ahb_access_ctrl access;
+	int rc;
+
+	/* Request ownership */
+	memset ( &access, 0, sizeof ( access ) );
+	BIT_FILL_2 ( &access,
+		     sw_ahb_sel, 1,
+		     sw_sel_ahb_trgt, QIB7322_AHB_LOC_TARGET ( location ) );
+	qib7322_writeq ( qib7322, &access, QIB_7322_ahb_access_ctrl_offset );
+
+	/* Wait for ownership to be granted */
+	if ( ( rc = qib7322_ahb_wait ( qib7322 ) ) != 0 )  {
+		DBGC ( qib7322, "QIB7322 %p could not obtain AHB ownership: "
+		       "%s\n", qib7322, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Release ownership of the AHB
+ *
+ * @v qib7322		QIB7322 device
+ */
+static void qib7322_ahb_release ( struct qib7322 *qib7322 ) {
+	struct QIB_7322_ahb_access_ctrl access;
+
+	memset ( &access, 0, sizeof ( access ) );
+	qib7322_writeq ( qib7322, &access, QIB_7322_ahb_access_ctrl_offset );
+}
+
+/**
+ * Read data via AHB
+ *
+ * @v qib7322		QIB7322 device
+ * @v location		AHB location
+ * @v data		Data to read
+ * @ret rc		Return status code
+ *
+ * You must have already acquired ownership of the AHB.
+ */
+static int qib7322_ahb_read ( struct qib7322 *qib7322, unsigned int location,
+			      uint32_t *data ) {
+	struct QIB_7322_ahb_transaction_reg xact;
+	int rc;
+
+	/* Avoid returning uninitialised data on error */
+	*data = 0;
+
+	/* Initiate transaction */
+	memset ( &xact, 0, sizeof ( xact ) );
+	BIT_FILL_2 ( &xact,
+		     ahb_address, QIB7322_AHB_LOC_ADDRESS ( location ),
+		     write_not_read, 0 );
+	qib7322_writeq ( qib7322, &xact, QIB_7322_ahb_transaction_reg_offset );
+
+	/* Wait for transaction to complete */
+	if ( ( rc = qib7322_ahb_wait ( qib7322 ) ) != 0 )
+		return rc;
+
+	/* Read transaction data */
+	qib7322_readq ( qib7322, &xact, QIB_7322_ahb_transaction_reg_offset );
+	*data = BIT_GET ( &xact, ahb_data );
+	return 0;
+}
+
+/**
+ * Write data via AHB
+ *
+ * @v qib7322		QIB7322 device
+ * @v location		AHB location
+ * @v data		Data to write
+ * @ret rc		Return status code
+ *
+ * You must have already acquired ownership of the AHB.
+ */
+static int qib7322_ahb_write ( struct qib7322 *qib7322, unsigned int location,
+			       uint32_t data ) {
+	struct QIB_7322_ahb_transaction_reg xact;
+	int rc;
+
+	/* Initiate transaction */
+	memset ( &xact, 0, sizeof ( xact ) );
+	BIT_FILL_3 ( &xact,
+		     ahb_address, QIB7322_AHB_LOC_ADDRESS ( location ),
+		     write_not_read, 1,
+		     ahb_data, data );
+	qib7322_writeq ( qib7322, &xact, QIB_7322_ahb_transaction_reg_offset );
+
+	/* Wait for transaction to complete */
+	if ( ( rc = qib7322_ahb_wait ( qib7322 ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Read/modify/write AHB register
+ *
+ * @v qib7322		QIB7322 device
+ * @v location		AHB location
+ * @v value		Value to set
+ * @v mask		Mask to apply to old value
+ * @ret rc		Return status code
+ */
+static int qib7322_ahb_mod_reg ( struct qib7322 *qib7322, unsigned int location,
+				 uint32_t value, uint32_t mask ) {
+	uint32_t old_value;
+	uint32_t new_value;
+	int rc;
+
+	DBG_DISABLE ( DBGLVL_IO );
+
+	/* Sanity check */
+	assert ( ( value & mask ) == value );
+
+	/* Acquire bus ownership */
+	if ( ( rc = qib7322_ahb_request ( qib7322, location ) ) != 0 )
+		goto out;
+
+	/* Read existing value */
+	if ( ( rc = qib7322_ahb_read ( qib7322, location, &old_value ) ) != 0 )
+		goto out_release;
+
+	/* Update value */
+	new_value = ( ( old_value & ~mask ) | value );
+	DBGCP ( qib7322, "QIB7322 %p AHB %x %#08x => %#08x\n",
+		qib7322, location, old_value, new_value );
+	if ( ( rc = qib7322_ahb_write ( qib7322, location, new_value ) ) != 0 )
+		goto out_release;
+
+ out_release:
+	/* Release bus */
+	qib7322_ahb_release ( qib7322 );
+ out:
+	DBG_ENABLE ( DBGLVL_IO );
+	return rc;
+}
+
+/**
+ * Read/modify/write AHB register across all ports and channels
+ *
+ * @v qib7322		QIB7322 device
+ * @v reg		AHB register
+ * @v value		Value to set
+ * @v mask		Mask to apply to old value
+ * @ret rc		Return status code
+ */
+static int qib7322_ahb_mod_reg_all ( struct qib7322 *qib7322, unsigned int reg,
+				     uint32_t value, uint32_t mask ) {
+	unsigned int port;
+	unsigned int channel;
+	unsigned int location;
+	int rc;
+
+	for ( port = 0 ; port < QIB7322_MAX_PORTS ; port++ ) {
+		for ( channel = 0 ; channel < QIB7322_MAX_WIDTH ; channel++ ) {
+			location = QIB7322_AHB_LOCATION ( port, channel, reg );
+			if ( ( rc = qib7322_ahb_mod_reg ( qib7322, location,
+							  value, mask ) ) != 0 )
+				return rc;
+		}
+	}
+	return 0;
+}
+
+/***************************************************************************
+ *
+ * Infiniband SerDes initialisation
+ *
+ ***************************************************************************
+ */
+
+/**
+ * Initialise the IB SerDes
+ *
+ * @v qib7322		QIB7322 device
+ * @ret rc		Return status code
+ */
+static int qib7322_init_ib_serdes ( struct qib7322 *qib7322 ) {
+	struct QIB_7322_IBCCtrlA_0 ibcctrla;
+	struct QIB_7322_IBCCtrlB_0 ibcctrlb;
+	struct QIB_7322_IBPCSConfig_0 ibpcsconfig;
+
+	/* Configure sensible defaults for IBC */
+	memset ( &ibcctrla, 0, sizeof ( ibcctrla ) );
+	BIT_FILL_5 ( &ibcctrla, /* Tuning values taken from Linux driver */
+		     FlowCtrlPeriod, 0x03,
+		     FlowCtrlWaterMark, 0x05,
+		     MaxPktLen, ( ( QIB7322_RECV_HEADER_SIZE +
+				    QIB7322_RECV_PAYLOAD_SIZE +
+				    4 /* ICRC */ ) >> 2 ),
+		     PhyerrThreshold, 0xf,
+		     OverrunThreshold, 0xf );
+	qib7322_writeq ( qib7322, &ibcctrla, QIB_7322_IBCCtrlA_0_offset );
+	qib7322_writeq ( qib7322, &ibcctrla, QIB_7322_IBCCtrlA_1_offset );
+
+	/* Force SDR only to avoid needing all the DDR tuning,
+	 * Mellanox compatibility hacks etc.  SDR is plenty for
+	 * boot-time operation.
+	 */
+	qib7322_readq ( qib7322, &ibcctrlb, QIB_7322_IBCCtrlB_0_offset );
+	BIT_SET ( &ibcctrlb, IB_ENHANCED_MODE, 0 );
+	BIT_SET ( &ibcctrlb, SD_SPEED_SDR, 1 );
+	BIT_SET ( &ibcctrlb, SD_SPEED_DDR, 0 );
+	BIT_SET ( &ibcctrlb, SD_SPEED_QDR, 0 );
+	BIT_SET ( &ibcctrlb, IB_NUM_CHANNELS, 1 ); /* 4X only */
+	BIT_SET ( &ibcctrlb, IB_LANE_REV_SUPPORTED, 0 );
+	BIT_SET ( &ibcctrlb, HRTBT_ENB, 0 );
+	BIT_SET ( &ibcctrlb, HRTBT_AUTO, 0 );
+	qib7322_writeq ( qib7322, &ibcctrlb, QIB_7322_IBCCtrlB_0_offset );
+	qib7322_writeq ( qib7322, &ibcctrlb, QIB_7322_IBCCtrlB_1_offset );
+
+	/* Tune SerDes */
+	qib7322_ahb_mod_reg_all ( qib7322, 2, 0, 0x00000e00UL );
+
+	/* Bring XGXS out of reset */
+	memset ( &ibpcsconfig, 0, sizeof ( ibpcsconfig ) );
+	qib7322_writeq ( qib7322, &ibpcsconfig, QIB_7322_IBPCSConfig_0_offset );
+	qib7322_writeq ( qib7322, &ibpcsconfig, QIB_7322_IBPCSConfig_1_offset );
+
+	return 0;
+}
+
+/***************************************************************************
+ *
+ * PCI layer interface
+ *
+ ***************************************************************************
+ */
+
+/**
+ * Reset QIB7322
+ *
+ * @v qib7322		QIB7322 device
+ * @v pci		PCI device
+ * @ret rc		Return status code
+ */
+static void qib7322_reset ( struct qib7322 *qib7322, struct pci_device *pci ) {
+	struct QIB_7322_Control control;
+	struct pci_config_backup backup;
+
+	/* Back up PCI configuration space */
+	pci_backup ( pci, &backup, NULL );
+
+	/* Assert reset */
+	memset ( &control, 0, sizeof ( control ) );
+	BIT_FILL_1 ( &control, SyncReset, 1 );
+	qib7322_writeq ( qib7322, &control, QIB_7322_Control_offset );
+
+	/* Wait for reset to complete */
+	mdelay ( 1000 );
+
+	/* Restore PCI configuration space */
+	pci_restore ( pci, &backup, NULL );
+}
+
+/**
+ * Probe PCI device
+ *
+ * @v pci		PCI device
+ * @v id		PCI ID
+ * @ret rc		Return status code
+ */
+static int qib7322_probe ( struct pci_device *pci ) {
+	struct qib7322 *qib7322;
+	struct QIB_7322_Revision revision;
+	struct ib_device *ibdev;
+	unsigned int link_speed_supported;
+	int i;
+	int rc;
+
+	/* Allocate QIB7322 device */
+	qib7322 = zalloc ( sizeof ( *qib7322 ) );
+	if ( ! qib7322 ) {
+		rc = -ENOMEM;
+		goto err_alloc_qib7322;
+	}
+	pci_set_drvdata ( pci, qib7322 );
+
+	/* Fix up PCI device */
+	adjust_pci_device ( pci );
+
+	/* Get PCI BARs */
+	qib7322->regs = ioremap ( pci->membase, QIB7322_BAR0_SIZE );
+	DBGC2 ( qib7322, "QIB7322 %p has BAR at %08lx\n",
+		qib7322, pci->membase );
+
+	/* Reset device */
+	qib7322_reset ( qib7322, pci );
+
+	/* Print some general data */
+	qib7322_readq ( qib7322, &revision, QIB_7322_Revision_offset );
+	DBGC2 ( qib7322, "QIB7322 %p board %02lx v%ld.%ld.%ld.%ld\n", qib7322,
+		BIT_GET ( &revision, BoardID ),
+		BIT_GET ( &revision, R_SW ),
+		BIT_GET ( &revision, R_Arch ),
+		BIT_GET ( &revision, R_ChipRevMajor ),
+		BIT_GET ( &revision, R_ChipRevMinor ) );
+
+	/* Initialise I2C subsystem */
+	if ( ( rc = qib7322_init_i2c ( qib7322 ) ) != 0 )
+		goto err_init_i2c;
+
+	/* Read EEPROM parameters */
+	if ( ( rc = qib7322_read_eeprom ( qib7322 ) ) != 0 )
+		goto err_read_eeprom;
+
+	/* Initialise send datapath */
+	if ( ( rc = qib7322_init_send ( qib7322 ) ) != 0 )
+		goto err_init_send;
+
+	/* Initialise receive datapath */
+	if ( ( rc = qib7322_init_recv ( qib7322 ) ) != 0 )
+		goto err_init_recv;
+
+	/* Initialise the IB SerDes */
+	if ( ( rc = qib7322_init_ib_serdes ( qib7322 ) ) != 0 )
+		goto err_init_ib_serdes;
+
+	/* Allocate Infiniband devices */
+	for ( i = 0 ; i < QIB7322_MAX_PORTS ; i++ ) {
+		link_speed_supported =
+			qib7322_link_speed_supported ( qib7322, i );
+		if ( ! link_speed_supported )
+			continue;
+		ibdev = alloc_ibdev ( 0 );
+		if ( ! ibdev ) {
+			rc = -ENOMEM;
+			goto err_alloc_ibdev;
+		}
+		qib7322->ibdev[i] = ibdev;
+		ibdev->dev = &pci->dev;
+		ibdev->op = &qib7322_ib_operations;
+		ibdev->port = ( QIB7322_PORT_BASE + i );
+		ibdev->link_width_enabled = ibdev->link_width_supported =
+			IB_LINK_WIDTH_4X; /* 1x does not work */
+		ibdev->link_speed_enabled = ibdev->link_speed_supported =
+			IB_LINK_SPEED_SDR; /* to avoid need for link tuning */
+		memcpy ( &ibdev->node_guid, &qib7322->guid,
+			 sizeof ( ibdev->node_guid ) );
+		memcpy ( &ibdev->gid.s.guid, &qib7322->guid,
+			 sizeof ( ibdev->gid.s.guid ) );
+		assert ( ( ibdev->gid.s.guid.bytes[7] & i ) == 0 );
+		ibdev->gid.s.guid.bytes[7] |= i;
+		ib_set_drvdata ( ibdev, qib7322 );
+	}
+
+	/* Register Infiniband devices */
+	for ( i = 0 ; i < QIB7322_MAX_PORTS ; i++ ) {
+		if ( ! qib7322->ibdev[i] )
+			continue;
+		if ( ( rc = register_ibdev ( qib7322->ibdev[i] ) ) != 0 ) {
+			DBGC ( qib7322, "QIB7322 %p port %d could not register "
+			       "IB device: %s\n", qib7322, i, strerror ( rc ) );
+			goto err_register_ibdev;
+		}
+	}
+
+	return 0;
+
+	i = QIB7322_MAX_PORTS;
+ err_register_ibdev:
+	for ( i-- ; i >= 0 ; i-- ) {
+		if ( qib7322->ibdev[i] )
+			unregister_ibdev ( qib7322->ibdev[i] );
+	}
+	i = QIB7322_MAX_PORTS;
+ err_alloc_ibdev:
+	for ( i-- ; i >= 0 ; i-- )
+		ibdev_put ( qib7322->ibdev[i] );
+ err_init_ib_serdes:
+	qib7322_fini_send ( qib7322 );
+ err_init_send:
+	qib7322_fini_recv ( qib7322 );
+ err_init_recv:
+ err_read_eeprom:
+ err_init_i2c:
+	free ( qib7322 );
+ err_alloc_qib7322:
+	return rc;
+}
+
+/**
+ * Remove PCI device
+ *
+ * @v pci		PCI device
+ */
+static void qib7322_remove ( struct pci_device *pci ) {
+	struct qib7322 *qib7322 = pci_get_drvdata ( pci );
+	int i;
+
+	for ( i = ( QIB7322_MAX_PORTS - 1 ) ; i >= 0 ; i-- ) {
+		if ( qib7322->ibdev[i] )
+			unregister_ibdev ( qib7322->ibdev[i] );
+	}
+	for ( i = ( QIB7322_MAX_PORTS - 1 ) ; i >= 0 ; i-- )
+		ibdev_put ( qib7322->ibdev[i] );
+	qib7322_fini_send ( qib7322 );
+	qib7322_fini_recv ( qib7322 );
+	free ( qib7322 );
+}
+
+static struct pci_device_id qib7322_nics[] = {
+	PCI_ROM ( 0x1077, 0x7322, "iba7322", "IBA7322 QDR InfiniBand HCA", 0 ),
+};
+
+struct pci_driver qib7322_driver __pci_driver = {
+	.ids = qib7322_nics,
+	.id_count = ( sizeof ( qib7322_nics ) / sizeof ( qib7322_nics[0] ) ),
+	.probe = qib7322_probe,
+	.remove = qib7322_remove,
+};